diff --git a/src/pfaedle/osm/output/PBFWriter.cpp b/src/pfaedle/osm/output/PBFWriter.cpp index 5170b94..62afd8a 100644 --- a/src/pfaedle/osm/output/PBFWriter.cpp +++ b/src/pfaedle/osm/output/PBFWriter.cpp @@ -8,6 +8,7 @@ #include "pfaedle/_config.h" #include "pfaedle/osm/output/PBFWriter.h" +#include "util/protobuf/Protobuf.h" #ifdef ZLIB_FOUND #include #endif @@ -15,6 +16,7 @@ using pfaedle::osm::output::PBFWriter; using pfaedle::osm::source::OsmSource; using pfaedle::osm::source::PBFSource; +using namespace util::protobuf; static const size_t BUFFER_S = 33 * 1024 * 1024; static const double RES = .000000001; @@ -68,11 +70,11 @@ void PBFWriter::writeBlob(const std::string& type, auto c = _writeBuffer + 4; // type - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); writeString(type, c); // datasize - writeTypeAndId({PBFSource::VarType::V, 3}, c); + writeTypeAndId({VarType::V, 3}, c); #ifdef ZLIB_FOUND writeVarUInt(compressedLen + 2 + varUIntNumBytes(compressedLen) + varUIntNumBytes(blob.datasize), @@ -90,11 +92,11 @@ void PBFWriter::writeBlob(const std::string& type, #ifdef ZLIB_FOUND // raw uncompressed size - writeTypeAndId({PBFSource::VarType::V, 2}, c); + writeTypeAndId({VarType::V, 2}, c); writeVarUInt(blob.datasize, c); // zlib compressed data - writeTypeAndId({PBFSource::VarType::S, 3}, c); + writeTypeAndId({VarType::S, 3}, c); writeVarUInt(compressedLen, c); // flush write buffer, which now contains BlobHeader and Blob w/o data @@ -111,7 +113,7 @@ void PBFWriter::writeBlob(const std::string& type, } #else // if zlib not available, write raw - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); writeVarUInt(compressedLen, c); // flush write buffer, which now contains BlobHeader and Blob w/o data @@ -139,7 +141,7 @@ void PBFWriter::writeDenseNodes() { auto c = _blockBuffer; // string table - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); if (_strings.size() == 0) { checkBlockBufferBounds(c + 1); writeVarUInt(0, c); @@ -147,14 +149,14 @@ void PBFWriter::writeDenseNodes() { checkBlockBufferBounds(c + varUIntNumBytes(_stringTable.size() + 2) + 1 + 1 + _stringTable.size()); writeVarUInt(_stringTable.size() + 2, c); - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); writeVarUInt(0, c); writeBuf(_stringTable.data(), _stringTable.size(), c); } // primitive group checkBlockBufferBounds(c + 1); - writeTypeAndId({PBFSource::VarType::S, 2}, c); + writeTypeAndId({VarType::S, 2}, c); size_t groupSize = 0; groupSize += _denseNdIds.size() + varUIntNumBytes(_denseNdIds.size()) + 1; groupSize += _denseNdLats.size() + varUIntNumBytes(_denseNdLats.size()) + 1; @@ -166,34 +168,34 @@ void PBFWriter::writeDenseNodes() { // dense nodes checkBlockBufferBounds(c + 1 + varUIntNumBytes(groupSize)); - writeTypeAndId({PBFSource::VarType::S, 2}, c); + writeTypeAndId({VarType::S, 2}, c); writeVarUInt(groupSize, c); // ids checkBlockBufferBounds(c + 1 + varUIntNumBytes(_denseNdIds.size()) + _denseNdIds.size()); - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); writeVarUInt(_denseNdIds.size(), c); writeBuf(_denseNdIds.data(), _denseNdIds.size(), c); // lats checkBlockBufferBounds(c + 1 + varUIntNumBytes(_denseNdLats.size()) + _denseNdLats.size()); - writeTypeAndId({PBFSource::VarType::S, 8}, c); + writeTypeAndId({VarType::S, 8}, c); writeVarUInt(_denseNdLats.size(), c); writeBuf(_denseNdLats.data(), _denseNdLats.size(), c); // lngs checkBlockBufferBounds(c + 1 + varUIntNumBytes(_denseNdLngs.size()) + _denseNdLngs.size()); - writeTypeAndId({PBFSource::VarType::S, 9}, c); + writeTypeAndId({VarType::S, 9}, c); writeVarUInt(_denseNdLngs.size(), c); writeBuf(_denseNdLngs.data(), _denseNdLngs.size(), c); // attrs checkBlockBufferBounds(c + 1 + varUIntNumBytes(_denseNdAttrs.size()) + _denseNdAttrs.size()); - writeTypeAndId({PBFSource::VarType::S, 10}, c); + writeTypeAndId({VarType::S, 10}, c); writeVarUInt(_denseNdAttrs.size(), c); writeBuf(_denseNdAttrs.data(), _denseNdAttrs.size(), c); @@ -223,7 +225,7 @@ void PBFWriter::writeWays() { // string table checkBlockBufferBounds(c + 1); - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); if (_strings.size() == 0) { checkBlockBufferBounds(c + 1); writeVarUInt(0, c); @@ -231,14 +233,14 @@ void PBFWriter::writeWays() { checkBlockBufferBounds(c + varUIntNumBytes(_stringTable.size() + 2) + 1 + 1 + _stringTable.size()); writeVarUInt(_stringTable.size() + 2, c); - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); writeVarUInt(0, c); writeBuf(_stringTable.data(), _stringTable.size(), c); } // primitive group checkBlockBufferBounds(c + 1 + varUIntNumBytes(_ways.size()) + _ways.size()); - writeTypeAndId({PBFSource::VarType::S, 2}, c); + writeTypeAndId({VarType::S, 2}, c); writeVarUInt(_ways.size(), c); // ways @@ -263,7 +265,7 @@ void PBFWriter::writeRels() { // string table checkBlockBufferBounds(c + 1); - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); if (_strings.size() == 0) { checkBlockBufferBounds(c + 1); writeVarUInt(0, c); @@ -271,14 +273,14 @@ void PBFWriter::writeRels() { checkBlockBufferBounds(c + varUIntNumBytes(_stringTable.size() + 2) + 1 + 1 + _stringTable.size()); writeVarUInt(_stringTable.size() + 2, c); - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); writeVarUInt(0, c); writeBuf(_stringTable.data(), _stringTable.size(), c); } // primitive group checkBlockBufferBounds(c + 1 + varUIntNumBytes(_rels.size()) + _rels.size()); - writeTypeAndId({PBFSource::VarType::S, 2}, c); + writeTypeAndId({VarType::S, 2}, c); writeVarUInt(_rels.size(), c); // rels @@ -301,29 +303,29 @@ const PBFSource::Blob PBFWriter::writeOSMHeader( auto c = _blockBuffer; // header bbox - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); size_t s = varIntNumBytes(header.bbox.getLowerLeft().getX() / RES) + varIntNumBytes(header.bbox.getLowerLeft().getY() / RES) + varIntNumBytes(header.bbox.getUpperRight().getX() / RES) + varIntNumBytes(header.bbox.getUpperRight().getY() / RES) + 4; writeVarUInt(s, c); - writeTypeAndId({PBFSource::VarType::V, 1}, c); + writeTypeAndId({VarType::V, 1}, c); writeVarInt(header.bbox.getLowerLeft().getX() / RES, c); - writeTypeAndId({PBFSource::VarType::V, 4}, c); + writeTypeAndId({VarType::V, 4}, c); writeVarInt(header.bbox.getLowerLeft().getY() / RES, c); - writeTypeAndId({PBFSource::VarType::V, 2}, c); + writeTypeAndId({VarType::V, 2}, c); writeVarInt(header.bbox.getUpperRight().getX() / RES, c); - writeTypeAndId({PBFSource::VarType::V, 3}, c); + writeTypeAndId({VarType::V, 3}, c); writeVarInt(header.bbox.getUpperRight().getY() / RES, c); // writing program - writeTypeAndId({PBFSource::VarType::S, 16}, c); + writeTypeAndId({VarType::S, 16}, c); writeString(header.writingProgram, c); // features - writeTypeAndId({PBFSource::VarType::S, 4}, c); + writeTypeAndId({VarType::S, 4}, c); writeString("OsmSchema-V0.6", c); - writeTypeAndId({PBFSource::VarType::S, 4}, c); + writeTypeAndId({VarType::S, 4}, c); writeString("DenseNodes", c); ret.datasize = c - _blockBuffer; @@ -331,68 +333,6 @@ const PBFSource::Blob PBFWriter::writeOSMHeader( return ret; } -// _____________________________________________________________________________ -size_t PBFWriter::varUIntNumBytes(uint64_t val) const { - if (val == 0) return 1; - // 7 bit payload per byte - return ceil((floor(log2(val)) + 1) / 7); -} - -// _____________________________________________________________________________ -size_t PBFWriter::varIntNumBytes(int64_t val) const { - // 7 bit payload per byte - return varUIntNumBytes((static_cast(val) << 1) ^ - static_cast(val >> 63)); -} - -// _____________________________________________________________________________ -void PBFWriter::writeVarInt(int64_t val, unsigned char*& c) const { - writeVarUInt( - (static_cast(val) << 1) ^ static_cast(val >> 63), c); -} - -// _____________________________________________________________________________ -void PBFWriter::writeVarUInt(uint64_t val, unsigned char*& c) const { - while (val >= 128) { // as long as we have a value that doesnt fit in 7 bits - // take one byte, ignore highest bit and set to 1 (continue) - *c = static_cast(val) | 128; - c++; - // shift by the written 7 bits - val = val >> 7; - } - - // remaining last byte, with unset continuation bit - *c = static_cast(val); - c++; -} - -// _____________________________________________________________________________ -void PBFWriter::writeString(const std::string& str, unsigned char*& c) const { - writeVarUInt(str.size(), c); - memcpy(c, str.c_str(), str.size()); - c += str.size(); -} - -// _____________________________________________________________________________ -void PBFWriter::writeBuf(const unsigned char* src, size_t s, - unsigned char*& c) const { - memcpy(c, src, s); - c += s; -} - -// _____________________________________________________________________________ -void PBFWriter::writeTypeAndId( - const std::pair& typeId, - unsigned char*& c) const { - uint64_t byte = 0; - // set type - byte |= typeId.first; - // set id - byte |= (static_cast(typeId.second) << 3); - - writeVarUInt(byte, c); -} - // _____________________________________________________________________________ void PBFWriter::writeWay(const OsmWay& w) { // make sure dense nodes are flushed @@ -428,24 +368,24 @@ void PBFWriter::writeWay(const OsmWay& w) { varUIntNumBytes(_curKeys.size()) + 1 + _curKeys.size() + varUIntNumBytes(_curVals.size()) + 1 + _curVals.size() + varUIntNumBytes(_curRefs.size()) + 1 + _curRefs.size()); - writeTypeAndId({PBFSource::VarType::V, 1}, c); + writeTypeAndId({VarType::V, 1}, c); writeVarUInt(w.id, c); - writeTypeAndId({PBFSource::VarType::S, 2}, c); + writeTypeAndId({VarType::S, 2}, c); writeVarUInt(_curKeys.size(), c); writeBuf(_curKeys.data(), _curKeys.size(), c); - writeTypeAndId({PBFSource::VarType::S, 3}, c); + writeTypeAndId({VarType::S, 3}, c); writeVarUInt(_curVals.size(), c); writeBuf(_curVals.data(), _curVals.size(), c); - writeTypeAndId({PBFSource::VarType::S, 8}, c); + writeTypeAndId({VarType::S, 8}, c); writeVarUInt(_curRefs.size(), c); writeBuf(_curRefs.data(), _curRefs.size(), c); // write way to ways c = need(_ways, 1 + varUIntNumBytes(_curWay.size()) + _curWay.size()); - writeTypeAndId({PBFSource::VarType::S, 3}, c); + writeTypeAndId({VarType::S, 3}, c); writeVarUInt(_curWay.size(), c); writeBuf(_curWay.data(), _curWay.size(), c); @@ -562,32 +502,32 @@ void PBFWriter::writeRel(const OsmRel& rel, const OsmIdList& nodes, _curRefs.size() + varUIntNumBytes(_curRoles.size()) + 1 + _curRoles.size() + varUIntNumBytes(_curTypes.size()) + 1 + _curTypes.size()); - writeTypeAndId({PBFSource::VarType::V, 1}, c); + writeTypeAndId({VarType::V, 1}, c); writeVarUInt(rel.id, c); - writeTypeAndId({PBFSource::VarType::S, 2}, c); + writeTypeAndId({VarType::S, 2}, c); writeVarUInt(_curKeys.size(), c); writeBuf(_curKeys.data(), _curKeys.size(), c); - writeTypeAndId({PBFSource::VarType::S, 3}, c); + writeTypeAndId({VarType::S, 3}, c); writeVarUInt(_curVals.size(), c); writeBuf(_curVals.data(), _curVals.size(), c); - writeTypeAndId({PBFSource::VarType::S, 9}, c); + writeTypeAndId({VarType::S, 9}, c); writeVarUInt(_curRefs.size(), c); writeBuf(_curRefs.data(), _curRefs.size(), c); - writeTypeAndId({PBFSource::VarType::S, 8}, c); + writeTypeAndId({VarType::S, 8}, c); writeVarUInt(_curRoles.size(), c); writeBuf(_curRoles.data(), _curRoles.size(), c); - writeTypeAndId({PBFSource::VarType::S, 10}, c); + writeTypeAndId({VarType::S, 10}, c); writeVarUInt(_curTypes.size(), c); writeBuf(_curTypes.data(), _curTypes.size(), c); // write rel to rels c = need(_rels, 1 + varUIntNumBytes(_curRel.size()) + _curRel.size()); - writeTypeAndId({PBFSource::VarType::S, 4}, c); + writeTypeAndId({VarType::S, 4}, c); writeVarUInt(_curRel.size(), c); writeBuf(_curRel.data(), _curRel.size(), c); @@ -606,7 +546,7 @@ size_t PBFWriter::getStringId(const std::string& str) { auto kStr = _strings.find(str); if (kStr == _strings.end()) { auto c = need(_stringTable, varUIntNumBytes(str.size()) + str.size() + 1); - writeTypeAndId({PBFSource::VarType::S, 1}, c); + writeTypeAndId({VarType::S, 1}, c); writeString(str, c); kStr = _strings .insert(std::pair{str, _strings.size() + 1}) diff --git a/src/pfaedle/osm/output/PBFWriter.h b/src/pfaedle/osm/output/PBFWriter.h index 342aeaa..c186f5d 100644 --- a/src/pfaedle/osm/output/PBFWriter.h +++ b/src/pfaedle/osm/output/PBFWriter.h @@ -51,17 +51,8 @@ class PBFWriter : public OsmWriter { void writeBlob(const std::string& type, const source::PBFSource::Blob& blob); const source::PBFSource::Blob writeOSMHeader( const source::PBFSource::OSMHeader& header); - void writeTypeAndId( - const std::pair& typeId, - unsigned char*& c) const; - void writeVarUInt(uint64_t val, unsigned char*& c) const; - size_t varUIntNumBytes(uint64_t val) const; - void writeVarInt(int64_t val, unsigned char*& c) const; - size_t varIntNumBytes(int64_t val) const; - void writeString(const std::string& str, unsigned char*& c) const; size_t getStringId(const std::string& str); unsigned char* need(std::vector& v, size_t n) const; - void writeBuf(const unsigned char* src, size_t s, unsigned char*& c) const; void checkBlockBufferBounds(const unsigned char* c) const; }; diff --git a/src/pfaedle/osm/source/PBFSource.cpp b/src/pfaedle/osm/source/PBFSource.cpp index 4747b0a..bab27ee 100644 --- a/src/pfaedle/osm/source/PBFSource.cpp +++ b/src/pfaedle/osm/source/PBFSource.cpp @@ -8,6 +8,7 @@ #include "pfaedle/osm/source/PBFSource.h" #include "util/Misc.h" +#include "util/protobuf/Protobuf.h" #ifdef ZLIB_FOUND #include #endif @@ -19,6 +20,7 @@ using pfaedle::osm::source::OsmSourceRelationMember; using pfaedle::osm::source::OsmSourceWay; using pfaedle::osm::source::PBFSource; using util::readAll; +using namespace util::protobuf; static const size_t BUFFER_S = 33 * 1024 * 1024; static const double RES = .000000001; @@ -80,7 +82,7 @@ bool PBFSource::getNextBlock() { // _____________________________________________________________________________ util::geo::Box PBFSource::parseHeaderBBox(unsigned char*& c) { - auto len = parseVarUInt(c); + auto len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); @@ -90,16 +92,16 @@ util::geo::Box PBFSource::parseHeaderBBox(unsigned char*& c) { double urx = 180; while ((size_t)(c - start) < len) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) - llx = RES * parseInt(typeId, c); + llx = RES * util::protobuf::parseInt(typeId, c); else if (typeId.second == 4) - lly = RES * parseInt(typeId, c); + lly = RES * util::protobuf::parseInt(typeId, c); else if (typeId.second == 2) - urx = RES * parseInt(typeId, c); + urx = RES * util::protobuf::parseInt(typeId, c); else if (typeId.second == 3) - ury = RES * parseInt(typeId, c); + ury = RES * util::protobuf::parseInt(typeId, c); else skipType(typeId.first, c); } @@ -116,21 +118,21 @@ PBFSource::PrimitiveBlock PBFSource::parseOSMData(const Blob& blob) { PBFSource::PrimitiveBlock block; while ((size_t)(c - start) < blob.datasize) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) { block.stringTable = parseStringTable(c); } else if (typeId.second == 2) { block.primitiveGroups.push(c); - c += parseVarUInt(c); + c += util::protobuf::parseVarUInt(c); } else if (typeId.second == 17) { - block.granularity = parseUInt(typeId, c); + block.granularity = util::protobuf::parseUInt(typeId, c); } else if (typeId.second == 19) { - block.latOffset = parseUInt(typeId, c); + block.latOffset = util::protobuf::parseUInt(typeId, c); } else if (typeId.second == 20) { - block.lonOffset = parseUInt(typeId, c); + block.lonOffset = util::protobuf::parseUInt(typeId, c); } else if (typeId.second == 18) { - block.dateGranularity = parseUInt(typeId, c); + block.dateGranularity = util::protobuf::parseUInt(typeId, c); } else { skipType(typeId.first, c); } @@ -141,7 +143,7 @@ PBFSource::PrimitiveBlock PBFSource::parseOSMData(const Blob& blob) { // _____________________________________________________________________________ std::vector PBFSource::parseStringTable(unsigned char*& c) { - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); @@ -149,9 +151,9 @@ std::vector PBFSource::parseStringTable(unsigned char*& c) { table.reserve(len / 10); while ((size_t)(c - start) < len) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) - table.push_back(parseString(c)); + table.push_back(util::protobuf::parseString(c)); else skipType(typeId.first, c); } @@ -170,16 +172,16 @@ PBFSource::OSMHeader PBFSource::parseOSMHeader(const Blob& blob) { header.bbox = {{-180, -90}, {180, 90}}; while ((size_t)(c - start) < blob.datasize) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) header.bbox = parseHeaderBBox(c); else if (typeId.second == 4) - header.requiredFeatures.push_back(parseString(c)); + header.requiredFeatures.push_back(util::protobuf::parseString(c)); else if (typeId.second == 5) - header.optionalFeatures.push_back(parseString(c)); + header.optionalFeatures.push_back(util::protobuf::parseString(c)); else if (typeId.second == 16) - header.writingProgram = parseString(c); + header.writingProgram = util::protobuf::parseString(c); else skipType(typeId.first, c); } @@ -213,7 +215,7 @@ PBFSource::Blob PBFSource::parseBlob(size_t len) { } else if (typeId.second == 3) { #ifdef ZLIB_FOUND // ZLIB compression - if (typeId.first != PBFSource::VarType::S) { + if (typeId.first != VarType::S) { throw std::runtime_error(_path + ": " + "expected byte array value"); } size_t size = parseVarUInt(); @@ -263,150 +265,62 @@ PBFSource::BlobHeader PBFSource::parseBlobHeader(size_t len) { } // _____________________________________________________________________________ -uint32_t PBFSource::parseFixedUInt32() { return parseFixedUInt32(_c); } - -// _____________________________________________________________________________ -uint32_t PBFSource::parseFixedUInt32(unsigned char*& c) { - // manual assembly to ensure correct endianess - uint32_t ret = - (static_cast(static_cast(c[0]))) | - (static_cast(static_cast(c[1])) << 8) | - (static_cast(static_cast(c[2])) << 16) | - (static_cast(static_cast(c[3])) << 24); - c += 4; - return ret; -} - -// _____________________________________________________________________________ -uint64_t PBFSource::parseFixedUInt64() { return parseFixedUInt64(_c); } - -// _____________________________________________________________________________ -uint64_t PBFSource::parseFixedUInt64(unsigned char*& c) { - // manual assembly to ensure correct endianess - uint64_t ret = - (static_cast(static_cast(c[0]))) | - (static_cast(static_cast(c[1])) << 8) | - (static_cast(static_cast(c[2])) << 16) | - (static_cast(static_cast(c[3])) << 24) | - (static_cast(static_cast(c[4])) << 32) | - (static_cast(static_cast(c[5])) << 40) | - (static_cast(static_cast(c[6])) << 48) | - (static_cast(static_cast(c[7])) << 56); - c += 8; - return ret; -} - -// _____________________________________________________________________________ -int32_t PBFSource::parseFixedInt32() { return parseFixedInt32(_c); } - -// _____________________________________________________________________________ -int32_t PBFSource::parseFixedInt32(unsigned char*& c) { - uint32_t i = parseFixedUInt32(c); - return (i >> 1) ^ (-(i & 1)); +uint32_t PBFSource::parseFixedUInt32() { + return util::protobuf::parseFixedUInt32(_c); } // _____________________________________________________________________________ -int64_t PBFSource::parseFixedInt64() { return parseFixedInt64(_c); } - -// _____________________________________________________________________________ -int64_t PBFSource::parseFixedInt64(unsigned char*& c) { - uint32_t i = parseFixedUInt64(c); - return (i >> 1) ^ (-(i & 1)); +uint64_t PBFSource::parseFixedUInt64() { + return util::protobuf::parseFixedUInt64(_c); } // _____________________________________________________________________________ -std::string PBFSource::parseString() { return parseString(_c); } - -// _____________________________________________________________________________ -std::string PBFSource::parseString(unsigned char*& c) { - auto len = parseVarUInt(c); - std::string ret = {reinterpret_cast(c), len}; - c += len; - return ret; +int32_t PBFSource::parseFixedInt32() { + return util::protobuf::parseFixedInt32(_c); } // _____________________________________________________________________________ -uint64_t PBFSource::parseUInt(std::pair typeId, - unsigned char*& c) { - if (typeId.first == PBFSource::VarType::I) return parseFixedUInt32(c); - if (typeId.first == PBFSource::VarType::D) return parseFixedUInt64(c); - if (typeId.first == PBFSource::VarType::V) return parseVarUInt(c); - throw std::runtime_error(_path + ": " + "expected unsigned integer value"); +int64_t PBFSource::parseFixedInt64() { + return util::protobuf::parseFixedInt64(_c); } // _____________________________________________________________________________ -int64_t PBFSource::parseInt(std::pair typeId, - unsigned char*& c) { - if (typeId.first == PBFSource::VarType::I) return parseFixedInt32(c); - if (typeId.first == PBFSource::VarType::D) return parseFixedInt64(c); - if (typeId.first == PBFSource::VarType::V) return parseVarInt(c); - throw std::runtime_error(_path + ": " + "expected signed integer value"); -} +std::string PBFSource::parseString() { return util::protobuf::parseString(_c); } // _____________________________________________________________________________ uint64_t PBFSource::parseUInt(std::pair typeId) { - return parseUInt(typeId, _c); + return util::protobuf::parseUInt(typeId, _c); } // _____________________________________________________________________________ int64_t PBFSource::parseInt(std::pair typeId) { - return parseInt(typeId, _c); -} - -// _____________________________________________________________________________ -int64_t PBFSource::parseVarInt(unsigned char*& c) { - int64_t i = parseVarUInt(c); - return (i >> 1) ^ (-(i & 1)); -} - -// _____________________________________________________________________________ -int64_t PBFSource::parseVarInt() { return parseVarInt(_c); } - -// _____________________________________________________________________________ -uint64_t PBFSource::parseVarUInt(unsigned char*& c) { - int i = 0; - uint64_t ret = 0; - uint64_t cur; - - do { - cur = *c; - ret |= (uint64_t)((uint8_t)(cur << 1) >> 1) << (i * 7); - i++; - c++; - } while (cur & (1 << 7)); - - return ret; + return util::protobuf::parseInt(typeId, _c); } // _____________________________________________________________________________ -uint64_t PBFSource::parseVarUInt() { return parseVarUInt(_c); } +int64_t PBFSource::parseVarInt() { return util::protobuf::parseVarInt(_c); } // _____________________________________________________________________________ -std::pair PBFSource::nextTypeAndId() { - return nextTypeAndId(_c); -} +uint64_t PBFSource::parseVarUInt() { return util::protobuf::parseVarUInt(_c); } // _____________________________________________________________________________ -std::pair PBFSource::nextTypeAndId( - unsigned char*& c) { - uint64_t varInt = parseVarUInt(c); - auto type = PBFSource::VarType(((unsigned char)(varInt << 5) >> 5)); - return {type, (varInt << 1) >> 4}; +std::pair PBFSource::nextTypeAndId() { + return util::protobuf::nextTypeAndId(_c); } // _____________________________________________________________________________ -void PBFSource::skipType(PBFSource::VarType type) { skipType(type, _c); } +void PBFSource::skipType(VarType type) { skipType(type, _c); } // _____________________________________________________________________________ -void PBFSource::skipType(PBFSource::VarType type, unsigned char*& c) { - if (type == PBFSource::VarType::V) - parseVarUInt(c); - else if (type == PBFSource::VarType::D) - parseFixedUInt64(c); - else if (type == PBFSource::VarType::S) - parseString(c); - else if (type == PBFSource::VarType::I) - parseFixedUInt32(c); +void PBFSource::skipType(VarType type, unsigned char*& c) { + if (type == VarType::V) + util::protobuf::parseVarUInt(c); + else if (type == VarType::D) + util::protobuf::parseFixedUInt64(c); + else if (type == VarType::S) + util::protobuf::parseString(c); + else if (type == VarType::I) + util::protobuf::parseFixedUInt32(c); else throw std::runtime_error(_path + ": " + "parse error"); } @@ -432,7 +346,7 @@ const OsmSourceNode* PBFSource::nextNode() { } while (checkGroup()) { - auto typeId = nextTypeAndId(_curBlock.c); + auto typeId = util::protobuf::nextTypeAndId(_curBlock.c); if (typeId.second == 1) { parseNode(_curBlock.c); _curAttr = 0; @@ -443,9 +357,9 @@ const OsmSourceNode* PBFSource::nextNode() { _curBlock.denseNodePtr = 0; return nextNode(); } else if (typeId.second == 4) { - skip(parseVarUInt(_curBlock.c)); + skip(util::protobuf::parseVarUInt(_curBlock.c)); } else { - skip(parseVarUInt(_curBlock.c)); + skip(util::protobuf::parseVarUInt(_curBlock.c)); } } @@ -484,7 +398,7 @@ bool PBFSource::checkGroup() { if (_curBlock.c == 0) { _curBlock.c = _curBlock.primitiveGroups.front(); - _curBlock.curGroupLen = parseVarUInt(_curBlock.c); + _curBlock.curGroupLen = util::protobuf::parseVarUInt(_curBlock.c); _curBlock.primitiveGroups.front() = _curBlock.c; } @@ -502,7 +416,7 @@ bool PBFSource::checkGroup() { // _____________________________________________________________________________ void PBFSource::parseWay(unsigned char*& c) { // normal node - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); @@ -515,42 +429,42 @@ void PBFSource::parseWay(unsigned char*& c) { _curWayMember = 0; while ((size_t)(c - start) < len) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) { // ID - _curWay.id = parseUInt(typeId, c); + _curWay.id = util::protobuf::parseUInt(typeId, c); } else if (typeId.second == 8) { // member IDs - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { - auto memberId = parseVarInt(c); + auto memberId = util::protobuf::parseVarInt(c); if (curMemberID > 0) memberId = _curWayMembers[curMemberID - 1] + memberId; if (curMemberID >= _curWayMembers.size()) _curWayMembers.push_back(0); _curWayMembers[curMemberID++] = memberId; } } else if (typeId.second == 2) { // attr keys - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { if (curKey >= _curAttrs.size()) _curAttrs.push_back({0, 0}); - _curAttrs[curKey++].first = parseVarUInt(c); + _curAttrs[curKey++].first = util::protobuf::parseVarUInt(c); } } else if (typeId.second == 3) { // attr values - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { if (curValue >= _curAttrs.size()) _curAttrs.push_back({0, 0}); - _curAttrs[curValue++].second = parseVarUInt(c); + _curAttrs[curValue++].second = util::protobuf::parseVarUInt(c); } } else { - c += parseVarUInt(c); + c += util::protobuf::parseVarUInt(c); } } } @@ -558,7 +472,7 @@ void PBFSource::parseWay(unsigned char*& c) { // _____________________________________________________________________________ void PBFSource::parseRelation(unsigned char*& c) { // normal node - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); @@ -573,12 +487,12 @@ void PBFSource::parseRelation(unsigned char*& c) { _curRelMember = 0; while ((size_t)(c - start) < len) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) { // ID - _curRelation.id = parseVarUInt(c); + _curRelation.id = util::protobuf::parseVarUInt(c); } else if (typeId.second == 8) { // roles - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); @@ -586,15 +500,15 @@ void PBFSource::parseRelation(unsigned char*& c) { if (curRole >= _curRelMembers.size()) _curRelMembers.push_back({0, 0, 0}); _curRelMembers[curRole++].role = - _curBlock.stringTable[parseVarUInt(c)].c_str(); + _curBlock.stringTable[util::protobuf::parseVarUInt(c)].c_str(); } } else if (typeId.second == 9) { // member IDs - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { - auto memberId = parseVarInt(c); + auto memberId = util::protobuf::parseVarInt(c); if (curMemberID > 0) memberId = _curRelMembers[curMemberID - 1].id + memberId; if (curMemberID >= _curRelMembers.size()) @@ -602,35 +516,35 @@ void PBFSource::parseRelation(unsigned char*& c) { _curRelMembers[curMemberID++].id = memberId; } } else if (typeId.second == 10) { // member types - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { if (curType >= _curRelMembers.size()) _curRelMembers.push_back({0, 0, 0}); - _curRelMembers[curType++].type = parseVarUInt(c); + _curRelMembers[curType++].type = util::protobuf::parseVarUInt(c); } } else if (typeId.second == 2) { // attr keys - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { if (curKey >= _curAttrs.size()) _curAttrs.push_back({0, 0}); - _curAttrs[curKey++].first = parseVarUInt(c); + _curAttrs[curKey++].first = util::protobuf::parseVarUInt(c); } } else if (typeId.second == 3) { // attr values - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { if (curValue >= _curAttrs.size()) _curAttrs.push_back({0, 0}); - _curAttrs[curValue++].second = parseVarUInt(c); + _curAttrs[curValue++].second = util::protobuf::parseVarUInt(c); } } else { - c += parseVarUInt(c); + c += util::protobuf::parseVarUInt(c); } } } @@ -638,7 +552,7 @@ void PBFSource::parseRelation(unsigned char*& c) { // _____________________________________________________________________________ void PBFSource::parseNode(unsigned char*& c) { // normal node - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); @@ -648,36 +562,38 @@ void PBFSource::parseNode(unsigned char*& c) { _curAttrs = {}; while ((size_t)(c - start) < len) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) { // ID - _curNode.id = parseInt(typeId, c); + _curNode.id = util::protobuf::parseInt(typeId, c); } else if (typeId.second == 8) { // lat - _curNode.lat = RES * (_curBlock.latOffset + - (parseInt(typeId, c) * _curBlock.granularity)); + _curNode.lat = + RES * (_curBlock.latOffset + + ((double)util::protobuf::parseInt(typeId, c) * _curBlock.granularity)); } else if (typeId.second == 9) { // lon - _curNode.lon = RES * (_curBlock.lonOffset + - (parseInt(typeId, c) * _curBlock.granularity)); + _curNode.lon = + RES * (_curBlock.lonOffset + + ((double)util::protobuf::parseInt(typeId, c) * _curBlock.granularity)); } else if (typeId.second == 2) { // attr keys - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { if (curKey >= _curAttrs.size()) _curAttrs.push_back({0, 0}); - _curAttrs[curKey++].first = parseVarUInt(c); + _curAttrs[curKey++].first = util::protobuf::parseVarUInt(c); } } else if (typeId.second == 3) { // attr values - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); while ((size_t)(c - start) < len) { if (curValue >= _curAttrs.size()) _curAttrs.push_back({0, 0}); - _curAttrs[curValue++].second = parseVarUInt(c); + _curAttrs[curValue++].second = util::protobuf::parseVarUInt(c); } } else { - c += parseVarUInt(c); + c += util::protobuf::parseVarUInt(c); } } } @@ -686,7 +602,7 @@ void PBFSource::parseNode(unsigned char*& c) { std::vector PBFSource::parseDenseNodes(unsigned char*& c) { std::vector ret; - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); @@ -696,16 +612,16 @@ std::vector PBFSource::parseDenseNodes(unsigned char*& c) { size_t curTagPos = 0; while ((size_t)(c - start) < len) { - auto typeId = nextTypeAndId(c); + auto typeId = util::protobuf::nextTypeAndId(c); if (typeId.second == 1) { // IDs - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); // delta encoded while ((size_t)(c - start) < len) { - uint64_t nid = parseVarInt(c); + uint64_t nid = util::protobuf::parseVarInt(c); if (curIdPos > 0) nid = ret[curIdPos - 1].id + nid; if (curIdPos >= ret.size()) @@ -716,15 +632,15 @@ std::vector PBFSource::parseDenseNodes(unsigned char*& c) { curIdPos++; } } else if (typeId.second == 5) { // denseinfo - c += parseVarUInt(c); + c += util::protobuf::parseVarUInt(c); } else if (typeId.second == 8) { // lat - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); // delta encoded while ((size_t)(c - start) < len) { - double lat = parseVarInt(c); + double lat = util::protobuf::parseVarInt(c); if (curLatPos > 0) lat = ret[curLatPos - 1].lat + lat; if (curLatPos >= ret.size()) @@ -735,13 +651,13 @@ std::vector PBFSource::parseDenseNodes(unsigned char*& c) { curLatPos++; } } else if (typeId.second == 9) { // lon - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); // delta encoded while ((size_t)(c - start) < len) { - double lon = parseVarInt(c); + double lon = util::protobuf::parseVarInt(c); if (curLonPos > 0) lon = ret[curLonPos - 1].lon + lon; if (curLonPos >= ret.size()) @@ -752,20 +668,20 @@ std::vector PBFSource::parseDenseNodes(unsigned char*& c) { curLonPos++; } } else if (typeId.second == 10) { // attrs - size_t len = parseVarUInt(c); + size_t len = util::protobuf::parseVarUInt(c); auto start = c; checkBufferBounds(c + len); // delta encoded while ((size_t)(c - start) < len) { - auto key = parseVarUInt(c); + auto key = util::protobuf::parseVarUInt(c); if (key == 0) { curTagPos++; continue; } if (curTagPos >= ret.size()) ret.push_back({0, 0, 0, {}}); - ret[curTagPos].tags.push_back({key, parseVarUInt(c)}); + ret[curTagPos].tags.push_back({key, util::protobuf::parseVarUInt(c)}); } } } @@ -779,7 +695,7 @@ void PBFSource::skip(size_t n) { _curBlock.c += n; } // _____________________________________________________________________________ const OsmSourceWay* PBFSource::nextWay() { while (checkGroup()) { - auto typeId = nextTypeAndId(_curBlock.c); + auto typeId = util::protobuf::nextTypeAndId(_curBlock.c); if (typeId.second == 3) { parseWay(_curBlock.c); @@ -787,7 +703,7 @@ const OsmSourceWay* PBFSource::nextWay() { _curAttrsPtr = &_curAttrs; return &_curWay; } else { - skip(parseVarUInt(_curBlock.c)); + skip(util::protobuf::parseVarUInt(_curBlock.c)); } } @@ -811,7 +727,7 @@ uint64_t PBFSource::nextMemberNode() { // _____________________________________________________________________________ const OsmSourceRelation* PBFSource::nextRel() { while (checkGroup()) { - auto typeId = nextTypeAndId(_curBlock.c); + auto typeId = util::protobuf::nextTypeAndId(_curBlock.c); if (typeId.second == 4) { parseRelation(_curBlock.c); @@ -819,7 +735,7 @@ const OsmSourceRelation* PBFSource::nextRel() { _curAttrsPtr = &_curAttrs; return &_curRelation; } else { - skip(parseVarUInt(_curBlock.c)); + skip(util::protobuf::parseVarUInt(_curBlock.c)); } } diff --git a/src/pfaedle/osm/source/PBFSource.h b/src/pfaedle/osm/source/PBFSource.h index 896a83e..6c9d1ff 100644 --- a/src/pfaedle/osm/source/PBFSource.h +++ b/src/pfaedle/osm/source/PBFSource.h @@ -9,20 +9,16 @@ #include "pfaedle/osm/source/OsmSource.h" #include "util/geo/Geo.h" +#include "util/protobuf/Protobuf.h" namespace pfaedle { namespace osm { namespace source { +using util::protobuf::VarType; + class PBFSource : public OsmSource { public: - enum VarType : uint8_t { - V = 0, - D = 1, - S = 2, - I = 5, - }; - struct Node { double lat, lon; uint64_t id; @@ -98,25 +94,15 @@ class PBFSource : public OsmSource { std::vector parseStringTable(unsigned char*& c); Blob parseBlob(size_t len); std::pair nextTypeAndId(); - std::pair nextTypeAndId(unsigned char*& c); std::string parseString(); - std::string parseString(unsigned char*& c); uint32_t parseFixedUInt32(); - uint32_t parseFixedUInt32(unsigned char*& c); uint64_t parseFixedUInt64(); - uint64_t parseFixedUInt64(unsigned char*& c); int32_t parseFixedInt32(); - int32_t parseFixedInt32(unsigned char*& c); int64_t parseFixedInt64(); - int64_t parseFixedInt64(unsigned char*& c); int64_t parseVarInt(); - int64_t parseVarInt(unsigned char*& c); uint64_t parseVarUInt(); - uint64_t parseVarUInt(unsigned char*& c); uint64_t parseUInt(std::pair typeId); int64_t parseInt(std::pair typeId); - uint64_t parseUInt(std::pair typeId, unsigned char*& c); - int64_t parseInt(std::pair typeId, unsigned char*& c); void skipType(VarType type, unsigned char*& c); void skipType(VarType type); void skip(size_t n); diff --git a/src/util b/src/util index 6b4a60c..205dbe6 160000 --- a/src/util +++ b/src/util @@ -1 +1 @@ -Subproject commit 6b4a60c2d4e966dee56fc4a614c42718ce7c7cdf +Subproject commit 205dbe6b415db9c7419f68f22cd40fef6a7da387