Merge pull request #58 from ndsev/end-of-stream

mapget 2024.2

.clang-tidy

@@ -1 +1 @@
-Checks: '-misc-no-recursion,-cert-err34-c'
+Checks: '-misc-no-recursion,-cert-err34-c,-readability-use-anyofallof'

.github/workflows/coverage.yml

@@ -40,10 +40,12 @@ jobs:
       run: |
         mkdir coverage
         gcovr --html-details coverage/coverage.html \
-              --filter libs/
+              --filter libs/ \
+              --gcov-ignore-parse-errors=negative_hits.warn_once_per_file
         gcovr --cobertura coverage.xml \
               --html coverage.html \
-              --filter libs/
+              --filter libs/ \
+              --gcov-ignore-parse-errors=negative_hits.warn_once_per_file
 
     - name: Publish Coverage HTML
       uses: actions/upload-artifact@v3

CMakeLists.txt

@@ -16,7 +16,7 @@ option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-servic
 project(mapget)
 include(FetchContent)
 
-set(MAPGET_VERSION 2024.1)
+set(MAPGET_VERSION 2024.2)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -162,6 +162,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
     set(WITH_BENCHMARK_TOOLS NO CACHE BOOL "rocksdb without benchmarking")
     set(BENCHMARK_ENABLE_GTEST_TESTS NO CACHE BOOL "rocksdb without gtest")
     set(DISABLE_WARNING_AS_ERROR 1 CACHE BOOL "rocksdb warnings are ok")
+    set(PORTABLE YES CACHE BOOL "rocksdb with max compatibility/portability")
 
     FetchContent_Declare(rocksdb
       GIT_REPOSITORY "https://github.com/facebook/rocksdb.git"

libs/http-service/src/http-service.cpp

@@ -24,9 +24,16 @@ struct HttpService::Impl
 
         std::stringstream buffer_;
         std::string responseType_;
+        std::unique_ptr<TileLayerStream::Writer> writer_;
         std::vector<LayerTilesRequest::Ptr> requests_;
         TileLayerStream::FieldOffsetMap fieldsOffsets_;
 
+        HttpTilesRequestState() {
+            writer_ = std::make_unique<TileLayerStream::Writer>(
+                [&, this](auto&& msg, auto&& msgType) { buffer_ << msg; },
+                fieldsOffsets_);
+        }
+
         void parseRequestFromJson(nlohmann::json const& requestJson)
         {
             std::string mapId = requestJson["mapId"];
@@ -59,12 +66,10 @@ struct HttpService::Impl
             log().debug("Response ready: {}", MapTileKey(*result).toString());
             if (responseType_ == binaryMimeType) {
                 // Binary response
-                TileLayerStream::Writer writer{
-                    [&, this](auto&& msg, auto&& msgType) { buffer_ << msg; },
-                    fieldsOffsets_};
-                writer.write(result);
+                writer_->write(result);
             }
             else {
+                // JSON response
                 buffer_ << nlohmann::to_string(result->toGeoJson())+"\n";
             }
             resultEvent_.notify_one();
@@ -147,11 +152,13 @@ struct HttpService::Impl
                     lock,
                     [&]
                     {
-                        strBuf = state->buffer_.str();
                         allDone = std::all_of(
                             state->requests_.begin(),
                             state->requests_.end(),
                             [](const auto& r) { return r->isDone(); });
+                        if (allDone)
+                            state->writer_->sendEndOfStream();
+                        strBuf = state->buffer_.str();
                         return !strBuf.empty() || allDone;
                     });
 

libs/model/include/mapget/model/attr.h

@@ -43,7 +43,7 @@ class Attribute : public simfil::ProceduralObject<2, Attribute>
     /**
      * Read-only attribute name accessor.
      */
-    std::string_view name();
+    std::string_view name() const;
 
 protected:
 

libs/model/include/mapget/model/attrlayer.h

@@ -32,6 +32,13 @@ class AttributeLayer : public simfil::Object
      */
     void addAttribute(model_ptr<Attribute> a);
 
+    /**
+     * Iterate over the stored attributes. The passed lambda must return
+     * true to continue iterating, or false to abort iteration.
+     * @return True if all attributes were visited, false if the callback ever returned false.
+     */
+    bool forEachAttribute(std::function<bool(model_ptr<Attribute> const& attr)> const& cb) const;
+
 protected:
     AttributeLayer(simfil::ArrayIndex i, simfil::ModelConstPtr l, simfil::ModelNodeAddress a);
     AttributeLayer() = default;
@@ -60,6 +67,15 @@ class AttributeLayerList : public simfil::Object
      */
     void addLayer(std::string_view const& name, model_ptr<AttributeLayer> l);
 
+    /**
+     * Iterate over the stored layers. The passed lambda must return
+     * true to continue iterating, or false to abort iteration.
+     * @return True if all layers were visited, false if the callback ever returned false.
+     */
+    bool forEachLayer(
+        std::function<bool(std::string_view, model_ptr<AttributeLayer> const& layer)> const& cb
+    ) const;
+
 protected:
     AttributeLayerList(simfil::ArrayIndex i, simfil::ModelConstPtr l, simfil::ModelNodeAddress a);
     AttributeLayerList() = default;

libs/model/include/mapget/model/featurelayer.h

@@ -30,6 +30,8 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool
     friend class Feature;
     friend class FeatureId;
     friend class Relation;
+    friend class AttributeLayer;
+    friend class AttributeLayerList;
 
 public:
     /**

libs/model/include/mapget/model/stream.h

@@ -27,6 +27,7 @@ class TileLayerStream
         None = 0,
         Fields = 1,
         TileFeatureLayer = 2,
+        EndOfStream = 128
     };
 
     struct CachedFieldsProvider;
@@ -115,6 +116,9 @@ class TileLayerStream
         /** Serialize a tile feature layer and the required part of a Fields cache. */
         void write(TileFeatureLayer::Ptr const& tileFeatureLayer);
 
+        /** Send an EndOfStream message. */
+        void sendEndOfStream();
+
     private:
         void sendMessage(std::string&& bytes, MessageType msgType);
 

libs/model/src/attr.cpp

@@ -64,7 +64,7 @@ Attribute::Direction Attribute::direction() const
     return data_->direction_;
 }
 
-std::string_view Attribute::name()
+std::string_view Attribute::name() const
 {
     if (auto s = model().fieldNames()->resolve(data_->name_))
         return *s;

libs/model/src/attrlayer.cpp

@@ -1,5 +1,6 @@
 #include "attrlayer.h"
 #include "featurelayer.h"
+#include "mapget/log.h"
 
 namespace mapget
 {
@@ -26,6 +27,22 @@ void AttributeLayer::addAttribute(model_ptr<Attribute> a)
     addField(a->name(), simfil::ModelNode::Ptr(a));
 }
 
+bool AttributeLayer::forEachAttribute(const std::function<bool(const model_ptr<Attribute>&)>& cb) const
+{
+    if (!cb)
+        return false;
+    for (auto const& [_, value] : fields()) {
+        if (value->addr().column() != TileFeatureLayer::Attributes) {
+            log().warn("Don't add anything other than Attributes into AttributeLayers!");
+            continue;
+        }
+        auto attr = reinterpret_cast<TileFeatureLayer&>(model()).resolveAttribute(*value);
+        if (!cb(attr))
+            return false;
+    }
+    return true;
+}
+
 AttributeLayerList::AttributeLayerList(
     simfil::ArrayIndex i,
     simfil::ModelConstPtr l,
@@ -48,4 +65,23 @@ void AttributeLayerList::addLayer(const std::string_view& name, model_ptr<Attrib
     addField(name, simfil::ModelNode::Ptr(std::move(l)));
 }
 
+bool AttributeLayerList::forEachLayer(
+    const std::function<bool(std::string_view, const model_ptr<AttributeLayer>&)>& cb) const
+{
+    if (!cb)
+        return false;
+    for(auto const& [fieldId, value] : fields()) {
+        if (auto layerName = model().fieldNames()->resolve(fieldId)) {
+            if (value->addr().column() != TileFeatureLayer::AttributeLayers) {
+                log().warn("Don't add anything other than AttributeLayers into AttributeLayerLists!");
+                continue;
+            }
+            auto attrLayer = reinterpret_cast<TileFeatureLayer&>(model()).resolveAttributeLayer(*value);
+            if (!cb(*layerName, attrLayer))
+                return false;
+        }
+    }
+    return true;
+}
+
 }

libs/model/src/stream.cpp

@@ -162,6 +162,11 @@ void TileLayerStream::Writer::sendMessage(std::string&& bytes, TileLayerStream::
     onMessage_(message.str(), msgType);
 }
 
+void TileLayerStream::Writer::sendEndOfStream()
+{
+    sendMessage("", MessageType::EndOfStream);
+}
+
 std::shared_ptr<Fields> TileLayerStream::CachedFieldsProvider::operator()(const std::string_view& nodeId)
 {
     {

libs/model/src/tileid.cpp

@@ -43,17 +43,19 @@ bool TileId::operator< (TileId const& other) const {
 
 TileId TileId::fromWgs84(double longitude, double latitude, uint16_t zoomLevel)
 {
+    zoomLevel = std::min((uint16_t)62, zoomLevel);
+
     longitude = fmod(longitude, LON_EXTENT) + MAX_LON;
     if (longitude < 0.)
         longitude += LON_EXTENT;
 
     // Calculate the number of subdivisions
-    auto numCols = static_cast<uint16_t>(1 << (zoomLevel + 1));
-    auto numRows = static_cast<uint16_t>(1 << zoomLevel);
+    auto numCols = static_cast<int64_t>(1ull << (zoomLevel + 1));
+    auto numRows = static_cast<int64_t>(1ull << zoomLevel);
 
     // Convert to grid coordinates
-    auto x = static_cast<int64_t>((longitude / LON_EXTENT) * numCols);
-    auto y = static_cast<int64_t>(((MAX_LAT - latitude) / LAT_EXTENT) * numRows);
+    auto x = static_cast<int64_t>((longitude / LON_EXTENT) * static_cast<double>(numCols));
+    auto y = static_cast<int64_t>(((MAX_LAT - latitude) / LAT_EXTENT) * static_cast<double>(numRows));
 
     x %= numCols;
     y %= numRows * 2;