Merge pull request #62 from ndsev/addon-datasources

Addon Datasources

.clang-tidy

@@ -1 +1 @@
-Checks: '-misc-no-recursion,-cert-err34-c,-readability-use-anyofallof'
+Checks: '-misc-no-recursion,-cert-err34-c,-readability-use-anyofallof,-performance-avoid-endl,-bugprone-chained-comparison'

CMakeLists.txt

@@ -16,7 +16,7 @@ option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-servic
 project(mapget)
 include(FetchContent)
 
-set(MAPGET_VERSION 2024.2)
+set(MAPGET_VERSION 2024.3)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -64,7 +64,7 @@ set(SIMFIL_WITH_MODEL_JSON YES)
 if (NOT TARGET simfil)
   FetchContent_Declare(simfil
     GIT_REPOSITORY "https://github.com/Klebert-Engineering/simfil.git"
-    GIT_TAG        "main"
+    GIT_TAG        "5af9ada73898c952657d1bf1358205e54e82a584"
     GIT_SHALLOW    ON)
   FetchContent_MakeAvailable(simfil)
 endif()

examples/cpp/http-datasource/main.cpp

@@ -32,7 +32,7 @@ class MyRemoteDataSource
     void fill(TileFeatureLayer::Ptr const& tile)
     {
         // Add some ID parts that are shared by all features in the tile.
-        tile->setPrefix({{"areaId", "BestArea"}});
+        tile->setIdPrefix({{"areaId", "BestArea"}});
 
         // Create a feature with line geometry
         auto feature1 = tile->newFeature("Way", {{"wayId", 42}});

examples/cpp/local-datasource/main.cpp

@@ -41,7 +41,7 @@ class MyLocalDataSource : public mapget::DataSource
 
     void fill(TileFeatureLayer::Ptr const& tile) override {
         // Add some ID parts that are shared by all features in the tile.
-        tile->setPrefix({{"areaId", "BestArea"}});
+        tile->setIdPrefix({{"areaId", "BestArea"}});
 
         // Create a feature with line geometry
         auto feature1 = tile->newFeature("Way", {{"wayId", 42}});

examples/python/datasource.py

@@ -52,7 +52,8 @@ def handle_tile_request(tile: mapget.TileFeatureLayer):
     attr.add_field("speedLimit", 50)
 
     # Add a child feature ID
-    feature.children().append(tile.new_feature_id("Way", [("wayId", 10)]))
+    # TODO: Add Python bindings for relations.
+    # feature.children().append(tile.new_feature_id("Way", [("wayId", 10)]))
 
 
 # Instantiate a data source with a minimal mandatory set

libs/http-datasource/include/mapget/http-datasource/datasource-client.h

@@ -30,12 +30,15 @@ class RemoteDataSource : public DataSource
     DataSourceInfo info() override;
     void fill(TileFeatureLayer::Ptr const& featureTile) override;
     TileFeatureLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override;
-    std::optional<LocateResponse> locate(const mapget::LocateRequest &req) override;
+    std::vector<LocateResponse> locate(const mapget::LocateRequest &req) override;
 
 private:
     // DataSourceInfo is fetched in the constructor
     DataSourceInfo info_;
 
+    // Error string, written in get() and set in fill().
+    std::string error_;
+
     // Multiple http clients allow parallel GET requests
     std::vector<httplib::Client> httpClients_;
     std::atomic_uint64_t nextClient_{0};
@@ -65,7 +68,7 @@ class RemoteDataSourceProcess : public DataSource
     DataSourceInfo info() override;
     void fill(TileFeatureLayer::Ptr const& featureTile) override;
     TileFeatureLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override;
-    std::optional<LocateResponse> locate(const mapget::LocateRequest &req) override;
+    std::vector<LocateResponse> locate(const mapget::LocateRequest &req) override;
 
 private:
     std::unique_ptr<RemoteDataSource> remoteSource_;
@@ -74,4 +77,4 @@ class RemoteDataSourceProcess : public DataSource
     std::condition_variable cv_;
 };
 
-}
+}

libs/http-datasource/include/mapget/http-datasource/datasource-server.h

@@ -39,7 +39,7 @@ class DataSourceServer : public HttpServer
      * must process according to its available data.
      */
     DataSourceServer&
-    onLocateRequest(std::function<std::optional<LocateResponse>(LocateRequest const&)> const&);
+    onLocateRequest(std::function<std::vector<LocateResponse>(LocateRequest const&)> const&);
 
     /**
      * Get the DataSourceInfo metadata which this instance was constructed with.

libs/http-datasource/src/datasource-client.cpp

@@ -13,12 +13,12 @@ RemoteDataSource::RemoteDataSource(const std::string& host, uint16_t port)
     httplib::Client client(host, port);
     auto fetchedInfoJson = client.Get("/info");
     if (!fetchedInfoJson || fetchedInfoJson->status >= 300)
-        throw logRuntimeError("Failed to fetch datasource info.");
+        raise("Failed to fetch datasource info.");
     info_ = DataSourceInfo::fromJson(nlohmann::json::parse(fetchedInfoJson->body));
 
     if (info_.nodeId_.empty()) {
         // Unique node IDs are required for the field offsets.
-        throw logRuntimeError(
+        raise(
             fmt::format("Remote data source is missing node ID! Source info: {}",
                 fetchedInfoJson->body));
     }
@@ -36,7 +36,7 @@ DataSourceInfo RemoteDataSource::info()
 void RemoteDataSource::fill(const TileFeatureLayer::Ptr& featureTile)
 {
     // If we get here, an error occurred.
-    featureTile->setError("Error while contacting remote data source.");
+    featureTile->setError(fmt::format("Error while contacting remote data source: {}", error_));
 }
 
 TileFeatureLayer::Ptr
@@ -57,7 +57,18 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn
         // Forward to base class get(). This will instantiate a
         // default TileFeatureLayer and call fill(). In our implementation
         // of fill, we set an error.
-        // TODO: Read HTTPLIB_ERROR header, more log output.
+        if (tileResponse->has_header("HTTPLIB_ERROR")) {
+            error_ = tileResponse->get_header_value("HTTPLIB_ERROR");
+        }
+        else if (tileResponse->has_header("EXCEPTION_WHAT")) {
+            error_ = tileResponse->get_header_value("EXCEPTION_WHAT");
+        }
+        else {
+            error_ = fmt::format("Code {}", tileResponse->status);
+        }
+
+        // Use tile instantiation logic of the base class,
+        // the error is then set in fill().
         return DataSource::get(k, cache, info);
     }
 
@@ -72,7 +83,7 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn
     return result;
 }
 
-std::optional<LocateResponse> RemoteDataSource::locate(const LocateRequest& req)
+std::vector<LocateResponse> RemoteDataSource::locate(const LocateRequest& req)
 {
     // Round-robin usage of http clients to facilitate parallel requests.
     auto& client = httpClients_[(nextClient_++) % httpClients_.size()];
@@ -96,7 +107,12 @@ std::optional<LocateResponse> RemoteDataSource::locate(const LocateRequest& req)
         return {};
     }
 
-    return LocateResponse(responseJson);
+    // Parse the resulting responses.
+    std::vector<LocateResponse> responseVector;
+    for (auto const& responseJsonAlternative : responseJson) {
+        responseVector.emplace_back(responseJsonAlternative);
+    }
+    return responseVector;
 }
 
 RemoteDataSourceProcess::RemoteDataSourceProcess(std::string const& commandLine)
@@ -143,7 +159,7 @@ RemoteDataSourceProcess::RemoteDataSourceProcess(std::string const& commandLine)
 #if defined(NDEBUG)
     if (!cv_.wait_for(lock, std::chrono::seconds(10), [this] { return remoteSource_ != nullptr; }))
     {
-        throw logRuntimeError(
+        raise(
             "Timeout waiting for the child process to initialize the remote data source.");
     }
 #else
@@ -163,29 +179,29 @@ RemoteDataSourceProcess::~RemoteDataSourceProcess()
 DataSourceInfo RemoteDataSourceProcess::info()
 {
     if (!remoteSource_)
-        throw logRuntimeError("Remote data source is not initialized.");
+        raise("Remote data source is not initialized.");
     return remoteSource_->info();
 }
 
 void RemoteDataSourceProcess::fill(TileFeatureLayer::Ptr const& featureTile)
 {
     if (!remoteSource_)
-        throw logRuntimeError("Remote data source is not initialized.");
+        raise("Remote data source is not initialized.");
     remoteSource_->fill(featureTile);
 }
 
 TileFeatureLayer::Ptr
 RemoteDataSourceProcess::get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info)
 {
     if (!remoteSource_)
-        throw logRuntimeError("Remote data source is not initialized.");
+        raise("Remote data source is not initialized.");
     return remoteSource_->get(k, cache, info);
 }
 
-std::optional<LocateResponse> RemoteDataSourceProcess::locate(const LocateRequest& req)
+std::vector<LocateResponse> RemoteDataSourceProcess::locate(const LocateRequest& req)
 {
     if (!remoteSource_)
-        throw logRuntimeError("Remote data source is not initialized.");
+        raise("Remote data source is not initialized.");
     return remoteSource_->locate(req);
 }
 

libs/http-datasource/src/datasource-server.cpp

@@ -12,7 +12,7 @@ struct DataSourceServer::Impl
 {
     DataSourceInfo info_;
     std::function<void(TileFeatureLayer::Ptr)> tileCallback_;
-    std::function<std::optional<LocateResponse>(const LocateRequest&)> locateCallback_;
+    std::function<std::vector<LocateResponse>(const LocateRequest&)> locateCallback_;
     std::shared_ptr<Fields> fields_;
 
     explicit Impl(DataSourceInfo info)
@@ -37,7 +37,7 @@ DataSourceServer::onTileRequest(std::function<void(TileFeatureLayer::Ptr)> const
 }
 
 DataSourceServer& DataSourceServer::onLocateRequest(
-    const std::function<std::optional<LocateResponse>(const LocateRequest&)>& callback)
+    const std::function<std::vector<LocateResponse>(const LocateRequest&)>& callback)
 {
     impl_->locateCallback_ = callback;
     return *this;
@@ -107,11 +107,11 @@ void DataSourceServer::setup(httplib::Server& server)
         "/locate",
         [this](const httplib::Request& req, httplib::Response& res) {
             LocateRequest parsedReq(nlohmann::json::parse(req.body));
-            auto responseJson = nlohmann::json();
+            auto responseJson = nlohmann::json::array();
 
             if (impl_->locateCallback_) {
-                if (auto response = impl_->locateCallback_(parsedReq)) {
-                    responseJson = response->serialize();
+                for (auto const& response : impl_->locateCallback_(parsedReq)) {
+                    responseJson.emplace_back(response.serialize());
                 }
             }
 

libs/http-datasource/src/http-server.cpp

@@ -56,7 +56,7 @@ void HttpServer::go(
     }
 
     if (impl_->server_.is_running() || impl_->serverThread_.joinable())
-        throw logRuntimeError("HttpServer is already running");
+        raise("HttpServer is already running");
 
     if (port == 0) {
         impl_->port_ = impl_->server_.bind_to_any_port(interfaceAddr);
@@ -78,7 +78,7 @@ void HttpServer::go(
 
     std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
     if (!impl_->server_.is_running() || !impl_->server_.is_valid())
-        throw logRuntimeError(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port));
+        raise(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port));
 }
 
 bool HttpServer::isRunning() {

libs/http-service/src/cli.cpp

@@ -73,7 +73,7 @@ struct ServeCommand
             cache = std::make_shared<MemCache>(cacheMaxTiles_);
         }
         else {
-            logRuntimeError(fmt::format("Cache type {} not supported!", cacheType_));
+            raise(fmt::format("Cache type {} not supported!", cacheType_));
         }
 
         HttpService srv(cache);
@@ -122,6 +122,7 @@ struct FetchCommand
 {
     std::string server_, map_, layer_;
     std::vector<uint64_t> tiles_;
+    bool mute_ = false;
 
     explicit FetchCommand(CLI::App& app)
     {
@@ -130,6 +131,8 @@ struct FetchCommand
             ->required();
         fetchCmd->add_option("-m,--map", map_, "Map to retrieve.")->required();
         fetchCmd->add_option("-l,--layer", layer_, "Layer of the map to retrieve.")->required();
+        fetchCmd->add_option("--mute",
+            mute_, "Mute the actual tile GeoJSON output.");
         fetchCmd
             ->add_option(
                 "-t,--tile",
@@ -143,7 +146,7 @@ struct FetchCommand
     {
         if (log().level() <= spdlog::level::debug) {
             // Skips building the tile list string if it will not be logged.
-            std::string tileList = "";
+            std::string tileList;
             for (auto& tile : tiles_) {
                 tileList += std::to_string(tile) + " ";
             }
@@ -164,18 +167,20 @@ struct FetchCommand
             map_,
             layer_,
             std::vector<TileId>{tiles_.begin(), tiles_.end()},
-            [](auto&& tile)
+            [this](TileFeatureLayer::Ptr const& tile)
             {
-                if (log().level() == spdlog::level::trace) {
-                    log().trace(tile->toGeoJson().dump());
-                }
+                if (!mute_)
+                    std::cout << tile->toGeoJson().dump() << std::endl;
+                if (tile->error())
+                    raise(
+                        fmt::format("Tile {}: {}", tile->id().toString(), *tile->error()));
             });
         cli.request(request)->wait();
 
         if (request->getStatus() == NoDataSource)
-            throw logRuntimeError("Failed to fetch sources: no matching data source.");
+            raise("Failed to fetch sources: no matching data source.");
         if (request->getStatus() == Aborted)
-            throw logRuntimeError("Failed to fetch sources: request aborted.");
+            raise("Failed to fetch sources: request aborted.");
     }
 };
 
@@ -209,6 +214,9 @@ int runFromCommandLine(std::vector<std::string> args)
     catch (const CLI::ParseError& e) {
         return app.exit(e);
     }
+    catch (std::runtime_error const& e) {
+        return 1;
+    }
     return 0;
 }
 

libs/http-service/src/http-client.cpp

@@ -16,7 +16,7 @@ struct HttpClient::Impl {
         client_.set_keep_alive(false);
         auto sourcesJson = client_.Get("/sources");
         if (!sourcesJson || sourcesJson->status != 200)
-            throw logRuntimeError(
+            raise(
                 fmt::format("Failed to fetch sources: [{}]", sourcesJson->status));
         for (auto const& info : nlohmann::json::parse(sourcesJson->body)) {
             auto parsedInfo = DataSourceInfo::fromJson(info);
@@ -29,7 +29,7 @@ struct HttpClient::Impl {
     {
         auto mapIt = sources_.find(std::string(map));
         if (mapIt == sources_.end())
-            throw logRuntimeError("Could not find map data source info");
+            raise("Could not find map data source info");
         return mapIt->second.getLayer(std::string(layer));
     }
 };

libs/http-service/src/http-service.cpp

@@ -57,7 +57,7 @@ struct HttpService::Impl
                 responseType_ = binaryMimeType;
                 return;
             }
-            throw logRuntimeError(fmt::format("Unknown Accept-Header value {}", responseType_));
+            raise(fmt::format("Unknown Accept-Header value {}", responseType_));
         }
 
         void addResult(TileFeatureLayer::Ptr const& result)

libs/logging/CMakeLists.txt

@@ -10,7 +10,8 @@ add_library(mapget-log STATIC
 target_link_libraries(mapget-log
   PUBLIC
     spdlog::spdlog
-    fmt::fmt)
+    fmt::fmt
+    simfil::simfil)
 
 target_include_directories(mapget-log
   PUBLIC

libs/logging/include/mapget/log.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "spdlog/spdlog.h"
+#include "simfil/exception-handler.h"
 
 namespace mapget
 {
@@ -23,15 +24,15 @@ spdlog::logger& log();
 void setLogLevel(std::string logLevel, spdlog::logger& logInstance);
 
 /**
- * Log a runtime error and return the throwable object.
- * @param what Runtime error message.
- * @return std::runtime_error to throw.
+ * Log an exception and throw it via simfil::raise().
  */
-template <typename error_t = std::runtime_error>
-error_t logRuntimeError(std::string const& what)
+template<typename ExceptionType=std::runtime_error, typename... Args>
+[[noreturn]] void raise(Args&&... args)
 {
-    log().error(what);
-    return error_t(what);
+    ExceptionType exceptionInstance(std::forward<Args>(args)...);
+    if constexpr (requires {exceptionInstance.what();})
+        log().error(exceptionInstance.what());
+    simfil::raise<ExceptionType>(exceptionInstance);
 }
 
 }