Merge pull request #54 from ndsev/relations

Relations/Locate

.github/workflows/cmake.yaml

@@ -43,7 +43,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, windows-2019]
+        os: [macos-13, windows-2019]  # Currently, macos-latest is macos 12
         python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v2
@@ -56,7 +56,7 @@ jobs:
       - run: python -m pip install setuptools wheel
       - run: mkdir build
       - name: Build (macOS)
-        if: matrix.os == 'macos-latest' || matrix.os == 'macos-10.15'
+        if: matrix.os == 'macos-13'
         working-directory: build
         run: |
           python -m pip install delocate

.github/workflows/deploy.yaml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-        os: [macos-latest, ubuntu-latest, windows-2019]
+        os: [macos-13, ubuntu-latest, windows-2019]
     runs-on: ubuntu-latest
     steps:
       - name: Fetch

CMakeLists.txt

@@ -16,7 +16,7 @@ option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-servic
 project(mapget)
 include(FetchContent)
 
-set(MAPGET_VERSION 0.3.0)
+set(MAPGET_VERSION 2024.1)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -60,22 +60,6 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/cmake-modules")
 ##############
 # deps
 
-if (NOT TARGET stx)
-  FetchContent_Declare(stx
-    GIT_REPOSITORY "https://github.com/Klebert-Engineering/stx.git"
-    GIT_TAG        "main"
-    GIT_SHALLOW    ON)
-  FetchContent_MakeAvailable(stx)
-endif()
-
-if (NOT TARGET spdlog::spdlog)
-  FetchContent_Declare(spdlog
-    GIT_REPOSITORY "https://github.com/gabime/spdlog.git"
-    GIT_TAG        "v1.x"
-    GIT_SHALLOW    ON)
-  FetchContent_MakeAvailable(spdlog)
-endif()
-
 set(SIMFIL_WITH_MODEL_JSON YES)
 if (NOT TARGET simfil)
   FetchContent_Declare(simfil
@@ -85,12 +69,13 @@ if (NOT TARGET simfil)
   FetchContent_MakeAvailable(simfil)
 endif()
 
-if (NOT TARGET bitsery)
-  FetchContent_Declare(bitsery
-    GIT_REPOSITORY "https://github.com/fraillt/bitsery.git"
-    GIT_TAG        "master"
+if (NOT TARGET spdlog::spdlog)
+  set(SPDLOG_FMT_EXTERNAL TRUE CACHE BOOL "Ensure that spdlog uses fmt from conan." FORCE)
+  FetchContent_Declare(spdlog
+    GIT_REPOSITORY "https://github.com/gabime/spdlog.git"
+    GIT_TAG        "v1.x"
     GIT_SHALLOW    ON)
-  FetchContent_MakeAvailable(bitsery)
+  FetchContent_MakeAvailable(spdlog)
 endif()
 
 if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
@@ -180,8 +165,8 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
 
     FetchContent_Declare(rocksdb
       GIT_REPOSITORY "https://github.com/facebook/rocksdb.git"
-      GIT_TAG        dc87847e65449ef1cb6f787c5d753cbe8562bff1 # Use version greater than v8.6.7 once released.
-      GIT_SHALLOW    OFF)  # Turn ON when using released tag.
+      GIT_TAG        v8.11.3
+      GIT_SHALLOW    ON)
     FetchContent_MakeAvailable(rocksdb)
   endif()
 

README.md

@@ -76,6 +76,10 @@ A feature consists of a unique ID, some attributes, and some geometry. *mapget*
 also allows features to have a list of child feature IDs. **Note:** Feature geometry
 in *mapget* may always be 3D.
 
+* TODO: Document JSON representation.
+* TODO: Document Feature ID schemes.
+* TODO: Document Geometry Types.
+
 ## Map Tiles
 
 For performance reasons, *mapget* features are always served in a set covering
@@ -174,7 +178,7 @@ This example shows how you can write a minimal networked data source service.
 This example shows, how you can write a data source service in Python.
 You can simply `pip install mapget` to get access to the mapget Python API.
 
-## Retrieval Interface
+## REST API
 
 The `mapget` library provides simple C++ and HTTP/REST interfaces, which may be
 used to satisfy the following use-cases:
@@ -188,46 +192,14 @@ used to satisfy the following use-cases:
 The HTTP interface implemented in `mapget::HttpService` is a view on the C++ interface,
 which is implemented in `mapget::Service`. Detailed endpoint descriptions:
 
-```c++
-// Describe the connected Data Sources
-+ GET /sources():  list<DataSourceInfo>
-
-// Get streamed features, according to hard constraints.
-// With Accept-Encoding text/jsonl or application/binary
-+ POST /tiles(list<{
-    mapId: string,
-    layerId: string,
-    tileIds: list<TileId>,
-       maxKnownFieldIds*
-  }>, filter: optional<string>):
-  bytes<TileLayerStream>
-
-// Server status page
-+ GET /status(): text/html
-
-// Obtain a list of tile-layer combinations which provide a
-// feature that satisfies the given ID field constraints.
-// TODO: Not implemented
-+ GET /locate(typeId, map<string, Scalar>)
-  list<pair<TileId, LayerId>>
-
-// Instruct the cache to *try* to fill itself with map data
-// according to the provided constraints. No guarantees are
-// made as to the timeliness or completeness of this task.
-// TODO: Not implemented
-+ POST /populate(
-    spatialConstraint*,
-    {mapId*, layerId*, featureType*, zoomLevel*}) 
-
-// Write GeoJSON features for a tile back to a map data source
-// which has supportedOperations&WRITE.
-// TODO: Not implemented
-+ POST /tile(
-    mapId: string,
-    layerId: string,
-    tileId: TileId,
-    features: list<object>): void
-```
+| Endpoint   | Method | Description                                                                                                       | Input                                                                                                                                               | Output                                                                                                                                                                            |
+|------------|--------|-------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `/sources` | GET    | Describe the connected Data Sources                                                                               | None                                                                                                                                                | `application/json`: List of DataSourceInfo objects.                                                                                                                               |
+| `/tiles`   | POST   | Get streamed features, according to hard constraints. Accepts encoding types `text/jsonl` or `application/binary` | List of objects containing `mapId`, `layerId`, `tileIds`, and optional `maxKnownFieldIds`.                                                          | `text/jsonl` or `application/binary`                                                                                                                                              |
+| `/status`  | GET    | Server status page                                                                                                | None                                                                                                                                                | text/html                                                                                                                                                                         |
+| `/locate`  | POST   | Obtain a list of tile-layer combinations providing a feature that satisfies given ID field constraints.           | `application/json`: List of external references, where each is a Request object with `mapId`, `typeId` and `featureId` (list of external ID parts). | `application/json`: List of lists of Resolution objects, where each corresponds to the Request object index. Each Resolution object includes `tileId`, `typeId`, and `featureId`. |
+
+### Curl Call Example
 
 For example, the following curl call could be used to stream GeoJSON feature objects
 from the `MyMap` data source defined previously:
@@ -248,6 +220,8 @@ curl -X POST \
 }' "http://localhost:8080/tiles"
 ```
 
+### C++ Call Example
+
 If we use `"Accept: application/binary"` instead, we get a binary stream of
 tile data which we can also parse in C++, Python or JS. Here is an example in C++, using
 the `mapget::HttpClient` class:
@@ -277,6 +251,25 @@ void main(int argc, char const *argv[])
 
 Keep in mind, that you can also run a `mapget` service without any RPCs in your application. Check out [`examples/cpp/local-datasource`](examples/cpp/local-datasource/main.cpp) on how to do that.
 
+### About `locate`
+
+The `/locate` endpoint allows clients to obtain a list of tile-layer combinations that provide a feature satisfying given ID field constraints. This is crucial for applications needing to find specific data points within the massive datasets typically associated with map services. The endpoint uses a POST method due to the complexity and length of the queries, which involve resolving external references to data.
+
+**Details:**
+
+- **Input:** The input is a list of requests, each corresponding to an external reference that needs to be resolved. Each request object includes:
+    - `typeId`: Specifies the type of feature to locate.
+    - `featureId`: An array representing the external ID parts, where each part consists of a field name and value. This array is used to identify the feature uniquely. The used id scheme may be a secondary scheme.
+
+- **Output:** The output is a nested list structure where each outer list corresponds to an input request object. Each of these lists contains resolution objects that provide details about where the requested feature can be found within the map data. Each resolution object includes:
+    - `tileId`: The key of the map tile containing the feature.
+    - `typeId`: The type of feature found, which should match the `typeId` specified in the request.
+    - `featureId`: An array of ID parts similar to the input but typically using the primary feature ID scheme of the data source.
+
+This design allows clients to batch queries for multiple features in a single request, improving efficiency and reducing the number of required HTTP requests. It also supports the use of different ID schemes, accommodating scenarios where the request and response might use different identifiers for the same data due to varying external reference standards.
+
+Note, that a locate resolution must be provided by a datasource for the specified map, which implements the `onLocateRequest` callback.
+
 ### erdblick-mapget-datasource communication pattern
 
 TODO: expand and polish this section stub.

libs/http-datasource/include/mapget/http-datasource/datasource-client.h

@@ -30,6 +30,7 @@ class RemoteDataSource : public DataSource
     DataSourceInfo info() override;
     void fill(TileFeatureLayer::Ptr const& featureTile) override;
     TileFeatureLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override;
+    std::optional<LocateResponse> locate(const mapget::LocateRequest &req) override;
 
 private:
     // DataSourceInfo is fetched in the constructor
@@ -64,6 +65,7 @@ class RemoteDataSourceProcess : public DataSource
     DataSourceInfo info() override;
     void fill(TileFeatureLayer::Ptr const& featureTile) override;
     TileFeatureLayer::Ptr get(MapTileKey const& k, Cache::Ptr& cache, DataSourceInfo const& info) override;
+    std::optional<LocateResponse> locate(const mapget::LocateRequest &req) override;
 
 private:
     std::unique_ptr<RemoteDataSource> remoteSource_;

libs/http-datasource/include/mapget/http-datasource/datasource-server.h

@@ -2,6 +2,7 @@
 
 #include "mapget/model/featurelayer.h"
 #include "mapget/detail/http-server.h"
+#include "mapget/service/locate.h"
 
 namespace mapget
 {
@@ -32,6 +33,14 @@ class DataSourceServer : public HttpServer
      */
     DataSourceServer& onTileRequest(std::function<void(TileFeatureLayer::Ptr)> const&);
 
+    /**
+     * Set the callback which will be invoked when a `/locate`-request is received.
+     * The callback argument is a LocateRequest, which the callback
+     * must process according to its available data.
+     */
+    DataSourceServer&
+    onLocateRequest(std::function<std::optional<LocateResponse>(LocateRequest const&)> const&);
+
     /**
      * Get the DataSourceInfo metadata which this instance was constructed with.
      */

libs/http-datasource/src/datasource-client.cpp

@@ -19,7 +19,7 @@ RemoteDataSource::RemoteDataSource(const std::string& host, uint16_t port)
     if (info_.nodeId_.empty()) {
         // Unique node IDs are required for the field offsets.
         throw logRuntimeError(
-            stx::format("Remote data source is missing node ID! Source info: {}",
+            fmt::format("Remote data source is missing node ID! Source info: {}",
                 fetchedInfoJson->body));
     }
 
@@ -46,7 +46,7 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn
     auto& client = httpClients_[(nextClient_++) % httpClients_.size()];
 
     // Send a GET tile request.
-    auto tileResponse = client.Get(stx::format(
+    auto tileResponse = client.Get(fmt::format(
         "/tile?layer={}&tileId={}&fieldsOffset={}",
         k.layerId_,
         k.tileId_.value_,
@@ -72,6 +72,33 @@ RemoteDataSource::get(const MapTileKey& k, Cache::Ptr& cache, const DataSourceIn
     return result;
 }
 
+std::optional<LocateResponse> RemoteDataSource::locate(const LocateRequest& req)
+{
+    // Round-robin usage of http clients to facilitate parallel requests.
+    auto& client = httpClients_[(nextClient_++) % httpClients_.size()];
+
+    // Send a GET tile request.
+    auto locateResponse = client.Post(
+        fmt::format("/locate"), req.serialize().dump(), "application/json");
+
+    // Check that the response is OK.
+    if (!locateResponse || locateResponse->status >= 300) {
+        // Forward to base class get(). This will instantiate a
+        // default TileFeatureLayer and call fill(). In our implementation
+        // of fill, we set an error.
+        // TODO: Read HTTPLIB_ERROR header, more log output.
+        return {};
+    }
+
+    // Check the response body for expected content.
+    auto responseJson = nlohmann::json::parse(locateResponse->body);
+    if (responseJson.is_null()) {
+        return {};
+    }
+
+    return LocateResponse(responseJson);
+}
+
 RemoteDataSourceProcess::RemoteDataSourceProcess(std::string const& commandLine)
 {
     auto stderrCallback = [this](const char* bytes, size_t n)
@@ -155,4 +182,11 @@ RemoteDataSourceProcess::get(MapTileKey const& k, Cache::Ptr& cache, DataSourceI
     return remoteSource_->get(k, cache, info);
 }
 
+std::optional<LocateResponse> RemoteDataSourceProcess::locate(const LocateRequest& req)
+{
+    if (!remoteSource_)
+        throw logRuntimeError("Remote data source is not initialized.");
+    return remoteSource_->locate(req);
+}
+
 }

libs/http-datasource/src/datasource-server.cpp

@@ -12,6 +12,7 @@ struct DataSourceServer::Impl
 {
     DataSourceInfo info_;
     std::function<void(TileFeatureLayer::Ptr)> tileCallback_;
+    std::function<std::optional<LocateResponse>(const LocateRequest&)> locateCallback_;
     std::shared_ptr<Fields> fields_;
 
     explicit Impl(DataSourceInfo info)
@@ -29,11 +30,19 @@ DataSourceServer::DataSourceServer(DataSourceInfo const& info)
 DataSourceServer::~DataSourceServer() = default;
 
 DataSourceServer&
-DataSourceServer::onTileRequest(std::function<void(TileFeatureLayer::Ptr)> const& callback) {
+DataSourceServer::onTileRequest(std::function<void(TileFeatureLayer::Ptr)> const& callback)
+{
     impl_->tileCallback_ = callback;
     return *this;
 }
 
+DataSourceServer& DataSourceServer::onLocateRequest(
+    const std::function<std::optional<LocateResponse>(const LocateRequest&)>& callback)
+{
+    impl_->locateCallback_ = callback;
+    return *this;
+}
+
 DataSourceInfo const& DataSourceServer::info() {
     return impl_->info_;
 }
@@ -92,6 +101,22 @@ void DataSourceServer::setup(httplib::Server& server)
             nlohmann::json j = impl_->info_.toJson();
             res.set_content(j.dump(), "application/json");
         });
+
+    // Set up POST /locate endpoint
+    server.Post(
+        "/locate",
+        [this](const httplib::Request& req, httplib::Response& res) {
+            LocateRequest parsedReq(nlohmann::json::parse(req.body));
+            auto responseJson = nlohmann::json();
+
+            if (impl_->locateCallback_) {
+                if (auto response = impl_->locateCallback_(parsedReq)) {
+                    responseJson = response->serialize();
+                }
+            }
+
+            res.set_content(responseJson.dump(), "application/json");
+        });
 }
 
 }  // namespace mapget

libs/http-datasource/src/http-server.cpp

@@ -4,9 +4,9 @@
 #include "httplib.h"
 #include <csignal>
 #include <atomic>
+#include <ranges>
 
-#include "stx/format.h"
-#include "stx/string.h"
+#include "fmt/format.h"
 
 namespace mapget
 {
@@ -78,7 +78,7 @@ void HttpServer::go(
 
     std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
     if (!impl_->server_.is_running() || !impl_->server_.is_valid())
-        throw logRuntimeError(stx::format("Could not start HttpServer on {}:{}", interfaceAddr, port));
+        throw logRuntimeError(fmt::format("Could not start HttpServer on {}:{}", interfaceAddr, port));
 }
 
 bool HttpServer::isRunning() {
@@ -115,10 +115,13 @@ void HttpServer::waitForSignal() {
 
 bool HttpServer::mountFileSystem(const std::string& pathFromTo)
 {
-    auto parts = stx::split(pathFromTo, ":");
-    if (parts.size() == 1)
-        return impl_->server_.set_mount_point("/", parts[0]);
-    return impl_->server_.set_mount_point(parts[0], parts[1]);
+    using namespace std::ranges;
+    auto parts = pathFromTo | views::split(':') | views::transform([](auto&& s){return std::string(&*s.begin(), distance(s));});
+    auto partsVec = std::vector<std::string>(parts.begin(), parts.end());
+
+    if (partsVec.size() == 1)
+        return impl_->server_.set_mount_point("/", partsVec[0]);
+    return impl_->server_.set_mount_point(partsVec[0], partsVec[1]);
 }
 
 void HttpServer::printPortToStdOut(bool enabled) {