Version 2024.3.1: Datasource Config

.github/workflows/cmake.yaml

@@ -12,8 +12,9 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
     runs-on: ubuntu-latest
-    container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}:2023.1
+    container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:2024.1
     env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
       SCCACHE_GHA_ENABLED: "true"
     steps:
       - uses: actions/checkout@v2
@@ -48,7 +49,7 @@ jobs:
         timeout-minutes: 30
         run: |
           . ./venv/bin/activate
-          ctest --preset conan-release -C Release --verbose --no-test=fail
+          ctest --preset conan-release -C Release --verbose --no-tests=error
       - name: Deploy
         uses: actions/upload-artifact@v2
         with:
@@ -89,6 +90,7 @@ jobs:
                 -DPython3_FIND_FRAMEWORK=LAST \
                 -DCMAKE_C_COMPILER_LAUNCHER=sccache \
                 -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+                -DMAPGET_ENABLE_TESTING=ON \
                 -GNinja
           cmake --build --preset conan-release
           cd build/Release
@@ -114,4 +116,4 @@ jobs:
       - name: Test
         timeout-minutes: 30
         run: |
-          ctest --preset conan-release -C Release --verbose --no-test=fail
+          ctest --preset conan-release -C Release --verbose --no-tests=error

.github/workflows/coverage.yml

@@ -42,7 +42,7 @@ jobs:
         cmake --build --preset conan-debug
     - name: Run Test
       run: |
-        ctest --preset conan-debug -C Debug --verbose --no-test=fail
+        ctest --preset conan-debug -C Debug --verbose --no-tests=error
     - name: Run Gcovr
       run: |
         mkdir coverage

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_policy(SET CMP0117 NEW)
 
 project(mapget CXX)
 
-set(MAPGET_VERSION 2024.3)
+set(MAPGET_VERSION 2024.3.1)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

README.md

@@ -33,12 +33,40 @@ mapget serve --help
 
 (or `python -m mapget --help` for the Python package).
 
-The `mapget` executable can parse a config file with arguments supported by the command line interface. The path to the config file can be provided to `mapget` via command line by specifying the `--config` parameter.
+The `mapget` executable can parse a `YAML`-based config file. The path to the config file can be provided to `mapget` via command line by specifying the `--config` parameter.
+The config file may have two top-level keys which are used by mapget: The `sources` key and the `mapget` key.
 
+### The `mapget` YAML key
+
+Under the `mapget` YAML key, any config options which can also be set via the command line may be set.
+Note, that changes in this section are not applied while mapget is running, you will need to restart it.
 Sample configuration files can be found under `examples/config`:
 
-- [sample-first-datasource.toml](examples/config/sample-first-datasource.toml) and [sample-second-datasource.toml](examples/config/sample-second-datasource.toml) will configure mapget to run a simple datasource with sample data. Note: the two formats in config files for subcommand parameters can be used interchangeably.
-- [sample-service.toml](examples/config/sample-service.toml) to execute the `mapget serve` command. The instance will fetch and serve data from sources started with `sample-*-datasource.toml` configs above.
+- [sample-first-datasource.yaml](examples/config/sample-first-datasource.yaml) and [sample-second-datasource.yaml](examples/config/sample-second-datasource.yaml) will configure mapget to run a simple datasource with sample data. Note: the two formats in config files for subcommand parameters can be used interchangeably.
+- [sample-service.yaml](examples/config/sample-service.yaml) to execute the `mapget serve` command. The instance will fetch and serve data from sources started with `sample-*-datasource.toml` configs above.
+
+### The `sources` YAML key
+
+Under the `sources` YAML key, you can configure datasources which are going to be served.
+Note, that changes from the sources section are going to be applied immediately once the config
+file is saved. This means, you can add and/or remove sources while mapget is running.
+This section has the following format: The `sources` key must have a list. Each entry in the list
+represents a datasource. The entry must have a `type` key, which denotes the specific datasource
+constructor to call. You may register additional datasource types using the
+`DatasourceConfigService` from `mapget/service/config.h`. By default, the following datasource types are supported:
+
+| Data Source Type        | Required Configurations | Optional Configurations     |
+|-------------------------|-------------------------|-----------------------------|
+| `DataSourceHost`        | `url`                   | N/A                         |
+| `DataSourceProcess`     | `cmd`                   | N/A                         |
+
+For example, the following would be a valid configuration:
+
+```yaml
+sources:
+  - type: DataSourceProcess
+    cmd: cpp-sample-http-datasource
+```
 
 ### Cache
 

deps.cmake

@@ -126,7 +126,7 @@ if (MAPGET_WITH_WHEEL)
   FetchContent_MakeAvailable(python-cmake-wheel)
 endif()
 
-set(BUILD_TESTING NO CACHE BOOL "")
+set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
 FetchContent_Declare(tiny-process-library
   GIT_REPOSITORY "https://gitlab.com/eidheim/tiny-process-library"
   GIT_TAG        v2.0.4

examples/config/sample-first-datasource.yaml

@@ -0,0 +1,13 @@
+# mapget: A client/server application for map data retrieval.
+mapget:
+    # From: trace, debug, info, warn, error, critical. Overrides MAPGET_LOG_LEVEL.
+    log-level: trace
+    fetch:
+        # Server to connect to in format <host:port>.
+        server: 127.0.0.1:61852
+        # Map to retrieve.
+        map: Tropico
+        # Layer of the map to retrieve.
+        layer: WayLayer
+        # Tile of the map to retrieve. Can be specified multiple times.
+        tile: 12345

examples/config/sample-second-datasource.yaml

@@ -0,0 +1,13 @@
+# mapget: A client/server application for map data retrieval.
+mapget:
+  # From: trace, debug, info, warn, error, critical. Overrides MAPGET_LOG_LEVEL.
+  log-level: trace
+  fetch:
+    # Server to connect to in format <host:port>.
+    server: 127.0.0.1:61852
+    # Map to retrieve.
+    map: TestMap
+    # Layer of the map to retrieve.
+    layer: WayLayer
+    # Tile of the map to retrieve. Can be specified multiple times.
+    tile: 12345

examples/config/sample-service.yaml

@@ -0,0 +1,15 @@
+# mapget: A client/server application for map data retrieval.
+mapget:
+    # From: trace, debug, info, warn, error, critical. Overrides MAPGET_LOG_LEVEL.
+    log-level: trace
+    serve:
+        # Port to start the server on. Default is 0.
+        port: 61852
+        # Datasources for the server in format <host:port>. Can be specified multiple times.
+        datasource-host: ["127.0.0.1:61853", "127.0.0.1:61854"]
+        # Using a persistent cache.
+        cache-type: rocksdb
+        # Datasource executable paths, including arguments, for the server. Can be specified multiple times.
+        # datasource-exe: ...
+        # Serve a static web application, in the format [<url-scope>:]<filesystem-path>.
+        # webapp: ...

libs/http-datasource/include/mapget/http-datasource/datasource-client.h

@@ -19,6 +19,11 @@ namespace mapget
 class RemoteDataSource : public DataSource
 {
 public:
+    /**
+     * Construct from joint host:port string.
+     */
+    static std::shared_ptr<RemoteDataSource> fromHostPort(std::string const& hostPort);
+
     /**
      * Construct a DataSource with the host and port of
      * a running DataSourceServer. Throws if the connection

libs/http-datasource/src/datasource-client.cpp

@@ -122,6 +122,15 @@ std::vector<LocateResponse> RemoteDataSource::locate(const LocateRequest& req)
     return responseVector;
 }
 
+std::shared_ptr<RemoteDataSource> RemoteDataSource::fromHostPort(const std::string& hostPort)
+{
+    auto delimiterPos = hostPort.find(':');
+    std::string dsHost = hostPort.substr(0, delimiterPos);
+    int dsPort = std::stoi(hostPort.substr(delimiterPos + 1, hostPort.size()));
+    log().info("Connecting to datasource at {}:{}.", dsHost, dsPort);
+    return std::make_shared<RemoteDataSource>(dsHost, dsPort);
+}
+
 RemoteDataSourceProcess::RemoteDataSourceProcess(std::string const& commandLine)
 {
     auto stderrCallback = [this](const char* bytes, size_t n)

libs/http-service/include/mapget/http-service/http-service.h

@@ -14,7 +14,7 @@ namespace mapget {
 class HttpService : public HttpServer, public Service
 {
 public:
-    explicit HttpService(Cache::Ptr cache = std::make_shared<MemCache>());
+    explicit HttpService(Cache::Ptr cache = std::make_shared<MemCache>(), bool watchConfig = false);
     ~HttpService() override;
 
 protected: