feat: Multidatasource

all/modules/datasources/MultiTileDataSource.i

@@ -0,0 +1,35 @@
+#ifndef _LOCALPACKAGEMANAGERTILEDATASOURCE_I
+#define _LOCALPACKAGEMANAGERTILEDATASOURCE_I
+
+%module(directors="1") MultiTileDataSource
+
+!proxy_imports(carto::MultiTileDataSource, core.MapTile, core.MapBounds, core.StringMap, datasources.TileDataSource,  datasources.components.TileData)
+%{
+#include "datasources/MultiTileDataSource.h"
+#include "components/Exceptions.h"
+#include <memory>
+%}
+
+%include <std_shared_ptr.i>
+%include <std_string.i>
+%include <cartoswig.i>
+
+%import "core/MapTile.i"
+%import "core/StringMap.i"
+%import "datasources/TileDataSource.i"
+#ifdef _CARTO_OFFLINE_SUPPORT
+%import "datasources/MBTilesTileDataSource.i"
+#endif
+%import "datasources/components/TileData.i"
+
+!polymorphic_shared_ptr(carto::MultiTileDataSource, datasources.MultiTileDataSource)
+
+%std_exceptions(carto::MultiTileDataSource::MultiTileDataSource)
+%std_exceptions(carto::LocalVectorDataSource::add)
+%std_exceptions(carto::LocalVectorDataSource::remove)
+
+%feature("director") carto::MultiTileDataSource;
+
+%include "datasources/MultiTileDataSource.h"
+
+#endif

all/native/datasources/MBTilesTileDataSource.cpp

@@ -286,6 +286,22 @@ namespace carto {
         return true;
     }
 
+    std::string MBTilesTileDataSource::getMetaData(const std::string &key) const {
+        // As a first step, try to use metadata
+        std::string result;
+        try {
+            sqlite3pp::query query(*_database, "SELECT value FROM metadata WHERE name=:name");
+            query.bind(":name", key.c_str());
+           for (auto it = query.begin(); it != query.end(); it++) {
+               result = (*it).get<const char*>(0);
+           }
+            query.finish();
+        }
+        catch (const std::exception& ex) {
+            Log::Errorf("MBTilesTileDataSource::getMetaData: Exception while reading %s metadata: %s", key, ex.what());
+        }
+        return result;
+    }
 }
 
 #endif

all/native/datasources/MBTilesTileDataSource.h

@@ -81,6 +81,13 @@ namespace carto {
          * @return Map containing meta data information (parameter names mapped to parameter values).
          */
         std::map<std::string, std::string> getMetaData() const;
+
+        /**
+         * Query a metadata value
+         * Possible parameters can be found in MBTiles specification.
+         * @return a string representation of the metadata value
+         */
+        std::string MBTilesTileDataSource::getMetaData(const std::string &key) const;
 
         virtual int getMinZoom() const;
 

all/native/datasources/MultiTileDataSource.cpp

@@ -0,0 +1,251 @@
+#include "MultiTileDataSource.h"
+#include "core/MapTile.h"
+#include "components/Exceptions.h"
+#include "utils/GeneralUtils.h"
+#include "utils/Log.h"
+#include "utils/Const.h"
+#include "packagemanager/PackageTileMask.h"
+
+#ifdef _CARTO_OFFLINE_SUPPORT
+#include "datasources/MBTilesTileDataSource.h"
+#endif
+
+#include <boost/lexical_cast.hpp>
+
+#include <memory>
+
+namespace carto {
+
+    MultiTileDataSource::MultiTileDataSource(int maxOpenedPackages) : TileDataSource(0, Const::MAX_SUPPORTED_ZOOM_LEVEL),
+        _dataSources(),
+        _cachedOpenDataSources(),
+        _maxOpenedPackages(maxOpenedPackages),
+        _mutex()
+    {
+        _dataSourceListener = std::make_shared<DataSourceListener>(*this);
+    }
+    MultiTileDataSource::MultiTileDataSource() : TileDataSource(0, Const::MAX_SUPPORTED_ZOOM_LEVEL),
+        _dataSources(),
+        _cachedOpenDataSources(),
+        _maxOpenedPackages(4),
+        _mutex()
+    {
+        _dataSourceListener = std::make_shared<DataSourceListener>(*this);
+    }
+
+    MultiTileDataSource::~MultiTileDataSource() {
+        _cachedOpenDataSources.clear();
+        for (auto it = _dataSources.begin(); it != _dataSources.end(); it++)
+        {
+            if (auto dataSource = std::dynamic_pointer_cast<TileDataSource>(it->second))
+            {
+                dataSource->unregisterOnChangeListener(_dataSourceListener);
+            }
+        }
+        _dataSourceListener.reset();
+    }
+
+    bool compare_datasource (std::pair<std::shared_ptr<PackageTileMask>, std::shared_ptr<TileDataSource>> dataSource1, std::pair<std::shared_ptr<PackageTileMask>, std::shared_ptr<TileDataSource>> dataSource2) {
+        return std::dynamic_pointer_cast<TileDataSource>(dataSource1.second) == std::dynamic_pointer_cast<TileDataSource>(dataSource2.second);
+    };
+
+    int MultiTileDataSource::getMinZoom() const {
+        int minZoom = Const::MAX_SUPPORTED_ZOOM_LEVEL;
+        for (auto it = _dataSources.begin(); it != _dataSources.end(); it++)
+        {
+            minZoom = std::min(minZoom, it->second->getMinZoom());
+        }
+        return minZoom;
+    }
+
+    int MultiTileDataSource::getMaxZoom() const {
+        int maxZoom = 0;
+        for (auto it = _dataSources.begin(); it != _dataSources.end(); it++)
+        {
+            maxZoom = std::max(maxZoom, it->second->getMaxZoom());
+        }
+        return maxZoom;
+    }
+
+    MapBounds MultiTileDataSource::getDataExtent() const {
+        MapBounds bounds;
+        for (auto it = _dataSources.begin(); it != _dataSources.end(); it++)
+        {
+            bounds.expandToContain( it->second->getDataExtent());
+        }
+        return bounds;
+    }
+
+    std::shared_ptr<TileData> MultiTileDataSource::loadTile(const MapTile &mapTile)
+    {
+        Log::Infof("MultiTileDataSource::loadTile: Loading %s", mapTile.toString().c_str());
+        try
+        {
+            MapTile mapTileFlipped = mapTile.getFlipped();
+
+            std::shared_ptr<TileData> tileData;
+            std::lock_guard<std::mutex> lock(_mutex);
+            bool tileOk = false;
+            const int zoom = mapTile.getZoom();
+            // Fast path: try already open packages
+            for (auto it = _cachedOpenDataSources.begin(); it != _cachedOpenDataSources.end(); it++)
+            {
+                auto dataSource = it->second;
+                if (zoom < dataSource->getMinZoom() || zoom > dataSource->getMaxZoom()) {
+                    continue;
+                }
+
+                std::shared_ptr<PackageTileMask> tileMask = it->first;
+                if (tileMask && tileMask->getTileStatus(mapTileFlipped) == PackageTileStatus::PACKAGE_TILE_STATUS_MISSING) {
+                    continue;
+                }
+
+                tileData = dataSource->loadTile(mapTile);
+                tileOk = tileData && tileData->getData();
+                if (tileOk)
+                {
+                    std::rotate(_cachedOpenDataSources.begin(), it, it + 1);
+                    break;
+                }
+            }
+            if (!tileOk && _cachedOpenDataSources.size() != _dataSources.size())
+            {
+                // Slow path: try other packages
+                for (auto it = _dataSources.begin(); it != _dataSources.end(); it++)
+                {
+                    if (auto dataSource = std::dynamic_pointer_cast<TileDataSource>(it->second))
+                    {
+                        auto it2 = std::find_if(_cachedOpenDataSources.begin(), _cachedOpenDataSources.end(), [&it](const std::pair<std::shared_ptr<PackageTileMask>, std::shared_ptr<TileDataSource>> pair)
+                        {  return pair.second == it->second; });
+                        if (it2 != _cachedOpenDataSources.end() && it2->second == it->second) {
+                            continue;
+                        }
+                        if (zoom < dataSource->getMinZoom() || zoom > dataSource->getMaxZoom()) {
+                            continue;
+                        }
+                        std::shared_ptr<PackageTileMask> tileMask = it->first;
+                        if (tileMask && tileMask->getTileStatus(mapTileFlipped) == PackageTileStatus::PACKAGE_TILE_STATUS_MISSING) {
+                            continue;
+                        }
+
+                        tileData = dataSource->loadTile(mapTile);
+                        tileOk = tileData && tileData->getData();
+                        if (tileOk) {
+                            _cachedOpenDataSources.insert(_cachedOpenDataSources.begin(), std::make_pair(tileMask, dataSource));
+                            if (_cachedOpenDataSources.size() > _maxOpenedPackages)
+                            {
+                                _cachedOpenDataSources.pop_back();
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+
+            if (!tileOk)
+            {
+
+                if (mapTile.getZoom() > getMinZoom())
+                {
+                    Log::Infof("MultiTileDataSource::loadTile: Tile data doesn't exist in the database, redirecting to parent");
+                    if (!tileData){
+                        tileData = std::make_shared<TileData>(std::shared_ptr<BinaryData>());
+                    }
+                    tileData->setReplaceWithParent(true);
+                }
+                else
+                {
+                    Log::Infof("MultiTileDataSource::loadTile: Tile data doesn't exist in the database");
+                    return std::shared_ptr<TileData>();
+                }
+            }
+            return tileData;
+        }
+        catch (const std::exception &ex)
+        {
+            Log::Errorf("PackageManagerTileDataSource::loadTile: Exception: %s", ex.what());
+        }
+        return std::shared_ptr<TileData>();
+    }
+
+    void MultiTileDataSource::onPackagesChanged(ChangeType changeType)
+    {
+        {
+            std::lock_guard<std::mutex> lock(_mutex);
+            _cachedOpenDataSources.clear();
+        }
+        notifyTilesChanged(changeType == PACKAGES_DELETED); // we need to remove tiles only if packages were deleted
+    }
+
+    void MultiTileDataSource::add(const std::shared_ptr<TileDataSource> &dataSource)
+    {
+        add(dataSource, std::string());
+    }
+
+    void MultiTileDataSource::add(const std::shared_ptr<TileDataSource> &dataSource, const std::string& tileMaskArg)
+    {
+        {
+            std::lock_guard<std::mutex> lock(_mutex);
+            auto it = std::find_if(_dataSources.begin(), _dataSources.end(), [&dataSource](const std::pair<std::shared_ptr<PackageTileMask>, std::shared_ptr<TileDataSource>> pair)
+            { return pair.second == dataSource; });
+            if (it != _dataSources.end()) {
+                return;
+            }
+            std::shared_ptr<PackageTileMask> tileMask;
+            std::string tileMaskStr = tileMaskArg;
+            if (tileMaskStr.empty()) {
+                if (auto mbtilesDatasource = std::dynamic_pointer_cast<MBTilesTileDataSource>(dataSource)) {
+                    tileMaskStr = mbtilesDatasource->getMetaData("tilemask");
+                }
+            }
+            if (!tileMaskStr.empty())
+            {
+                std::vector<std::string> parts = GeneralUtils::Split(tileMaskStr, ':');
+                if (!parts.empty())
+                {
+                    int zoomLevel;
+                    if (parts.size() > 1)
+                    {
+                        zoomLevel = boost::lexical_cast<int>(parts[1]);
+                    }
+                    else
+                    {
+                        zoomLevel = dataSource->getMaxZoom();
+                    }
+                    tileMask = std::make_shared<PackageTileMask>(parts[0], zoomLevel);
+                }
+            }
+            dataSource->registerOnChangeListener(_dataSourceListener);
+            _dataSources.emplace_back(tileMask, dataSource);
+        }
+        onPackagesChanged(PACKAGES_ADDED);
+    }
+
+
+    bool MultiTileDataSource::remove(const std::shared_ptr<TileDataSource> &dataSource)
+    {
+        {
+            std::lock_guard<std::mutex> lock(_mutex);
+            auto it = std::remove_if(_dataSources.begin(), _dataSources.end(), [&dataSource](
+                    const std::pair<std::shared_ptr<PackageTileMask>, std::shared_ptr<TileDataSource>> pair) {
+                return pair.second == dataSource;
+            });
+            if (it == _dataSources.end()) {
+                return false;
+            }
+            dataSource->unregisterOnChangeListener(_dataSourceListener);
+            _dataSources.erase(it);
+        }
+        onPackagesChanged(PACKAGES_DELETED);
+        return true;
+    }
+
+    MultiTileDataSource::DataSourceListener::DataSourceListener(MultiTileDataSource& combinedDataSource) :
+        _combinedDataSource(combinedDataSource)
+    {
+    }
+
+    void MultiTileDataSource::DataSourceListener::onTilesChanged(bool removeTiles) {
+        _combinedDataSource.notifyTilesChanged(removeTiles);
+    }
+}