Merge pull request #292 from ckan/dev-v2.0

Development v2.0 branch
ckan · Jan 26, 2023 · a3dbcd7 · a3dbcd7
2 parents 17d5a34 + 313506f
commit a3dbcd7
Show file tree

Hide file tree

Showing 117 changed files with 33,719 additions and 14,019 deletions.
diff --git a/.github/workflows/test-postgis.yml b/.github/workflows/test-postgis.yml
@@ -0,0 +1,104 @@
+name: Legacy PostGIS Tests
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.7'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        ckan-version: [2.9, 2.9-py2, 2.8, 2.7]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+      postgres:
+        image: postgis/postgis:10-3.1
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+      PGPASSWORD: postgres
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Create Database
+      run: |
+        psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test
+        psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test
+    - name: Install harvester
+      run: |
+        git clone https://github.com/ckan/ckanext-harvest
+        cd ckanext-harvest
+        pip install -r pip-requirements.txt
+        pip install -r dev-requirements.txt
+        pip install -e .
+    - name: Install dependencies (common)
+      run: |
+        apk add --no-cache \
+          geos \
+          geos-dev \
+          proj-util \
+          proj-dev \
+          libxml2 \
+          libxslt \
+          gcc \
+          libxml2-dev \
+          libxslt-dev
+    - name: Install dependencies (python2)
+      if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }}
+      run: |
+        apk add --no-cache \
+          python2-dev
+        pip install -r requirements-py2.txt
+        pip install -r requirements-postgis-py2.txt
+    - name: Install dependencies (python3)
+      if: ${{ matrix.ckan-version != '2.9-py2' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.7' }}
+      run: |
+        apk add --no-cache \
+          python3-dev
+        pip install -r requirements.txt
+        pip install -r requirements-postgis.txt
+    - name: Install requirements
+      run: |
+        pip install -e .
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+    - name: setup postgis
+      run: |
+        psql --host=postgres --username=postgres -d ckan_test --command="ALTER ROLE ckan_default WITH superuser;"
+        psql --host=postgres --username=postgres -d ckan_test --command="CREATE EXTENSION postgis;"
+    - name: Run tests
+      run: pytest --ckan-ini=test-postgis.ini --cov=ckanext.spatial --cov-report=xml --cov-append --disable-warnings ckanext/spatial/tests/postgis
+
+    - name: Upload coverage report to codecov
+      uses: codecov/codecov-action@v3
+      with:
+        file: ./coverage.xml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,8 +5,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - name: Install requirements
@@ -17,7 +17,13 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: [master, 2.9, 2.9-py2, 2.8, 2.7]
+        include:
+          - ckan-version: "2.10"
+            solr-image: "2.10-spatial"
+          - ckan-version: 2.9
+            solr-image: 2.9-solr8-spatial
+          - ckan-version: 2.9-py2
+            solr-image: 2.9-py2-solr8-spatial
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -26,9 +32,9 @@ jobs:
       image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
     services:
       solr:
-        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+        image: ckan/ckan-solr:${{ matrix.solr-image }}
       postgres:
-        image: postgis/postgis:10-3.1
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -45,22 +51,15 @@ jobs:
       PGPASSWORD: postgres
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Create Database
-      run: |
-        psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test
-        psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-        psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test
+    - uses: actions/checkout@v3
     - name: Install harvester
       run: |
         git clone https://github.com/ckan/ckanext-harvest
         cd ckanext-harvest
         pip install -r pip-requirements.txt
         pip install -r dev-requirements.txt
         pip install -e .
-    - name: Install dependency (common)
+    - name: Install dependencies (common)
       run: |
         apk add --no-cache \
           geos \
@@ -72,13 +71,13 @@ jobs:
           gcc \
           libxml2-dev \
           libxslt-dev
-    - name: Install dependency (python2)
+    - name: Install dependencies (python2)
       if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }}
       run: |
         apk add --no-cache \
           python2-dev
         pip install -r requirements-py2.txt
-    - name: Install dependency (python3)
+    - name: Install dependencies (python3)
       if: ${{ matrix.ckan-version != '2.9-py2' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.7' }}
       run: |
         apk add --no-cache \
@@ -89,14 +88,10 @@ jobs:
         pip install -e .
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-    - name: setup postgis
-      run: |
-        psql --host=postgres --username=postgres -d ckan_test --command="ALTER ROLE ckan_default WITH superuser;"
-        psql --host=postgres --username=postgres -d ckan_test --command="CREATE EXTENSION postgis;"
     - name: Run tests
       run: pytest --ckan-ini=test.ini --cov=ckanext.spatial --cov-report=xml --cov-append --disable-warnings ckanext/spatial/tests
 
     - name: Upload coverage report to codecov
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v3
       with:
         file: ./coverage.xml
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,13 +1,3 @@
 recursive-include ckanext/spatial/templates                                           *
 recursive-include ckanext/spatial/public                                              *
 recursive-include ckanext/spatial/validation/xml                                      *
-
-recursive-include ckanext/spatial/public/ckanext/spatial/js/openlayers/               *.js
-recursive-include ckanext/spatial/public/ckanext/spatial/js/openlayers/               *.txt
-recursive-include ckanext/spatial/public/ckanext/spatial/js/openlayers/               *.cfg
-recursive-include ckanext/spatial/public/ckanext/spatial/js/openlayers/img/           *.gif
-recursive-include ckanext/spatial/public/ckanext/spatial/js/openlayers/img/           *.png
-recursive-include ckanext/spatial/public/ckanext/spatial/js/openlayers/theme/default/ *.css
-recursive-include ckanext/spatial/public/ckanext/spatial/js/                          *.js
-recursive-include ckanext/spatial/public/ckanext/spatial/                             *.css
-
diff --git a/README.rst b/README.rst
@@ -23,6 +23,13 @@ Full documentation, including installation instructions, can be found at:
 
 https://docs.ckan.org/projects/ckanext-spatial/en/latest/
 
+Supported Versions
+------------------
+
+ckanext-spatial >= 2.0.0 supports CKAN 2.9 and CKAN 2.10.
+
+For previous CKAN versions please use the v1.x tags.
+
 
 Community
 ---------

diff --git a/ckanext/spatial/controllers/api.py b/ckanext/spatial/controllers/api.py
@@ -1,50 +1,16 @@
 import logging
 
-from pylons import response
-
-from ckan.lib.base import request, abort
+from ckan.lib.base import abort
 from ckan.controllers.api import ApiController as BaseApiController
 from ckan.model import Session
+from ckantoolkit import response
 
 from ckanext.harvest.model import HarvestObject, HarvestObjectExtra
-from ckanext.spatial.lib import get_srid, validate_bbox, bbox_query
 from ckanext.spatial import util
 
 log = logging.getLogger(__name__)
 
 
-class ApiController(BaseApiController):
-
-    def spatial_query(self):
-
-        error_400_msg = \
-            'Please provide a suitable bbox parameter [minx,miny,maxx,maxy]'
-
-        if 'bbox' not in request.params:
-            abort(400, error_400_msg)
-
-        bbox = validate_bbox(request.params['bbox'])
-
-        if not bbox:
-            abort(400, error_400_msg)
-
-        srid = get_srid(request.params.get('crs')) if 'crs' in \
-            request.params else None
-
-        extents = bbox_query(bbox, srid)
-
-        format = request.params.get('format', '')
-
-        return self._output_results(extents, format)
-
-    def _output_results(self, extents, format=None):
-
-        ids = [extent.package_id for extent in extents]
-        output = dict(count=len(ids), results=ids)
-
-        return self._finish_ok(output)
-
-
 class HarvestMetadataApiController(BaseApiController):
 
     def _get_content(self, id):

diff --git a/ckanext/spatial/geoalchemy_common.py b/ckanext/spatial/geoalchemy_common.py
diff --git a/ckanext/spatial/model/harvested_metadata.py → ckanext/spatial/harvested_metadata.py b/ckanext/spatial/model/harvested_metadata.py → ckanext/spatial/harvested_metadata.py
diff --git a/ckanext/spatial/harvesters/base.py b/ckanext/spatial/harvesters/base.py
@@ -31,7 +31,7 @@
 from ckanext.harvest.model import HarvestObject
 
 from ckanext.spatial.validation import Validators, all_validators
-from ckanext.spatial.model import ISODocument
+from ckanext.spatial.harvested_metadata import ISODocument
 from ckanext.spatial.interfaces import ISpatialHarvester
 from ckantoolkit import config
 
@@ -133,7 +133,7 @@ def guess_resource_format(resource_locator, use_mimetypes=True):
     resource_type = protocols.get(protocol)
     if resource_type:
         return resource_type
-    
+
     url = resource_locator.get('url').lower().strip()
 
     resource_types = {

diff --git a/ckanext/spatial/harvesters/gemini.py b/ckanext/spatial/harvesters/gemini.py
@@ -33,7 +33,7 @@
 from ckanext.harvest.interfaces import IHarvester
 from ckanext.harvest.model import HarvestObject
 
-from ckanext.spatial.model import GeminiDocument
+from ckanext.spatial.harvested_metadata import GeminiDocument
 from ckanext.spatial.lib.csw_client import CswService
 
 from ckanext.spatial.harvesters.base import SpatialHarvester, text_traceback