Merge pull request #16 from clarin-eric/dev

Dev

.github/workflows/release_wheel.yml

@@ -0,0 +1,106 @@
+name: Build and release wheel
+
+on:
+  push:
+    branches:
+      - "master"
+      - "wheel"
+    tags:
+    - '[0-9]+.[0-9]+.[0-9]+'
+    - '[0-9]+.[0-9]+.[0-9]+rc[x0-9]+'
+    - '[0-9]+.[0-9]+.[0-9]+-rc[x0-9]+'
+  pull_request:
+  release:
+    types: [published]
+  workflow_dispatch:
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  prepare:
+    name: "Prepare"
+    runs-on: ubuntu-22.04
+    outputs:
+      version: ${{ steps.get_version.outputs.VERSION }}
+      create_release: ${{ steps.check_release.outputs.CREATE_RELEASE }}
+    steps:
+      - name: Get the version
+        id: get_version
+        run: echo "VERSION=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
+      - name: Check create release for tag
+        if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+        id: check_release
+        run: |
+          URL="https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ steps.get_version.outputs.VERSION }}"
+          StatusCode=$(curl -o -I -L -s -w "%{http_code}" -X GET -G $URL)
+          if [ "$StatusCode" == 200 ]; then
+            echo "Release exists"
+            echo "CREATE_RELEASE=false" >> $GITHUB_OUTPUT
+          else
+            echo "Release does not exist"
+            echo "CREATE_RELEASE=true" >> $GITHUB_OUTPUT
+          fi
+  build_wheel:
+    needs: [prepare]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build wheel
+        run: |
+          cd ..
+          docker run -v ./DOGapp:/src -v ./whl:/whl registry.gitlab.com/clarin-eric/docker-alpine-wheeler:0.0.1-a10
+      - name: 'Tar wheeler output and changelog'
+        run: tar -cvf workspace.tar ../whl/*.whl ./CHANGELOG.md
+      - uses: actions/upload-artifact@v4
+        with:
+          name: workspace
+          path: workspace.tar
+          retention-days: 1
+  release:
+    name: "Release"
+    needs: [prepare, build_wheel]
+    # Run job for github releases and tag pushes (without github release)
+    if: github.event_name == 'release' || needs.prepare.outputs.create_release == 'true'
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+    steps:
+    - name: 'Download workspace'
+      uses: actions/download-artifact@v4
+      with:
+        name: workspace
+    - name: 'Untar workspace'
+      run: tar -xvf workspace.tar
+    # For github releases -> upload release package to existing release
+    # For tag pushes without github release -> create a github release with release package
+    - name: Create release
+      if: needs.prepare.outputs.create_release == 'true'
+      id: create_release
+      uses: ncipollo/release-action@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag: ${{ needs.prepare.outputs.version }}
+        name: ${{ needs.prepare.outputs.version }}
+        draft: false
+        prerelease: true
+        bodyFile: CHANGELOG.md
+    - name: Add to existing release
+      uses: shogo82148/actions-upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        # Pseudo-ternary expression: get "upload_url" from the release created above, or from github "release" event when release is pre-created
+        upload_url: ${{ needs.prepare.outputs.create_release == 'true' && steps.create_release.outputs.upload_url || github.event.release.upload_url }}
+        asset_path: ./whl/*.whl
+        asset_content_type: application/gzip
+    - uses: eregon/publish-release@v1
+      if: needs.prepare.outputs.create_release == 'true'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        release_id: ${{ steps.create_release.outputs.id }}

CHANGELOG.md

@@ -0,0 +1,26 @@
+#Changelog
+
+
+## [1.0.2] - 16.04.2024
+- build wheel in CI and distribute Alpine friendly wheels for internal CLARIN
+  use
+- new API endpoint `/expanddatatype?data_type=<data_type>` (not operational yet)
+- fetch includes `/expanddatatype` response by default
+
+## [1.0.1] - never released
+Changes:
+- Fixed stability issue by providing shared cache for uwsgi apps
+- `dogui` Django raw HTML templates
+- Unittests for parameter parsing
+- Fixed CI pipeline
+- Build config with `.toml + Poetry`
+- Unittests for endpoints
+- OpenAPI 3.0 endpoint documentation
+- Upgrade Python==3.11 and Django==4.2
+- Fix whitespaces in PID form breaking DOG
+- Replace `{**A, **B}` to Python >=3.9 `A | B`
+
+## [1.0.0] - 17.11.2022
+First release of Digital Object Gate web application
+- Expose DOGlib functionality as REST API microservice
+

README.md

@@ -1,16 +1,41 @@
 # DOGapp
-DOGapp is a Django application providing REST API for [DOGlib](https://github.com/clarin-eric/DOGlib) functionalities. 
-Its goal is to provide fair access to CLARIN resources from the metadata
+DOGapp is a modelless Django application exposing REST API for [DOGlib](https://github.com/clarin-eric/DOGlib) functionalities. 
+Its goal is to provide fair access to CLARIN resources from the metadata by exposing both UI interface and API endpoints. 
 
 ## API
 PID's can be passed as parameters to URL query in following formats:
 >?pid=val1&pid=val2&pid=val3 \
->?pid=val1,val2,val3
+>?pid=val1,val2,val3 \
+>?pid[]=val1&pid[]=val2
 
-`/sniff/?pid`, Checks whether PID points to resources in registered repository.
+Available endpoints:
 
-`/fetch/?pid`, Fetches all PIDs referenced in the metadata, supports PID and list of PIDs to metadata in formats.
+`/api/schema`, provides OpenApi 3.0 specification of the API.
 
-`/identify/?pid`, identifies PID (VLO request).
+`/api/sniff/?pid`, Checks whether PID points to resources in registered repository.
 
-`/swagger.json`, provides OpenApi 2.0 specification of the API.
+`/api/fetch/?pid`, Fetches all PIDs referenced in the metadata, supports PID and list of PIDs to metadata in formats.
+
+`/api/identify/?pid`, identifies PID (VLO request).
+
+## Installation
+Install packages by running following commands in the project's root dir (order matters):
+```bash
+pip install ./dogapi
+pip install ./dogui
+pip install ./dogconfig
+```
+
+## Testing
+In order to run the unittests simply run
+```bash
+python ./runtests.py
+```
+in the project's root directory. The script will take care of providing minimal testing configuration and testing all the applications
+
+## Running
+To start the project local installation of `dogapi` and `dogui` is required, config can be spinned from source by running
+```bash
+python ./dogconfig/manage.py runserver
+```
+By default DOG homepage should be available at your `localhost:8000` and API endpoints at `localhost:8000/api/<functionality>`

dogapi/dogapi/admin.py

@@ -1,3 +1 @@
 from django.contrib import admin
-
-# Register your models here.

dogapi/dogapi/models.py

@@ -1,5 +1,6 @@
 from doglib import DOG
 from doglib import expand_datatype as doglib_expand_datatype
+from doglib.pid import pid_factory as dog_pid_factory
 
 dog = DOG()
-
+pid_factory = dog_pid_factory

dogapi/dogapi/schemas.py

@@ -1,72 +1,26 @@
-from drf_yasg import openapi
+from drf_spectacular.utils import OpenApiParameter, OpenApiExample
 
 
-pid_queryparam: openapi.Parameter = openapi.Parameter(
-    name='pid',
-    in_=openapi.IN_QUERY,
-    description='Persistent identifier to a collection',
-    type=openapi.TYPE_ARRAY,
-    items=openapi.Items(type=openapi.TYPE_STRING))
 
-fetch_response_schema: openapi.Schema = openapi.Schema(
-    title="Fetch response schema",
-    description="Parse referenced PID's",
-    type=openapi.TYPE_OBJECT,
-    required=["pid"],
-    properties={
-        "pid": openapi.Schema(type=openapi.TYPE_OBJECT,
-                              required=["ref_files", "description", "license"],
-                              properties={
-                                  "ref_files": openapi.Schema(type=openapi.TYPE_ARRAY,
-                                                              description="Referenced PIDs in collection metadata",
-                                                              items=openapi.Schema(type=openapi.TYPE_STRING,
-                                                                                   description="Referenced PID")),
-                                  "description": openapi.Schema(type=openapi.TYPE_STRING,
-                                                                description="Collection's description"),
-                                  "license": openapi.Schema(type=openapi.TYPE_STRING,
-                                                            description="Collection's license")
-                              })
-    }
-)
-
-identify_response_schema: openapi.Schema = openapi.Schema(
-    title='Identify response schema',
-    description='Identify collection with its title and description',
-    type=openapi.TYPE_OBJECT,
-    required=["pid"],
-    properties={
-        "pid": openapi.Schema(type=openapi.TYPE_OBJECT,
-                              required=["title", "description"],
-                              properties={
-                                  "title": openapi.Schema(type=openapi.TYPE_STRING,
-                                                          description="Collection's title"),
-                                  "description": openapi.Schema(type=openapi.TYPE_STRING,
-                                                                description="Collection's description")
-                              })
-    }
-)
-
-is_collection_response_schema: openapi.Schema = openapi.Schema(
-    title='Is collection response schema',
-    description='Check whether PID points to another collection',
-    type=openapi.TYPE_BOOLEAN
-)
-
-sniff_response_schema: openapi.Schema = openapi.Schema(
-    title="Sniff response schema",
-    description="Collection's host repository identification",
-    type=openapi.TYPE_OBJECT,
-    required=["pid"],
-    properties={
-        "pid": openapi.Schema(type=openapi.TYPE_OBJECT,
-                              required=["name", "host_name", "host_netloc"],
-                              properties={
-                                  "name": openapi.Schema(type=openapi.TYPE_STRING,
-                                                         description="Name of the repository"),
-                                  "host_name": openapi.Schema(type=openapi.TYPE_STRING,
-                                                              description="Name of the hosting service used by a repository"),
-                                  "host_netloc": openapi.Schema(type=openapi.TYPE_STRING,
-                                                                description="Base URL to the repository")
-                              })
-    }
-)
+pid_queryparam: OpenApiParameter = OpenApiParameter(name='pid',
+                                                    location=OpenApiParameter.QUERY,
+                                                    description='Persistent identifier(s)',
+                                                    required=True,
+                                                    type={'type': 'array',
+                                                          'items': 'str'},
+                                                    examples=[
+                                                        OpenApiExample(
+                                                            'HDL',
+                                                            description='Handle',
+                                                            value='http://hdl.handle.net/21.11115/0000-000C-4E21-8'),
+                                                        OpenApiExample(
+                                                            'DOI',
+                                                            description='Digital Object Identifier',
+                                                            value='https://doi.org/10.15155/9-00-0000-0000-0000-001ABL'
+                                                        ),
+                                                        OpenApiExample(
+                                                            'URL',
+                                                            description='Uniform Resource Locator',
+                                                            value='https://clarin-pl.eu/dspace/handle/11321/6?format=cmdi'
+                                                        )
+                                                    ])