-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #16 from clarin-eric/dev
Dev
- Loading branch information
Showing
32 changed files
with
1,205 additions
and
311 deletions.
There are no files selected for viewing
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
name: Build and release wheel | ||
|
||
on: | ||
push: | ||
branches: | ||
- "master" | ||
- "wheel" | ||
tags: | ||
- '[0-9]+.[0-9]+.[0-9]+' | ||
- '[0-9]+.[0-9]+.[0-9]+rc[x0-9]+' | ||
- '[0-9]+.[0-9]+.[0-9]+-rc[x0-9]+' | ||
pull_request: | ||
release: | ||
types: [published] | ||
workflow_dispatch: | ||
defaults: | ||
run: | ||
shell: bash | ||
|
||
jobs: | ||
prepare: | ||
name: "Prepare" | ||
runs-on: ubuntu-22.04 | ||
outputs: | ||
version: ${{ steps.get_version.outputs.VERSION }} | ||
create_release: ${{ steps.check_release.outputs.CREATE_RELEASE }} | ||
steps: | ||
- name: Get the version | ||
id: get_version | ||
run: echo "VERSION=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT | ||
- name: Check create release for tag | ||
if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') | ||
id: check_release | ||
run: | | ||
URL="https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ steps.get_version.outputs.VERSION }}" | ||
StatusCode=$(curl -o -I -L -s -w "%{http_code}" -X GET -G $URL) | ||
if [ "$StatusCode" == 200 ]; then | ||
echo "Release exists" | ||
echo "CREATE_RELEASE=false" >> $GITHUB_OUTPUT | ||
else | ||
echo "Release does not exist" | ||
echo "CREATE_RELEASE=true" >> $GITHUB_OUTPUT | ||
fi | ||
build_wheel: | ||
needs: [prepare] | ||
runs-on: ubuntu-22.04 | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v4 | ||
with: | ||
fetch-depth: 0 | ||
- name: Build wheel | ||
run: | | ||
cd .. | ||
docker run -v ./DOGapp:/src -v ./whl:/whl registry.gitlab.com/clarin-eric/docker-alpine-wheeler:0.0.1-a10 | ||
- name: 'Tar wheeler output and changelog' | ||
run: tar -cvf workspace.tar ../whl/*.whl ./CHANGELOG.md | ||
- uses: actions/upload-artifact@v4 | ||
with: | ||
name: workspace | ||
path: workspace.tar | ||
retention-days: 1 | ||
release: | ||
name: "Release" | ||
needs: [prepare, build_wheel] | ||
# Run job for github releases and tag pushes (without github release) | ||
if: github.event_name == 'release' || needs.prepare.outputs.create_release == 'true' | ||
runs-on: ubuntu-22.04 | ||
permissions: | ||
contents: write | ||
steps: | ||
- name: 'Download workspace' | ||
uses: actions/download-artifact@v4 | ||
with: | ||
name: workspace | ||
- name: 'Untar workspace' | ||
run: tar -xvf workspace.tar | ||
# For github releases -> upload release package to existing release | ||
# For tag pushes without github release -> create a github release with release package | ||
- name: Create release | ||
if: needs.prepare.outputs.create_release == 'true' | ||
id: create_release | ||
uses: ncipollo/release-action@v1 | ||
env: | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
with: | ||
tag: ${{ needs.prepare.outputs.version }} | ||
name: ${{ needs.prepare.outputs.version }} | ||
draft: false | ||
prerelease: true | ||
bodyFile: CHANGELOG.md | ||
- name: Add to existing release | ||
uses: shogo82148/actions-upload-release-asset@v1 | ||
env: | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
with: | ||
# Pseudo-ternary expression: get "upload_url" from the release created above, or from github "release" event when release is pre-created | ||
upload_url: ${{ needs.prepare.outputs.create_release == 'true' && steps.create_release.outputs.upload_url || github.event.release.upload_url }} | ||
asset_path: ./whl/*.whl | ||
asset_content_type: application/gzip | ||
- uses: eregon/publish-release@v1 | ||
if: needs.prepare.outputs.create_release == 'true' | ||
env: | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
with: | ||
release_id: ${{ steps.create_release.outputs.id }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
#Changelog | ||
|
||
|
||
## [1.0.2] - 16.04.2024 | ||
- build wheel in CI and distribute Alpine friendly wheels for internal CLARIN | ||
use | ||
- new API endpoint `/expanddatatype?data_type=<data_type>` (not operational yet) | ||
- fetch includes `/expanddatatype` response by default | ||
|
||
## [1.0.1] - never released | ||
Changes: | ||
- Fixed stability issue by providing shared cache for uwsgi apps | ||
- `dogui` Django raw HTML templates | ||
- Unittests for parameter parsing | ||
- Fixed CI pipeline | ||
- Build config with `.toml + Poetry` | ||
- Unittests for endpoints | ||
- OpenAPI 3.0 endpoint documentation | ||
- Upgrade Python==3.11 and Django==4.2 | ||
- Fix whitespaces in PID form breaking DOG | ||
- Replace `{**A, **B}` to Python >=3.9 `A | B` | ||
|
||
## [1.0.0] - 17.11.2022 | ||
First release of Digital Object Gate web application | ||
- Expose DOGlib functionality as REST API microservice | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,41 @@ | ||
# DOGapp | ||
DOGapp is a Django application providing REST API for [DOGlib](https://github.com/clarin-eric/DOGlib) functionalities. | ||
Its goal is to provide fair access to CLARIN resources from the metadata | ||
DOGapp is a modelless Django application exposing REST API for [DOGlib](https://github.com/clarin-eric/DOGlib) functionalities. | ||
Its goal is to provide fair access to CLARIN resources from the metadata by exposing both UI interface and API endpoints. | ||
|
||
## API | ||
PID's can be passed as parameters to URL query in following formats: | ||
>?pid=val1&pid=val2&pid=val3 \ | ||
>?pid=val1,val2,val3 | ||
>?pid=val1,val2,val3 \ | ||
>?pid[]=val1&pid[]=val2 | ||
`/sniff/?pid`, Checks whether PID points to resources in registered repository. | ||
Available endpoints: | ||
|
||
`/fetch/?pid`, Fetches all PIDs referenced in the metadata, supports PID and list of PIDs to metadata in formats. | ||
`/api/schema`, provides OpenApi 3.0 specification of the API. | ||
|
||
`/identify/?pid`, identifies PID (VLO request). | ||
`/api/sniff/?pid`, Checks whether PID points to resources in registered repository. | ||
|
||
`/swagger.json`, provides OpenApi 2.0 specification of the API. | ||
`/api/fetch/?pid`, Fetches all PIDs referenced in the metadata, supports PID and list of PIDs to metadata in formats. | ||
|
||
`/api/identify/?pid`, identifies PID (VLO request). | ||
|
||
## Installation | ||
Install packages by running following commands in the project's root dir (order matters): | ||
```bash | ||
pip install ./dogapi | ||
pip install ./dogui | ||
pip install ./dogconfig | ||
``` | ||
|
||
## Testing | ||
In order to run the unittests simply run | ||
```bash | ||
python ./runtests.py | ||
``` | ||
in the project's root directory. The script will take care of providing minimal testing configuration and testing all the applications | ||
|
||
## Running | ||
To start the project local installation of `dogapi` and `dogui` is required, config can be spinned from source by running | ||
```bash | ||
python ./dogconfig/manage.py runserver | ||
``` | ||
By default DOG homepage should be available at your `localhost:8000` and API endpoints at `localhost:8000/api/<functionality>` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1 @@ | ||
from django.contrib import admin | ||
|
||
# Register your models here. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
from doglib import DOG | ||
from doglib import expand_datatype as doglib_expand_datatype | ||
from doglib.pid import pid_factory as dog_pid_factory | ||
|
||
dog = DOG() | ||
|
||
pid_factory = dog_pid_factory |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,72 +1,26 @@ | ||
from drf_yasg import openapi | ||
from drf_spectacular.utils import OpenApiParameter, OpenApiExample | ||
|
||
|
||
pid_queryparam: openapi.Parameter = openapi.Parameter( | ||
name='pid', | ||
in_=openapi.IN_QUERY, | ||
description='Persistent identifier to a collection', | ||
type=openapi.TYPE_ARRAY, | ||
items=openapi.Items(type=openapi.TYPE_STRING)) | ||
|
||
fetch_response_schema: openapi.Schema = openapi.Schema( | ||
title="Fetch response schema", | ||
description="Parse referenced PID's", | ||
type=openapi.TYPE_OBJECT, | ||
required=["pid"], | ||
properties={ | ||
"pid": openapi.Schema(type=openapi.TYPE_OBJECT, | ||
required=["ref_files", "description", "license"], | ||
properties={ | ||
"ref_files": openapi.Schema(type=openapi.TYPE_ARRAY, | ||
description="Referenced PIDs in collection metadata", | ||
items=openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Referenced PID")), | ||
"description": openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Collection's description"), | ||
"license": openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Collection's license") | ||
}) | ||
} | ||
) | ||
|
||
identify_response_schema: openapi.Schema = openapi.Schema( | ||
title='Identify response schema', | ||
description='Identify collection with its title and description', | ||
type=openapi.TYPE_OBJECT, | ||
required=["pid"], | ||
properties={ | ||
"pid": openapi.Schema(type=openapi.TYPE_OBJECT, | ||
required=["title", "description"], | ||
properties={ | ||
"title": openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Collection's title"), | ||
"description": openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Collection's description") | ||
}) | ||
} | ||
) | ||
|
||
is_collection_response_schema: openapi.Schema = openapi.Schema( | ||
title='Is collection response schema', | ||
description='Check whether PID points to another collection', | ||
type=openapi.TYPE_BOOLEAN | ||
) | ||
|
||
sniff_response_schema: openapi.Schema = openapi.Schema( | ||
title="Sniff response schema", | ||
description="Collection's host repository identification", | ||
type=openapi.TYPE_OBJECT, | ||
required=["pid"], | ||
properties={ | ||
"pid": openapi.Schema(type=openapi.TYPE_OBJECT, | ||
required=["name", "host_name", "host_netloc"], | ||
properties={ | ||
"name": openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Name of the repository"), | ||
"host_name": openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Name of the hosting service used by a repository"), | ||
"host_netloc": openapi.Schema(type=openapi.TYPE_STRING, | ||
description="Base URL to the repository") | ||
}) | ||
} | ||
) | ||
pid_queryparam: OpenApiParameter = OpenApiParameter(name='pid', | ||
location=OpenApiParameter.QUERY, | ||
description='Persistent identifier(s)', | ||
required=True, | ||
type={'type': 'array', | ||
'items': 'str'}, | ||
examples=[ | ||
OpenApiExample( | ||
'HDL', | ||
description='Handle', | ||
value='http://hdl.handle.net/21.11115/0000-000C-4E21-8'), | ||
OpenApiExample( | ||
'DOI', | ||
description='Digital Object Identifier', | ||
value='https://doi.org/10.15155/9-00-0000-0000-0000-001ABL' | ||
), | ||
OpenApiExample( | ||
'URL', | ||
description='Uniform Resource Locator', | ||
value='https://clarin-pl.eu/dspace/handle/11321/6?format=cmdi' | ||
) | ||
]) |
Oops, something went wrong.