Release 0.2.0

.github/workflows/build-production.yml

@@ -0,0 +1,79 @@
+name: Build Docker images for geolake components and push to the repository
+
+on:
+  push:
+    tags:        
+      - 'v*'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Install build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source for drivers
+      run: python3 -m build ./drivers    
+    - name: Set Docker image tag name
+      run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
+    - name: Login to Scaleway Container Registry
+      uses: docker/login-action@v2
+      with:
+        username: nologin
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        registry: ${{ vars.DOCKER_REGISTRY }}
+    - name: Get release tag
+      run: echo "RELEASE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV        
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2     
+    - name: Build and push drivers
+      uses: docker/build-push-action@v4
+      with:
+        context: ./drivers
+        file: ./drivers/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.RELEASE_TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest             
+    - name: Build and push datastore component
+      uses: docker/build-push-action@v4
+      with:
+        context: ./datastore
+        file: ./datastore/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.RELEASE_TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest    
+    - name: Build and push api component
+      uses: docker/build-push-action@v4
+      with:
+        context: ./api
+        file: ./api/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.RELEASE_TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest 
+    - name: Build and push executor component
+      uses: docker/build-push-action@v4
+      with:
+        context: ./executor
+        file: ./executor/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.RELEASE_TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest                           

.github/workflows/build-staging.yml

@@ -0,0 +1,77 @@
+name: Build Docker images for geolake components and push to the repository
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Install build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source for drivers
+      run: python3 -m build ./drivers    
+    - name: Set Docker image tag name
+      run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
+    - name: Login to Scaleway Container Registry
+      uses: docker/login-action@v2
+      with:
+        username: nologin
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        registry: ${{ vars.DOCKER_REGISTRY }}
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2     
+    - name: Build and push drivers
+      uses: docker/build-push-action@v4
+      with:
+        context: ./drivers
+        file: ./drivers/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest             
+    - name: Build and push datastore component
+      uses: docker/build-push-action@v4
+      with:
+        context: ./datastore
+        file: ./datastore/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest    
+    - name: Build and push api component
+      uses: docker/build-push-action@v4
+      with:
+        context: ./api
+        file: ./api/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest 
+    - name: Build and push executor component
+      uses: docker/build-push-action@v4
+      with:
+        context: ./executor
+        file: ./executor/Dockerfile
+        push: true
+        build-args: |
+          REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        tags: | 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }}  
+          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest                           

api/Dockerfile

@@ -1,15 +1,9 @@
-FROM continuumio/miniconda3
-WORKDIR /code
-COPY ./api/requirements.txt /code/requirements.txt
+ARG REGISTRY=rg.nl-ams.scw.cloud/geodds-production
+ARG TAG=latest
+FROM $REGISTRY/geolake-datastore:$TAG
+WORKDIR /app
+COPY requirements.txt /code/requirements.txt
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
-RUN conda install -c anaconda psycopg2
-COPY ./utils/wait-for-it.sh /code/wait-for-it.sh
-COPY ./db/dbmanager /code/db/dbmanager
-COPY ./geoquery/ /code/geoquery
-COPY ./resources /code/app/resources
-COPY ./api/app /code/app
+COPY app /app
 EXPOSE 80
-# VOLUME /code
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"]
-# if behind a proxy use --proxy-headers
-# CMD ["uvicorn", "app.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "80"]
+CMD ["uvicorn", "app.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "80"]

api/app/api_utils.py

@@ -0,0 +1,73 @@
+"""Utils module"""
+
+
+def convert_bytes(size_bytes: int, to: str) -> float:
+    """Converts size in bytes to the other unit - one out of:
+    ["kb", "mb", "gb"]
+
+    Parameters
+    ----------
+    size_bytes : int
+        Size in bytes
+    to : str
+        Unit to convert `size_bytes` to
+
+    size : float
+        `size_bytes` converted to the given unit
+    """
+    assert to is not None, "Expected unit cannot be `None`"
+    to = to.lower()
+    match to:
+        case "bytes":
+            return size_bytes
+        case "kb":
+            return size_bytes / 1024
+        case "mb":
+            return size_bytes / 1024**2
+        case "gb":
+            return size_bytes / 1024**3
+        case _:
+            raise ValueError(f"unsupported units: {to}")
+
+
+def make_bytes_readable_dict(
+    size_bytes: int, units: str | None = None
+) -> dict:
+    """Prepare dictionary representing size (in bytes) in more readable unit
+    to keep value in the range [0,1] - if `units` is `None`.
+    If `units` is not None, converts `size_bytes` to the size expressed by
+    that argument.
+
+    Parameters
+    ----------
+    size_bytes : int
+        Size expressed in bytes
+    units : optional str
+
+    Returns
+    -------
+    result : dict
+        A dictionary with size and units in the form:
+        {
+            "value": ...,
+            "units": ...
+        }
+    """
+    if units is None:
+        units = "bytes"
+    if units != "bytes":
+        converted_size = convert_bytes(size_bytes=size_bytes, to=units)
+        return {"value": converted_size, "units": units}
+    val = size_bytes
+    if val > 1024:
+        units = "kB"
+        val /= 1024
+    if val > 1024:
+        units = "MB"
+        val /= 1024
+    if val > 1024:
+        units = "GB"
+        val /= 1024
+    if val > 0.0 and (round(val, 2) == 0.00):
+        val = 0.01
+    return {"value": round(val, 2), "units": units}

api/app/auth/backend.py

@@ -0,0 +1,66 @@
+"""The module contains authentication backend"""
+from uuid import UUID
+
+from starlette.authentication import (
+    AuthCredentials,
+    AuthenticationBackend,
+    UnauthenticatedUser,
+)
+from dbmanager.dbmanager import DBManager
+
+import exceptions as exc
+from auth.models import DDSUser
+from auth import scopes
+
+
+class DDSAuthenticationBackend(AuthenticationBackend):
+    """Class managing authentication and authorization"""
+
+    async def authenticate(self, conn):
+        """Authenticate user based on `User-Token` header"""
+        if "User-Token" in conn.headers:
+            return self._manage_user_token_auth(conn.headers["User-Token"])
+        return AuthCredentials([scopes.ANONYMOUS]), UnauthenticatedUser()
+
+    def _manage_user_token_auth(self, user_token: str):
+        try:
+            user_id, api_key = self.get_authorization_scheme_param(user_token)
+        except exc.BaseDDSException as err:
+            raise err.wrap_around_http_exception()
+        user_dto = DBManager().get_user_details(user_id)
+        eligible_scopes = [scopes.AUTHENTICATED] + self._get_scopes_for_user(
+            user_dto=user_dto
+        )
+        if user_dto.api_key != api_key:
+            raise exc.AuthenticationFailed(
+                user_dto
+            ).wrap_around_http_exception()
+        return AuthCredentials(eligible_scopes), DDSUser(username=user_id)
+
+    def _get_scopes_for_user(self, user_dto) -> list[str]:
+        if user_dto is None:
+            return []
+        eligible_scopes = []
+        for role in user_dto.roles:
+            if "admin" == role.role_name:
+                eligible_scopes.append(scopes.ADMIN)
+                continue
+            # NOTE: Role-specific scopes
+            # Maybe need some more logic
+            eligible_scopes.append(role.role_name)
+        return eligible_scopes
+
+    def get_authorization_scheme_param(self, user_token: str):
+        """Get `user_id` and `api_key` if authorization scheme is correct."""
+        if user_token is None or user_token.strip() == "":
+            raise exc.EmptyUserTokenError
+        if ":" not in user_token:
+            raise exc.ImproperUserTokenError
+        user_id, api_key, *rest = user_token.split(":")
+        if len(rest) > 0:
+            raise exc.ImproperUserTokenError
+        try:
+            _ = UUID(user_id, version=4)
+        except ValueError as err:
+            raise exc.ImproperUserTokenError from err
+        return (user_id, api_key)

api/app/auth/manager.py

@@ -0,0 +1,72 @@
+"""Module with access/authentication functions"""
+from typing import Optional
+
+from utils.api_logging import get_dds_logger
+import exceptions as exc
+
+log = get_dds_logger(__name__)
+
+
+def is_role_eligible_for_product(
+    product_role_name: Optional[str] = None,
+    user_roles_names: Optional[list[str]] = None,
+):
+    """Check if given role is eligible for the product with the provided
+    `product_role_name`.
+
+    Parameters
+    ----------
+    product_role_name : str, optional, default=None
+        The role which is eligible for the given product.
+        If `None`, product_role_name is claimed to be public
+    user_roles_names: list of str, optional, default=None
+        A list of user roles names. If `None`, user_roles_names is claimed
+        to be public
+
+    Returns
+    -------
+    is_eligible : bool
+        Flag which indicate if any role within the given `user_roles_names`
+        is eligible for the product with `product_role_name`
+    """
+    log.debug(
+        "verifying eligibility of the product role '%s' against roles '%s'",
+        product_role_name,
+        user_roles_names,
+    )
+    if product_role_name == "public" or product_role_name is None:
+        return True
+    if user_roles_names is None:
+        # NOTE: it means, we consider the public profile
+        return False
+    if "admin" in user_roles_names:
+        return True
+    if product_role_name in user_roles_names:
+        return True
+    return False
+
+
+def assert_is_role_eligible(
+    product_role_name: Optional[str] = None,
+    user_roles_names: Optional[list[str]] = None,
+):
+    """Assert that user role is eligible for the product
+
+    Parameters
+    ----------
+    product_role_name : str, optional, default=None
+        The role which is eligible for the given product.
+        If `None`, product_role_name is claimed to be public
+    user_roles_names: list of str, optional, default=None
+        A list of user roles names. If `None`, user_roles_names is claimed
+        to be public
+
+    Raises
+    -------
+    AuthorizationFailed
+    """
+    if not is_role_eligible_for_product(
+        product_role_name=product_role_name,
+        user_roles_names=user_roles_names,
+    ):
+        raise exc.AuthorizationFailed