Merge pull request #21 from crim-ca/DAC-524-baseline-V2

Completes implementation part of https://crim-ca.atlassian.net/browse/DAC-551
Ready for rebuild of docker image and deployment.

.github/workflows/tests.yml

@@ -0,0 +1,106 @@
+# run test suites
+
+name: Tests
+on:
+  - pull_request
+  - push
+
+jobs:
+  # see: https://github.com/fkirc/skip-duplicate-actions
+  skip_duplicate:
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          concurrent_skipping: "same_content"
+          skip_after_successful_duplicate: "true"
+          do_not_skip: '["pull_request", "workflow_dispatch", "schedule"]'
+
+  # NOTE:
+  #   Run all the steps even if there are no tests defined for a given domain sub-directory.
+  #   This is to make sure that the environment definition is at the very least buildable.
+  tests:
+    needs: skip_duplicate
+    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.allow-failure }}
+    env:
+      CACHE_NUMBER: 0  # increment to reset cache
+
+    # ensure conda env activation is performed automatically
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        # somehow mamba with python 3.12 doesn't resolve spacy although available...
+        python-version: ["3.9", "3.10", "3.11"]
+        allow-failure: [false]
+        domain: ["eo", "nlp"]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+
+      - name: Setup Mamba
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+            auto-update-conda: true
+            python-version: ${{ matrix.python-version }}
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            activate-environment: github-ci-test-python${{ matrix.python-version }}-${{ matrix.domain }}
+            use-mamba: true
+            use-only-tar-bz2: true
+
+      - name: Set cache date
+        run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+
+      - uses: actions/cache@v2
+        id: cache
+        with:
+          path: ${{ env.CONDA }}/envs/github-ci-test-python${{ matrix.python-version }}-${{ matrix.domain }}
+          key: conda-python${{ matrix.python-version }}-${{ matrix.domain }}-${{ hashFiles('${{ matrix.domain }}/environment.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+
+      - name: Display Python
+        run: which python
+
+      - name: Update environment
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          echo "python=${{ matrix.python-version }}" > ${{ env.CONDA }}/envs/github-ci-test-python${{ matrix.python-version }}-${{ matrix.domain }}/conda-meta/pinned
+          mamba env update \
+            -n github-ci-test-python${{ matrix.python-version }}-${{ matrix.domain }} \
+            -f ${{ matrix.domain }}/environment.yml
+
+      - name: Display Packages
+        if: ${{ matrix.python-version != 'none' }}
+        run: pip freeze
+
+      - name: Display Environment Variables
+        run: |
+          hash -r
+          env | sort
+
+      - name: Check Tests
+        id: check_tests
+        run: |
+          echo "HAS_TEST_DIR=$(test -d ${{ matrix.domain }}/tests && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT
+
+      - name: Install Tests Dependencies
+        if: ${{ steps.check_tests.outputs.HAS_TEST_DIR == 'true' }}
+        run: pip install -r requirements-dev.txt
+
+      - name: Run Tests
+        if: ${{ steps.check_tests.outputs.HAS_TEST_DIR == 'true' }}
+        run: |
+          cd ${{ matrix.domain }}/notebooks
+          python -m pytest -vvv ../tests

.gitignore

@@ -1,8 +1,25 @@
 ### IDE
 **/.idea/
 **/.vscode/
+**/*.code-workspace
+
+### Caches
+**/__pycache__/
+**/cache/
+**/*tree-tagger-linux*
+**/.pytest_cache
+**/condaenv.*.requirements.txt
+
+## Chroma VDB caches
+**/*.bin
+**/*.pickle
+**/*.sqlite3
+
+### Binaries
+**/*.jar
 
 ### Notebooks
 # expect examples per domain
+# disallow notebooks at root
 ./*.ipynb
 **/.ipynb_checkpoints/

eo/environment.yml

@@ -9,7 +9,6 @@ dependencies:
   - intake-stac
   - pyproj
   - rasterio
-  - sat-search
   - shapely
 
   # TODO: These next packages could possibly be added to a more generic 'vision' image, from which 'eo' would be built

nlp/Dockerfile

@@ -39,6 +39,15 @@ RUN cd heideltime && \
     rmdir heideltime-standalone && \
     rm heideltime-standalone-2.2.1.tar.gz
 
+# Setup Haskell for Duckling server
+# https://github.com/facebook/duckling
+RUN curl -sSL https://get.haskellstack.org/ | sh && \
+    git clone https://github.com/facebook/duckling && \
+    cd duckling && \
+    stack build && \
+    stack install && \
+    rm -fr duckling
+
 # Give read&write permission to jenkins for config
 RUN chown -R jenkins heideltime
 
@@ -47,4 +56,3 @@ RUN mkdir /opt/conda/pkgs/cache && chown -R 1000:1000 /opt/conda/pkgs/cache
 
 # specify user because of problem running start-notebook.sh when being root
 USER jenkins
-

nlp/environment.yml

@@ -5,19 +5,26 @@ channels:
   - conda-forge
 
 dependencies:
-  - intake-esm==2021.1.15
-  - intake-stac==0.3.0
-  - sat-search==0.3.0
+  - intake-esm
+  - intake-stac>=0.4.0
   - threddsclient==0.4.2
-  - openjdk==8.0.152 
-  # python-flair=0.8 only works with numpy<=1.19.5
-  - python-flair=0.8
-  - numpy<=1.19.5
-  - spacy==3.1.0
-  - python-dateutil==2.7.5
-  - python-levenshtein==0.12.2
-  - requests=2.25.1
-  - pip==20.3.3
+  - openjdk==8.0.152
+  - python-flair
+  - numpy
+  - pydantic<2
+  - python-levenshtein
+  - requests
+  - pip>=22
   - pip:
     - textsearch==0.0.21
-    - spacy==3.1.0
+    - osmnx
+    - langchain
+    - spacy>=3.5,<4
+    - spacy-transformers
+    - transformers<4.31
+    - sentence_transformers
+    - chromadb
+    - shapely
+    - ipywidgets
+    - nltk
+    - pystac_client