Taskfile.yaml

version: '3'

vars:
  VERSION:
    sh: cat version
  PROJECT_ROOT:
    sh: pwd
  COMMIT:
    sh: git rev-parse HEAD
  GITHUB_URL: https://github.com/alexklibisz/elastiknn

tasks:

  annbClean:
    desc: Delete artifacts
    dir: ann-benchmarks
    cmds:
      - rm -rf venv

  annbCreateSubmodule:
    desc: Setup the ann-benchmarks submodule
    dir: ann-benchmarks
    status:
      - ls ann-benchmarks/.git
    cmds:
      - git submodule update --init ann-benchmarks

  annbInstallRequirements:
    desc: Install dependencies
    preconditions:
      - python --version | grep "3.10.8"
    dir: ann-benchmarks/ann-benchmarks
    deps:
      - annbCreateSubmodule
    env:
      PYTHONPATH: "{{ .PROJECT_ROOT }}/client-python"
    cmds:
      - python --version
      - python -m pip install --quiet virtualenv
      - python -m virtualenv venv
      - venv/bin/pip install --quiet -r requirements.txt
      - venv/bin/pip install --quiet -r {{ .PROJECT_ROOT }}/client-python/requirements.txt
      - venv/bin/python -c "import importlib; importlib.import_module('ann_benchmarks.algorithms.elastiknn'); print('ok')"

  annbRunLocalFashionMnist:
    desc: Run ann-benchmarks using the local Elastiknn branch.
    dir: ann-benchmarks/ann-benchmarks
    deps:
      - annbInstallRequirements
    env:
      PYTHONPATH: "{{ .PROJECT_ROOT }}/client-python"
      OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES"
      RESULTS_DIR: ../../docs/pages/performance/fashion-mnist
    cmds:
      - cp ../config.yml ann_benchmarks/algorithms/elastiknn/config.yml
      - venv/bin/python run.py --dataset fashion-mnist-784-euclidean --algorithm elastiknn-l2lsh --run-disabled --runs 3 --count 100 --parallelism 1 --force --local
      - mkdir -p $RESULTS_DIR
      - venv/bin/python plot.py --dataset fashion-mnist-784-euclidean --count 100 --output $RESULTS_DIR/plot.png | venv/bin/python ../parse_results.py > $RESULTS_DIR/results.md
      - base64 -w 0 -i $RESULTS_DIR/plot.png > $RESULTS_DIR/plot.b64
      - cat $RESULTS_DIR/results.md

  annbRunOfficialFashionMnist:
    desc: Run ann-benchmarks using released Elastiknn on the SIFT dataset.
    dir: ann-benchmarks/ann-benchmarks
    deps:
      - annbInstallRequirements
    env:
      RESULTS_DIR: ../../docs/pages/performance/fashion-mnist
    cmds:
      - venv/bin/python install.py --algorithm elastiknn
      - venv/bin/python run.py --dataset fashion-mnist-784-euclidean --algorithm elastiknn-l2lsh --run-disabled --runs 3 --count 100 --parallelism 3 --force
      - mkdir -p $RESULTS_DIR
      - venv/bin/python plot.py --dataset fashion-mnist-784-euclidean --count 100 --output $RESULTS_DIR/plot.png | venv/bin/python ../parse_results.py > $RESULTS_DIR/results.md
      - base64 -i $RESULTS_DIR/plot.png > $RESULTS_DIR/plot.b64
      - cat $RESULTS_DIR/results.md

  annbTest:
    desc: Run ann-benchmarks using the Elastiknn algo on random data.
    dir: ann-benchmarks/ann-benchmarks
    deps:
      - annbInstallRequirements
    env:
      PYTHONPATH: "{{ .PROJECT_ROOT }}/client-python"
      OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES"
    cmds:
      - rm -rf results/random-xs-20-euclidean
      - venv/bin/python run.py --algorithm elastiknn-exact --dataset random-xs-20-euclidean --run-disabled --timeout 30 --local --force --runs 1
      - ls results/random-xs-20-euclidean/10/elastiknn-exact
      - venv/bin/python run.py --algorithm elastiknn-l2lsh --dataset random-xs-20-euclidean --run-disabled --timeout 30 --local --force --runs 1
      - ls results/random-xs-20-euclidean/10/elastiknn-l2lsh
      - venv/bin/python plot.py --dataset random-xs-20-euclidean --count 10 --output plot.png

  clean:
    desc: Call clean commands in included taskfiles
    deps: 
      - jvmClean
      - pyClean
      - annbClean

  dockerLogBenchmarkingCluster:
    desc: Print logs from the benchmarking cluster
    cmds:
      - docker compose -f docker/docker-compose.benchmarking.yaml logs
      - docker ps -a

  dockerLogTestingCluster:
    desc: Print logs from the testing cluster
    cmds:
      - docker compose -f docker/docker-compose.testing.yaml logs
      - docker ps -a

  dockerRunBenchmarkingCluster:
    desc: Run a local cluster for benchmarking using docker compose.
    deps:
      - installDockerCompose
      - jvmAssemble
    cmds:
      - docker compose -f docker/docker-compose.benchmarking.yaml up --detach --build --force-recreate
      - python3 docker/cluster_ready.py

  dockerRunTestingCluster:
    desc: Run a local cluster for testing using docker compose.
    deps:
      - installDockerCompose
      - jvmAssemble
    cmds:
      - docker compose -f docker/docker-compose.testing.yaml up --detach --build --force-recreate
      - python3 docker/cluster_ready.py

  dockerStopBenchmarkingCluster:
    desc: Stop the local benchmarking cluster.
    cmds:
      - docker compose -f docker/docker-compose.benchmarking.yaml down

  dockerStopTestingCluster:
    desc: Stop the local testing cluster.
    cmds:
      - docker compose -f docker/docker-compose.testing.yaml down

  docsDev:
    desc: Start local development server.
    dir: docs
    cmds:
      - docker compose up dev

  docsCompile:
    desc: Compile docs into a static site.
    dir: docs
    cmds:
      - rm -rf _site
      - docker compose run --rm compile

  jvmClean:
    desc: Clean all JVM artifacts
    cmds:
      - find . -name target -type d | xargs rm -rf

  jvmCompile:
    desc: Compile Scala and Java code using SBT
    cmds:
      - sbt -client "compile; Test/compile"

  jvmLint:
    desc: Lint the JVM code
    cmds:
      - sbt -client scalafmtCheckAll

  jvmAssemble:
    desc: Build the plugin bundle using SBT
    cmds:
      - rm -rf elastiknn-plugin/target/*.zip
      - sbt -client elastiknn-plugin/elasticsearchPluginBundle

  jvmIntegrationTest:
    desc: Run JVM tests using SBT
    cmds:
      - sbt -client elastiknn-plugin-integration-tests/test

  jvmIntegrationTestQuick:
    desc: Run JVM tests using SBT, but only the ones that were recompiled
    cmds:
      - sbt -client elastiknn-plugin-integration-tests/testQuick

  jvmUnitTest:
    desc: Run JVM tests using SBT
    cmds:
      - sbt -client elastiknn-api4s/test
      - sbt -client elastiknn-client-elastic4s/test
      - sbt -client elastiknn-jmh-benchmarks/test
      - sbt -client elastiknn-lucene/test
      - sbt -client elastiknn-models/test
      - sbt -client elastiknn-plugin/test

  jvmUnitTestQuick:
    desc: Run JVM tests using SBT, but only the ones that were recompiled
    cmds:
      - sbt -client testQuick

  jvmRunLocal:
    desc: Run a local Elasticsearch cluster with the plugin using SBT
    env:
      ES_JAVA_OPTS: "-Xms4g -Xmx4g"
    cmds:
      - sbt -client elastiknn-plugin/elasticsearchPluginRun

  jvmRunLocalDebug:
    desc: Run a local Elasticsearch cluster in debug mode with the plugin using SBT
    env:
      ES_JAVA_OPTS: "-Xms4g -Xmx4g"
    cmds:
      - sbt -client elastiknn-plugin/elasticsearchPluginDebug

  jvmPublishRelease:
    desc: Publish an official release to Github
    deps:
      - jvmAssemble
    status:
      - curl -f -s -o /dev/null {{ .GITHUB_URL }}/releases/tag/{{ .VERSION }}
    cmds:
      - gh release create --generate-notes {{ .VERSION }} --target {{ .COMMIT }} elastiknn-plugin/target/elastiknn-*.zip

  jvmPublishSnapshot:
    desc: Publish a snapshot of the plugin to Github
    deps:
      - jvmAssemble
    cmds:
      - gh release delete {{ .VERSION }} --yes || true
      - gh release create --prerelease --generate-notes {{ .VERSION }} --target {{ .COMMIT }} elastiknn-plugin/target/elastiknn-*.zip

  installDockerCompose:
    desc: Install docker compose using pip
    status:
      - docker compose --version

  pyCreateVenv:
    desc: Create python virtual environment
    dir: client-python
    preconditions:
      - python --version | grep 3.10.8
    cmds:
      - python --version
      - python -m pip install --quiet virtualenv
      - python -m virtualenv venv
      - touch requirements.txt
    generates:
      - venv
    status:
      - test -d venv

  pyClean:
    desc: Delete python virtual environment
    dir: client-python
    cmds:
      - rm -rf venv

  pyInstallRequirements:
    desc: Install python dependencies from requirements.txt
    dir: client-python
    deps:
      - pyCreateVenv
    sources:
      - venv/bin/*
      - setup.py
      - requirements*txt
    cmds:
      - venv/bin/pip install --quiet -r requirements.txt
      - venv/bin/pip install --quiet -r requirements-build.txt

  pyTest:
    desc: Run python tests against local cluster
    dir: client-python
    deps:
      - pyInstallRequirements
    sources:
      - elastiknn/**/*.py
      - setup.py
    cmds:
      - venv/bin/python -m pytest -v -s -o log_cli=True

  pyDocs:
    desc: Generate python docs
    dir: client-python
    deps:
      - pyInstallRequirements
    sources:
      - elastiknn/**/*.py
      - setup.py
    cmds:
      - rm -rf pdoc
      - venv/bin/pdoc3 --html elastiknn -c show_type_annotations=True -o pdoc
      - ls -la pdoc/elastiknn

  pyPublishLocal:
    desc: Publish python library locally
    dir: client-python
    deps:
      - pyInstallRequirements
    cmds:
      - rm -rf dist
      - venv/bin/python setup.py --version {{ .VERSION }} sdist bdist_wheel
      - ls dist

  pyPublishSnapshot:
    desc: Publish python library to testpypi
    dir: client-python
    cmds:
      - task: pyPublishLocal
        force: True
      - venv/bin/python -m twine upload -r testpypi --verbose dist/*

  pyPublishRelease:
    desc: Publish python library to pypi
    dir: client-python
    status:
      - curl -f -s -o /dev/null https://pypi.org/project/elastiknn-client/{{ .VERSION }}/
    cmds:
      - task: pyPublishLocal
        force: True
      - task: pyDocs
        force: True
      - venv/bin/python -m twine upload -r pypi --verbose dist/*