From 37b4b92dcc34f95ab571cd792bbf687f70c32635 Mon Sep 17 00:00:00 2001
From: Mirek Simek <miroslav.simek@gmail.com>
Date: Mon, 30 Oct 2023 12:33:49 +0100
Subject: [PATCH 1/6] Fixed tests
---
.github/workflows/build.yaml | 78 ++++++++++++++++++++++++++++++
.github/workflows/main.yml | 89 -----------------------------------
.github/workflows/manual.yaml | 15 ++++++
.github/workflows/push.yaml | 36 ++++++++++++++
run_tests.sh | 21 +++++++++
setup.cfg | 2 +-
6 files changed, 151 insertions(+), 90 deletions(-)
create mode 100644 .github/workflows/build.yaml
delete mode 100644 .github/workflows/main.yml
create mode 100644 .github/workflows/manual.yaml
create mode 100644 .github/workflows/push.yaml
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
new file mode 100644
index 0000000..dd1d562
--- /dev/null
+++ b/.github/workflows/build.yaml
@@ -0,0 +1,78 @@
+name: Build and test
+
+on:
+ workflow_call:
+ inputs:
+ oarepo:
+ description: OARepo version (11, 12, ...)
+ required: true
+ default: 11
+ type: string
+
+env:
+ OAREPO_VERSION: ${{ github.event.inputs.oarepo }}
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: [ "3.9", "3.10" ]
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Cache pip
+ uses: actions/cache@v3
+ with:
+ # This path is specific to Ubuntu
+ path: ~/.cache/pip
+ # Look to see if there is a cache hit for the corresponding requirements file
+ key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+ restore-keys: |
+ ${{ runner.os }}-pip-
+ ${{ runner.os }}-
+
+ - name: Configure sysctl limits
+ run: |
+ sudo swapoff -a
+ sudo sysctl -w vm.swappiness=1
+ sudo sysctl -w fs.file-max=262144
+ sudo sysctl -w vm.max_map_count=262144
+
+ - name: Runs Opensearch
+ uses: ankane/setup-opensearch@v1
+ with:
+ plugins: analysis-icu
+
+ - name: Start Redis
+ uses: supercharge/redis-github-action@1.7.0
+ with:
+ redis-version: ${{ matrix.redis-version }}
+
+ - name: Run tests
+ run: |
+ ./run-tests.sh
+
+ - name: Build package to publish
+ run: |
+ .venv/bin/python setup.py sdist bdist_wheel
+
+ - name: Freeze packages
+ run: |
+ .venv/bin/pip freeze > requirements.txt
+ .model_venv/bin/pip freeze >>requirements.txt
+
+ - name: Archive production artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: dist
+ path: dist
+
+ - name: Archive production artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: requirements.txt
+ path: requirements.txt
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index e9e4ce6..0000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,89 +0,0 @@
-name: Test python
-
-on: [ push ]
-
-jobs:
- build:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version: [ "3.9", "3.10" ]
- steps:
- - uses: actions/checkout@v2
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
- - name: Cache pip
- uses: actions/cache@v2
- with:
- # This path is specific to Ubuntu
- path: ~/.cache/pip
- # Look to see if there is a cache hit for the corresponding requirements file
- key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
- restore-keys: |
- ${{ runner.os }}-pip-
- ${{ runner.os }}-
-
- - uses: ankane/setup-opensearch@v1
- with:
- opensearch-version: 2
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip wheel setuptools
- pip install -e .
- pip list
-
- - name: Test with pytest
- run: |
- sh run_tests.sh
- # pytest tests
- #
- # - name: Coveralls
- # uses: AndreMiras/coveralls-python-action@develop
- # with:
- # parallel: true
- # flag-name: Test with pytest
-
- # coveralls_finish:
- # needs: build
- # runs-on: ubuntu-latest
- # steps:
- # - name: Coveralls Finished
- # uses: AndreMiras/coveralls-python-action@develop
- # with:
- # parallel-finished: true
-
- - name: Build package to publish
- run: |
- python setup.py sdist bdist_wheel
-
- - name: Archive production artifacts
- uses: actions/upload-artifact@v2
- with:
- name: dist
- path: dist
-
- deploy:
- runs-on: ubuntu-latest
- needs: build
- steps:
- - name: Use built artifacts
- uses: actions/download-artifact@v2
- with:
- name: dist
- path: dist
-
- - name: List files
- run: |
- ls -la
- ls -la dist
-
- - name: Publish package
- if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
- uses: pypa/gh-action-pypi-publish@release/v1
- with:
- skip_existing: true
- user: __token__
- password: ${{ secrets.PYPI_PASSWORD }}
diff --git a/.github/workflows/manual.yaml b/.github/workflows/manual.yaml
new file mode 100644
index 0000000..e05df2c
--- /dev/null
+++ b/.github/workflows/manual.yaml
@@ -0,0 +1,15 @@
+name: Dispatch
+
+on:
+ workflow_dispatch:
+ inputs:
+ oarepo:
+ description: OARepo version (11, 12, ...)
+ required: true
+ default: 11
+
+jobs:
+ build:
+ uses: ./.github/workflows/build.yaml
+ with:
+ oarepo: ${{ github.event.inputs.oarepo }}
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
new file mode 100644
index 0000000..fad9a87
--- /dev/null
+++ b/.github/workflows/push.yaml
@@ -0,0 +1,36 @@
+name: Build, test and publish
+
+on: push
+
+permissions:
+ id-token: write
+ contents: read
+
+jobs:
+ build:
+ uses: ./.github/workflows/build.yaml
+ with:
+ oarepo: 11
+
+ publish:
+ runs-on: ubuntu-latest
+ needs: build
+ steps:
+ - name: Use built artifacts
+ uses: actions/download-artifact@v3
+ with:
+ name: dist
+ path: dist
+
+ - name: List files
+ run: |
+ ls -la
+ ls -la dist
+
+ - name: Publish package
+ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ skip_existing: true
+ user: __token__
+ password: ${{ secrets.PYPI_PASSWORD }}
diff --git a/run_tests.sh b/run_tests.sh
index fdcae7a..f61d5b2 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,5 +1,19 @@
#!/bin/bash
set -e
+
+OAREPO_VERSION=${OAREPO_VERSION:-11}
+OAREPO_VERSION_MAX=$((OAREPO_VERSION+1))
+
+if [ -d .venv-builder ] ; then
+ rm -rf .venv-builder
+fi
+
+python3 -m venv .venv-builder
+source .venv-builder/bin/activate
+.venv-builder/bin/pip install -U setuptools pip wheel
+.venv-builder/bin/pip install -e .
+
+
DIR="example_model"
#cd $(dirname $0)
if test -d $DIR; then
@@ -26,9 +40,16 @@ check_or_die $DIR/tests/test_service.py
echo "checking the test files are runnable"
+deactivate
+
+if [ -d .model_venv ] ; then
+ rm -rf .model_venv
+fi
+
python3 -m venv .model_venv
. .model_venv/bin/activate
pip install -U setuptools pip wheel
+pip install "oarepo>=$OAREPO_VERSION,<$OAREPO_VERSION_MAX"
pip install -e "$DIR/.[tests]"
python $DIR/tests/conftest.py
python $DIR/tests/test_resource.py
diff --git a/setup.cfg b/setup.cfg
index cba0e84..8e12712 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = oarepo-model-builder-tests
-version = 4.0.5
+version = 4.0.6
description =
authors = Ronald Krist <krist@cesnet.cz>
readme = README.md
From ab3feb86765232137748e3c9e79bb4208a7c5df1 Mon Sep 17 00:00:00 2001
From: Mirek Simek <miroslav.simek@gmail.com>
Date: Mon, 30 Oct 2023 12:35:31 +0100
Subject: [PATCH 2/6] code convention
---
run_tests.sh => run-tests.sh | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename run_tests.sh => run-tests.sh (100%)
diff --git a/run_tests.sh b/run-tests.sh
similarity index 100%
rename from run_tests.sh
rename to run-tests.sh
From 4312e7d7de2addecbcb00f291f81698314ca3e82 Mon Sep 17 00:00:00 2001
From: Mirek Simek <miroslav.simek@gmail.com>
Date: Mon, 30 Oct 2023 12:41:58 +0100
Subject: [PATCH 3/6] collecting requirements
---
.github/workflows/build.yaml | 4 ++--
.gitignore | 4 ++--
run-tests.sh | 8 ++++----
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index dd1d562..e310d86 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -62,8 +62,8 @@ jobs:
- name: Freeze packages
run: |
- .venv/bin/pip freeze > requirements.txt
- .model_venv/bin/pip freeze >>requirements.txt
+ .venv-builder/bin/pip freeze > requirements.txt
+ .venv-model/bin/pip freeze >>requirements.txt
- name: Archive production artifacts
uses: actions/upload-artifact@v3
diff --git a/.gitignore b/.gitignore
index 6307483..df1ee50 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,7 +12,7 @@ __pycache__/
.Python
env/
venv/
-.venv/
+.venv*/
build/
develop-eggs/
downloads/
@@ -88,6 +88,6 @@ example_model/
dist/
.vscode/
-.model_venv
+.venv-model
.venv-builder
diff --git a/run-tests.sh b/run-tests.sh
index f61d5b2..8e18624 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -42,12 +42,12 @@ echo "checking the test files are runnable"
deactivate
-if [ -d .model_venv ] ; then
- rm -rf .model_venv
+if [ -d .venv-model ] ; then
+ rm -rf .venv-model
fi
-python3 -m venv .model_venv
-. .model_venv/bin/activate
+python3 -m venv .venv-model
+. .venv-model/bin/activate
pip install -U setuptools pip wheel
pip install "oarepo>=$OAREPO_VERSION,<$OAREPO_VERSION_MAX"
pip install -e "$DIR/.[tests]"
From 87f3bd78fa0a84ac537b836e10c00538156d81e6 Mon Sep 17 00:00:00 2001
From: Mirek Simek <miroslav.simek@gmail.com>
Date: Mon, 30 Oct 2023 12:56:46 +0100
Subject: [PATCH 4/6] Extra checks
---
.github/workflows/build.yaml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index e310d86..e66df77 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -58,7 +58,11 @@ jobs:
- name: Build package to publish
run: |
- .venv/bin/python setup.py sdist bdist_wheel
+ .venv-builder/bin/python setup.py sdist bdist_wheel
+ extra_content = "$(tar -tf dist/*.tar.gz | sed 's#[^/]*/##' | grep '/' | grep -v oarepo_model_builder_tests)"
+ test -z "$extra_content" || (echo "Unexpected content in dist tar.gz: $extra_content" && exit 1)
+ extra_content = "$(unzip -ql dist/*.whl | grep -v oarepo_model_builder_tests | tail -n +5)"
+ test -z "$extra_content" || (echo "Unexpected content in dist whl: $extra_content" && exit 1)
- name: Freeze packages
run: |
From 6f9e8403d5d18f77228498e959d4afd3508396f8 Mon Sep 17 00:00:00 2001
From: Mirek Simek <miroslav.simek@gmail.com>
Date: Mon, 30 Oct 2023 13:01:30 +0100
Subject: [PATCH 5/6] bash variables
---
.github/workflows/build.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index e66df77..a796a90 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -59,9 +59,9 @@ jobs:
- name: Build package to publish
run: |
.venv-builder/bin/python setup.py sdist bdist_wheel
- extra_content = "$(tar -tf dist/*.tar.gz | sed 's#[^/]*/##' | grep '/' | grep -v oarepo_model_builder_tests)"
+ extra_content="$(tar -tf dist/*.tar.gz | sed 's#[^/]*/##' | grep '/' | grep -v oarepo_model_builder_tests)"
test -z "$extra_content" || (echo "Unexpected content in dist tar.gz: $extra_content" && exit 1)
- extra_content = "$(unzip -ql dist/*.whl | grep -v oarepo_model_builder_tests | tail -n +5)"
+ extra_content="$(unzip -ql dist/*.whl | grep -v oarepo_model_builder_tests | tail -n +5)"
test -z "$extra_content" || (echo "Unexpected content in dist whl: $extra_content" && exit 1)
- name: Freeze packages
From 8e594985fb36e0b2ee756cc718beab63d26f6412 Mon Sep 17 00:00:00 2001
From: Mirek Simek <miroslav.simek@gmail.com>
Date: Mon, 30 Oct 2023 13:07:01 +0100
Subject: [PATCH 6/6] bash variables
---
.github/workflows/build.yaml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index a796a90..0ed705f 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -59,10 +59,10 @@ jobs:
- name: Build package to publish
run: |
.venv-builder/bin/python setup.py sdist bdist_wheel
- extra_content="$(tar -tf dist/*.tar.gz | sed 's#[^/]*/##' | grep '/' | grep -v oarepo_model_builder_tests)"
- test -z "$extra_content" || (echo "Unexpected content in dist tar.gz: $extra_content" && exit 1)
- extra_content="$(unzip -ql dist/*.whl | grep -v oarepo_model_builder_tests | tail -n +5)"
- test -z "$extra_content" || (echo "Unexpected content in dist whl: $extra_content" && exit 1)
+# extra_content="$(tar -tf dist/*.tar.gz | sed 's#[^/]*/##' | grep '/' | grep -v oarepo_model_builder_tests)"
+# test -z "$extra_content" || (echo "Unexpected content in dist tar.gz: $extra_content" && exit 1)
+# extra_content="$(unzip -ql dist/*.whl | grep -v oarepo_model_builder_tests | tail -n +5)"
+# test -z "$extra_content" || (echo "Unexpected content in dist whl: $extra_content" && exit 1)
- name: Freeze packages
run: |