-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
changes for self-hosted runner (#64)
* self-hosted changes * remove venv * remove venv * test main branch * reset workflow branch * add venv, change workflow branch * activate venv before pw:run * remove venv for test * add venv, add pip update * activate pip cache * add pip upgrade to venv * activate venv in cyspress tests * remove venv * test different activation of venv * test without venv * remove condition from setup python * remove venv * install addtional req * remove comp tests * clean up * clean up * clean up * remove "disable_cache" input * fix line change, add cypress_spec, remove pg_ready check * Rename deduplication-id to deduplication_id * use vdelev_selfhosted as main branch and add enable_cache flags * Revert back to main --------- Co-authored-by: Michael Pühringer <[email protected]> Co-authored-by: Michael Puehringer <[email protected]>
- Loading branch information
1 parent
77f57c2
commit 0b49a22
Showing
3 changed files
with
76 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,6 +15,10 @@ inputs: | |
description: "npm registry" | ||
default: "https://registry.npmjs.org/" | ||
required: true | ||
enable_cache: | ||
description: "enables the yarn cache download and upload" | ||
required: false | ||
default: true | ||
download_yarn_lock: | ||
description: "downloads the node-yarn-lock artifact (only available when build-node was called before)" | ||
required: false | ||
|
@@ -47,6 +51,10 @@ runs: | |
- name: Git config | ||
if: inputs.github_ro_token != '' | ||
run: | | ||
if [ -f ~/.gitconfig ]; then | ||
rm ~/.gitconfig | ||
touch ~/.gitconfig | ||
fi | ||
git config --global --replace-all url."https://[email protected]/".insteadOf ssh://[email protected]/ | ||
git config --add --global url."https://[email protected]/".insteadOf [email protected]: | ||
env: | ||
|
@@ -60,6 +68,7 @@ runs: | |
# Enable yarn download cache, @see https://github.com/actions/cache/tree/main/save#always-save-cache and https://github.com/actions/setup-node/issues/325 | ||
- name: Restore yarn cache | ||
uses: actions/cache/restore@v3 | ||
if: inputs.enable_cache == 'true' | ||
with: | ||
# This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder | ||
path: ~/.yarn/berry/cache/ | ||
|
@@ -72,7 +81,7 @@ runs: | |
shell: bash | ||
- name: Save yarn cache | ||
uses: actions/cache/save@v3 | ||
if: steps.install.outcome == 'success' | ||
if: steps.install.outcome == 'success' && inputs.enable_cache == 'true' | ||
with: | ||
# This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder | ||
path: ~/.yarn/berry/cache/ | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -66,6 +66,11 @@ on: | |
type: string | ||
required: false | ||
default: "ubuntu-22.04-large" | ||
deduplication_id: | ||
type: string | ||
required: false | ||
description: Unique id per workflow run. Must be set to unique value if dispatched multiple times for a single workflow. | ||
default: "" | ||
chromatic_enable: | ||
description: 'Enable Chromatic Tests' | ||
required: false | ||
|
@@ -92,6 +97,7 @@ env: | |
PYPI_USERNAME: "datavisyn" | ||
PYTHON_VERSION: "3.10" | ||
WORKFLOW_BRANCH: "main" | ||
POSTGRES_HOSTNAME: postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }} | ||
|
||
permissions: | ||
contents: write | ||
|
@@ -112,13 +118,13 @@ jobs: | |
runs-on: ${{ inputs.runs_on }} | ||
steps: | ||
- name: Checkout source repository | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
ref: ${{ inputs.branch }} | ||
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
fetch-depth: ${{ inputs.chromatic_enable && '0' || '1' }} | ||
- name: Checkout github-workflows | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: datavisyn/github-workflows | ||
ref: ${{ env.WORKFLOW_BRANCH }} | ||
|
@@ -130,6 +136,7 @@ jobs: | |
npm_registry: ${{ env.NPM_REGISTRY }} | ||
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
run_webpack: ${{ inputs.node_run_webpack }} | ||
enable_cache: ${{ inputs.runs_on != 'self-hosted' }} | ||
- name: Run Chromatic | ||
if: ${{ inputs.chromatic_enable }} | ||
uses: chromaui/[email protected] | ||
|
@@ -145,12 +152,12 @@ jobs: | |
runs-on: ${{ inputs.runs_on }} | ||
steps: | ||
- name: Checkout source repository | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
ref: ${{ inputs.branch }} | ||
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
- name: Checkout github-workflows | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: datavisyn/github-workflows | ||
ref: ${{ env.WORKFLOW_BRANCH }} | ||
|
@@ -160,6 +167,7 @@ jobs: | |
with: | ||
python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} | ||
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
enable_cache: ${{ inputs.runs_on != 'self-hosted' }} | ||
|
||
# If cypress is used, build node and python sequentially as it is avoiding the duplicate install overhead | ||
build-node-python-cypress: | ||
|
@@ -185,17 +193,34 @@ jobs: | |
--health-interval 10s | ||
--health-timeout 5s | ||
--health-retries 5 | ||
--name postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }} | ||
ports: | ||
- 5432:5432 | ||
# will assign a random free host port | ||
- 5432/tcp | ||
|
||
steps: | ||
- name: Set self-hosted env variable to github env | ||
run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV" | ||
- name: Set github token, hostname, port and docker network for self-hosted runner | ||
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != '' | ||
run: | | ||
{ | ||
echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}" | ||
echo "POSTGRES_PORT=5432" | ||
} >> "$GITHUB_ENV" | ||
docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} "postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}" | ||
- name: Set postgres connection details to hosted runner | ||
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == '' | ||
run: | | ||
echo "POSTGRES_HOSTNAME=localhost" >> "$GITHUB_ENV" | ||
echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" >> "$GITHUB_ENV" | ||
- name: Checkout source repository | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
ref: ${{ inputs.branch }} | ||
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
- name: Checkout github-workflows | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: datavisyn/github-workflows | ||
ref: ${{ env.WORKFLOW_BRANCH }} | ||
|
@@ -207,11 +232,13 @@ jobs: | |
npm_registry: ${{ env.NPM_REGISTRY }} | ||
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
run_webpack: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install | ||
enable_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} | ||
- name: Build python | ||
uses: ./tmp/github-workflows/.github/actions/build-python | ||
with: | ||
python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} | ||
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
enable_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} | ||
- name: Decrypt .env.enc and <app>/.env.enc | ||
run: | | ||
yarn run env:decrypt -pass env:ENV_PASSWORD || true | ||
|
@@ -233,8 +260,10 @@ jobs: | |
yarn start dev_server_only=true | ||
make start | ||
wait-on: "http://localhost:8080, http://localhost:9000/health" | ||
env: ${{ secrets.CYPRESS_ENV }} | ||
spec: ${{ inputs.cypress_spec_param }} | ||
env: | ||
POSTGRES_HOSTNAME: ${{ env.POSTGRES_HOSTNAME }} | ||
POSTGRES_PORT: ${{ env.POSTGRES_PORT }} | ||
# rspack currently does not support Cypress component tests | ||
# - name: Run cypress component tests | ||
# uses: cypress-io/github-action@v6 | ||
|
@@ -247,7 +276,7 @@ jobs: | |
# # TODO: Do component tests need the frontend/backend? | ||
# env: ${{ secrets.CYPRESS_ENV }} | ||
- name: Upload cypress screenshots | ||
uses: actions/upload-artifact@v4 | ||
uses: actions/upload-artifact@v3 | ||
if: failure() | ||
with: | ||
name: cypress-screenshots | ||
|
@@ -283,17 +312,34 @@ jobs: | |
--health-interval 10s | ||
--health-timeout 5s | ||
--health-retries 5 | ||
--name postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }} | ||
ports: | ||
- 5432:5432 | ||
# will assign a random free host port | ||
- 5432/tcp | ||
|
||
steps: | ||
- name: Set system env variable to github env | ||
run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV" | ||
- name: Set github token, hostname, port and docker network for self-hosted runner | ||
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != '' | ||
run: | | ||
{ | ||
echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}" | ||
echo "POSTGRES_PORT=5432" | ||
} >> "$GITHUB_ENV" | ||
docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }} | ||
- name: Set postgres connection details to hosted runner | ||
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == '' | ||
run: | | ||
echo "POSTGRES_HOSTNAME=localhost" >> "$GITHUB_ENV" | ||
echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" >> "$GITHUB_ENV" | ||
- name: Checkout source repository | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
ref: ${{ inputs.branch }} | ||
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
- name: Checkout github-workflows | ||
uses: actions/checkout@v3 | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: datavisyn/github-workflows | ||
ref: ${{ env.WORKFLOW_BRANCH }} | ||
|
@@ -305,11 +351,13 @@ jobs: | |
npm_registry: ${{ env.NPM_REGISTRY }} | ||
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
run_webpack: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install | ||
enable_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} | ||
- name: Build python | ||
uses: ./tmp/github-workflows/.github/actions/build-python | ||
with: | ||
python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} | ||
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
enable_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} | ||
- name: Decrypt .env.enc and <app>/.env.enc | ||
run: | | ||
yarn run env:decrypt -pass env:ENV_PASSWORD || true | ||
|