From 784bc7429ddf1fd0258a7447991de04fae5e34ce Mon Sep 17 00:00:00 2001 From: Oleg Pudeyev Date: Sun, 24 Nov 2024 14:13:13 -0500 Subject: [PATCH] Revert "is this really a no-op" This reverts commit 1d62ab07f433271140933bcfa5967465b07786d4. --- .github/workflows/system-tests.yml | 799 ++++++++++++++--------------- 1 file changed, 391 insertions(+), 408 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 26c2be66ae3..e5073a39daf 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -1,474 +1,457 @@ ---- name: System Tests -true: + +on: push: branches: - - "**" + - "**" workflow_dispatch: {} schedule: - - cron: 00 04 * * 2-6 + - cron: '00 04 * * 2-6' + env: REGISTRY: ghcr.io REPO: ghcr.io/datadog/dd-trace-rb - SYSTEM_TESTS_REF: ruby-di-enable + SYSTEM_TESTS_REF: ruby-di-enable # This must always be set to `main` on dd-trace-rb's master branch + jobs: build-harness: strategy: fail-fast: false matrix: image: - - name: runner - internal: system_tests/runner:latest - - name: agent - internal: system_tests/agent:latest - - name: proxy - internal: datadog/system-tests:proxy-v1 + - name: runner + internal: system_tests/runner:latest + - name: agent + internal: system_tests/agent:latest + - name: proxy + internal: datadog/system-tests:proxy-v1 runs-on: ubuntu-latest permissions: packages: write name: Build (${{ matrix.image.name }}) steps: - - name: Checkout - uses: actions/checkout@v4 - with: - repository: DataDog/system-tests - ref: "${{ env.SYSTEM_TESTS_REF }}" - persist-credentials: false - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: "${{ secrets.DOCKERHUB_USERNAME }}" - password: "${{ secrets.DOCKERHUB_TOKEN }}" - - name: Pull released image - run: | - if docker pull ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest; then - docker tag ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest ${{ matrix.image.internal }} - fi - - name: Build image - run: | - cache_from=() - for tag in latest; do - cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.image.name }}:${tag}") - done - echo "cache args: ${cache_from[*]}" - ./build.sh --images ${{ matrix.image.name }} --docker --extra-docker-args "${cache_from[*]}" - - name: List images - run: 'docker image list - - ' - - name: Log in to the Container registry - run: 'echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u - ${{ github.actor }} --password-stdin - - ' - - name: Tag image for CI run - run: docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ - matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} - - name: Push image for CI run - run: 'docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ - github.run_id }}-g${{ github.sha }} - - ' - - name: Tag image for commit - run: docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ - matrix.image.name }}:g${{ github.sha }} - - name: Push image for commit - run: 'docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ - github.sha }} + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'DataDog/system-tests' + ref: ${{ env.SYSTEM_TESTS_REF }} + persist-credentials: false + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Pull released image + run: | + if docker pull ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest; then + docker tag ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest ${{ matrix.image.internal }} + fi + - name: Build image + run: | + cache_from=() + for tag in latest; do + cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.image.name }}:${tag}") + done + echo "cache args: ${cache_from[*]}" + ./build.sh --images ${{ matrix.image.name }} --docker --extra-docker-args "${cache_from[*]}" + - name: List images + run: | + docker image list + - name: Log in to the Container registry + run: | + echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin + - name: Tag image for CI run + run: + docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} + - name: Push image for CI run + run: | + docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} + - name: Tag image for commit + run: + docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} + - name: Push image for commit + run: | + docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} + - name: Tag image for release + if: ${{ github.ref == 'refs/heads/master' }} + run: + docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest + - name: Push image for release + if: ${{ github.ref == 'refs/heads/master' }} + run: | + docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest - ' - - name: Tag image for release - if: "${{ github.ref == 'refs/heads/master' }}" - run: docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ - matrix.image.name }}:latest - - name: Push image for release - if: "${{ github.ref == 'refs/heads/master' }}" - run: 'docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest - - ' build-apps: strategy: fail-fast: false matrix: image: - - weblog + - weblog library: - - name: ruby - repository: DataDog/dd-trace-rb - path: dd-trace-rb + - name: ruby + repository: DataDog/dd-trace-rb + path: dd-trace-rb app: - - rack - - sinatra14 - - sinatra20 - - sinatra21 - - sinatra22 - - sinatra30 - - sinatra31 - - sinatra32 - - sinatra40 - - rails42 - - rails50 - - rails51 - - rails52 - - rails60 - - rails61 - - rails70 - - rails71 - - rails72 - - rails80 - - graphql23 + - rack + - sinatra14 + - sinatra20 + - sinatra21 + - sinatra22 + - sinatra30 + - sinatra31 + - sinatra32 + - sinatra40 + - rails42 + - rails50 + - rails51 + - rails52 + - rails60 + - rails61 + - rails70 + - rails71 + - rails72 + - rails80 + - graphql23 runs-on: ubuntu-latest name: Build (${{ matrix.app }}) permissions: packages: write outputs: - FORCED_TESTS_LIST: "${{steps.read_forced_tests_list.outputs.FORCED_TESTS_LIST}}" + FORCED_TESTS_LIST: ${{steps.read_forced_tests_list.outputs.FORCED_TESTS_LIST}} steps: - - name: Checkout - uses: actions/checkout@v4 - with: - repository: DataDog/system-tests - ref: "${{ env.SYSTEM_TESTS_REF }}" - persist-credentials: false - - name: Checkout ${{ matrix.library.repository }} - uses: actions/checkout@v4 - with: - repository: "${{ matrix.library.repository }}" - path: binaries/${{ matrix.library.path }} - fetch-depth: 2 - persist-credentials: false - - name: Read forced-tests-list.json file - id: read_forced_tests_list - run: | - echo "FORCED_TESTS_LIST<> $GITHUB_OUTPUT - echo "$(cat binaries/dd-trace-rb/.github/forced-tests-list.json)" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: "${{ secrets.DOCKERHUB_USERNAME }}" - password: "${{ secrets.DOCKERHUB_TOKEN }}" - - name: Pull released image - run: | - if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then - docker tag ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest - fi - parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" - for sha in $parents; do - docker pull "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}" || true - done - - name: Log in to the Container registry - run: 'echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u - ${{ github.actor }} --password-stdin - - ' - - name: Build - run: | - cache_from=() - for tag in latest; do - cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:${tag}") - done - parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" - for sha in $parents; do - cache_from+=(--cache-from ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}) - done - echo "cache args: ${cache_from[*]}" - ./build.sh --library ${{ matrix.library.name }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args "${cache_from[*]}" - - name: Tag image for CI run - run: docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ - matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id - }}-g${{ github.sha }} - - name: Push image for CI run - run: 'docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ - matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha - }} - - ' - - name: Tag image for commit - run: docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ - matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha - }} - - name: Push image for commit - run: 'docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ - matrix.image }}-${{ matrix.app }}:g${{ github.sha }} + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'DataDog/system-tests' + ref: ${{ env.SYSTEM_TESTS_REF }} + persist-credentials: false + - name: Checkout ${{ matrix.library.repository }} + uses: actions/checkout@v4 + with: + repository: '${{ matrix.library.repository }}' + path: 'binaries/${{ matrix.library.path }}' + fetch-depth: 2 + persist-credentials: false + - name: Read forced-tests-list.json file + id: read_forced_tests_list + run: | + echo "FORCED_TESTS_LIST<> $GITHUB_OUTPUT + echo "$(cat binaries/dd-trace-rb/.github/forced-tests-list.json)" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Pull released image + run: | + if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then + docker tag ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest + fi + parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" + for sha in $parents; do + docker pull "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}" || true + done + - name: Log in to the Container registry + run: | + echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin + - name: Build + run: | + cache_from=() + for tag in latest; do + cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:${tag}") + done + parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" + for sha in $parents; do + cache_from+=(--cache-from ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}) + done + echo "cache args: ${cache_from[*]}" + ./build.sh --library ${{ matrix.library.name }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args "${cache_from[*]}" + - name: Tag image for CI run + run: + docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} + - name: Push image for CI run + run: | + docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} + - name: Tag image for commit + run: + docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} + - name: Push image for commit + run: | + docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} + - name: Tag image for release + if: ${{ github.ref == 'refs/heads/master' }} + run: + docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest + - name: Push image for release + if: ${{ github.ref == 'refs/heads/master' }} + run: | + docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest - ' - - name: Tag image for release - if: "${{ github.ref == 'refs/heads/master' }}" - run: docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ - matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest - - name: Push image for release - if: "${{ github.ref == 'refs/heads/master' }}" - run: 'docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ - matrix.image }}-${{ matrix.app }}:latest - - ' test: strategy: fail-fast: false matrix: library: - - ruby + - ruby app: - - rack - - sinatra14 - - sinatra20 - - sinatra21 - - sinatra22 - - sinatra30 - - sinatra31 - - sinatra32 - - sinatra40 - - rails42 - - rails50 - - rails51 - - rails52 - - rails60 - - rails61 - - rails70 - - rails71 - - rails72 - - rails80 + - rack + - sinatra14 + - sinatra20 + - sinatra21 + - sinatra22 + - sinatra30 + - sinatra31 + - sinatra32 + - sinatra40 + - rails42 + - rails50 + - rails51 + - rails52 + - rails60 + - rails61 + - rails70 + - rails71 + - rails72 + - rails80 scenario: - - DEFAULT - - APPSEC_DISABLED - - APPSEC_BLOCKING_FULL_DENYLIST - - APPSEC_REQUEST_BLOCKING - - APPSEC_STANDALONE + - DEFAULT + - APPSEC_DISABLED + - APPSEC_BLOCKING_FULL_DENYLIST + - APPSEC_REQUEST_BLOCKING + - APPSEC_STANDALONE include: - - library: ruby - app: rack - scenario: INTEGRATIONS - - library: ruby - app: rack - scenario: CROSSED_TRACING_LIBRARIES - - library: ruby - app: rack - scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD - - library: ruby - app: rack - scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES - - library: ruby - app: rack - scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE - - library: ruby - app: rack - scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE - - library: ruby - app: rack - scenario: APPSEC_CUSTOM_RULES - - library: ruby - app: rack - scenario: APPSEC_MISSING_RULES - - library: ruby - app: rack - scenario: APPSEC_CORRUPTED_RULES - - library: ruby - app: rack - scenario: APPSEC_LOW_WAF_TIMEOUT - - library: ruby - app: rack - scenario: APPSEC_CUSTOM_OBFUSCATION - - library: ruby - app: rack - scenario: APPSEC_RATE_LIMITER - - library: ruby - app: rails70 - scenario: APPSEC_AUTO_EVENTS_EXTENDED - - library: ruby - app: rails70 - scenario: APPSEC_API_SECURITY - - library: ruby - app: rack - scenario: APPSEC_RULES_MONITORING_WITH_ERRORS - - library: ruby - app: rails70 - scenario: DEBUGGER_PROBES_STATUS - - library: ruby - app: rails70 - scenario: DEBUGGER_METHOD_PROBES_SNAPSHOT - - library: ruby - app: rails70 - scenario: DEBUGGER_LINE_PROBES_SNAPSHOT - - library: ruby - app: rails70 - scenario: DEBUGGER_MIX_LOG_PROBE - - library: ruby - app: rails70 - scenario: DEBUGGER_PII_REDACTION - - library: ruby - app: rack - scenario: SAMPLING - - library: ruby - app: rack - scenario: PROFILING - - library: ruby - app: rack - scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED - - library: ruby - app: rack - scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED - - library: ruby - app: rack - scenario: TELEMETRY_LOG_GENERATION_DISABLED - - library: ruby - app: rack - scenario: TELEMETRY_METRIC_GENERATION_DISABLED - - library: ruby - app: graphql23 - scenario: GRAPHQL_APPSEC + - library: ruby + app: rack + scenario: INTEGRATIONS + - library: ruby + app: rack + scenario: CROSSED_TRACING_LIBRARIES + - library: ruby + app: rack + scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD + - library: ruby + app: rack + scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES + - library: ruby + app: rack + scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE + - library: ruby + app: rack + scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE + - library: ruby + app: rack + scenario: APPSEC_CUSTOM_RULES + - library: ruby + app: rack + scenario: APPSEC_MISSING_RULES + - library: ruby + app: rack + scenario: APPSEC_CORRUPTED_RULES + - library: ruby + app: rack + scenario: APPSEC_LOW_WAF_TIMEOUT + - library: ruby + app: rack + scenario: APPSEC_CUSTOM_OBFUSCATION + - library: ruby + app: rack + scenario: APPSEC_RATE_LIMITER + - library: ruby + app: rails70 + scenario: APPSEC_AUTO_EVENTS_EXTENDED + - library: ruby + app: rails70 + scenario: APPSEC_API_SECURITY + - library: ruby + app: rack + scenario: APPSEC_RULES_MONITORING_WITH_ERRORS + - library: ruby + app: rails70 + scenario: DEBUGGER_PROBES_STATUS + - library: ruby + app: rails70 + scenario: DEBUGGER_METHOD_PROBES_SNAPSHOT + - library: ruby + app: rails70 + scenario: DEBUGGER_LINE_PROBES_SNAPSHOT + - library: ruby + app: rails70 + scenario: DEBUGGER_MIX_LOG_PROBE + - library: ruby + app: rails70 + scenario: DEBUGGER_PII_REDACTION + - library: ruby + app: rack + scenario: SAMPLING + - library: ruby + app: rack + scenario: PROFILING + - library: ruby + app: rack + scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED + - library: ruby + app: rack + scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED + - library: ruby + app: rack + scenario: TELEMETRY_LOG_GENERATION_DISABLED + - library: ruby + app: rack + scenario: TELEMETRY_METRIC_GENERATION_DISABLED + - library: ruby + app: graphql23 + scenario: GRAPHQL_APPSEC runs-on: ubuntu-latest needs: - - build-harness - - build-apps + - build-harness + - build-apps name: Test (${{ matrix.app }}, ${{ matrix.scenario }}) steps: - - name: Checkout - uses: actions/checkout@v4 - with: - repository: DataDog/system-tests - ref: "${{ env.SYSTEM_TESTS_REF }}" - persist-credentials: false - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: "${{ secrets.DOCKERHUB_USERNAME }}" - password: "${{ secrets.DOCKERHUB_TOKEN }}" - - name: Pull runner image - run: | - docker pull ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} - docker tag ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest - - name: Pull proxy image - run: | - docker pull ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} - docker tag ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} datadog/system-tests:proxy-v1 - - name: Pull agent image - run: | - docker pull ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} - docker tag ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest - - name: Pull app image - run: | - docker pull ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} - docker tag ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest - - name: List images - run: 'docker image list + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'DataDog/system-tests' + ref: ${{ env.SYSTEM_TESTS_REF }} + persist-credentials: false + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Pull runner image + run: | + docker pull ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} + docker tag ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest + - name: Pull proxy image + run: | + docker pull ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} + docker tag ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} datadog/system-tests:proxy-v1 + - name: Pull agent image + run: | + docker pull ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} + docker tag ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest + - name: Pull app image + run: | + docker pull ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} + docker tag ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest + - name: List images + run: | + docker image list + - name: Run scenario + # The last escape sequence check if the JSON string exists, + # then check if the current scenario is contained in it, then create a string by chaining -F and forced tests. + # A potential error might come from an empty array in a scenario in the json file. + run: | + ./run.sh ++docker ${{ matrix.scenario }} \ + ${{ env.FORCED_TESTS_LIST && fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario] && format('-F {0}', join(fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario], ' -F ')) || ''}} + env: + DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }} + SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID }} + SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY }} + FORCED_TESTS_LIST: ${{ needs.build-apps.outputs.FORCED_TESTS_LIST }} + - name: Archive logs + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }} + path: logs* - ' - - name: Run scenario - run: | - ./run.sh ++docker ${{ matrix.scenario }} \ - ${{ env.FORCED_TESTS_LIST && fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario] && format('-F {0}', join(fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario], ' -F ')) || ''}} - env: - DD_API_KEY: "${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}" - SYSTEM_TESTS_AWS_ACCESS_KEY_ID: "${{ secrets.SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID - }}" - SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: "${{ secrets.SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY - }}" - FORCED_TESTS_LIST: "${{ needs.build-apps.outputs.FORCED_TESTS_LIST }}" - - name: Archive logs - uses: actions/upload-artifact@v4 - if: "${{ always() }}" - with: - name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario - }}-logs-gha${{ github.run_id }}-g${{ github.sha }} - path: logs* aggregate: strategy: fail-fast: false matrix: library: - - ruby + - ruby app: - - rack - - sinatra14 - - sinatra20 - - sinatra21 - - sinatra22 - - sinatra30 - - sinatra31 - - sinatra32 - - sinatra40 - - rails42 - - rails50 - - rails51 - - rails52 - - rails60 - - rails61 - - rails70 - - rails71 - - rails72 - - rails80 - - graphql23 + - rack + - sinatra14 + - sinatra20 + - sinatra21 + - sinatra22 + - sinatra30 + - sinatra31 + - sinatra32 + - sinatra40 + - rails42 + - rails50 + - rails51 + - rails52 + - rails60 + - rails61 + - rails70 + - rails71 + - rails72 + - rails80 + - graphql23 runs-on: ubuntu-latest needs: - - build-harness - - build-apps - - test - if: "${{ always() }}" + - build-harness + - build-apps + - test + if: ${{ always() }} name: Aggregate (${{ matrix.app }}) steps: - - name: Setup python 3.12 - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Checkout - uses: actions/checkout@v4 - with: - repository: DataDog/system-tests - ref: "${{ env.SYSTEM_TESTS_REF }}" - persist-credentials: false - - name: Retrieve logs - uses: actions/download-artifact@v4 - with: - pattern: system-tests-${{ matrix.library }}-${{ matrix.app }}-*-logs-gha${{ - github.run_id }}-g${{ github.sha }} - merge-multiple: true - path: "." - - name: Print fancy log report - run: | - find logs* - python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY + - name: Setup python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'DataDog/system-tests' + ref: ${{ env.SYSTEM_TESTS_REF }} + persist-credentials: false + - name: Retrieve logs + uses: actions/download-artifact@v4 + with: + pattern: system-tests-${{ matrix.library }}-${{ matrix.app }}-*-logs-gha${{ github.run_id }}-g${{ github.sha }} + merge-multiple: true + path: . + - name: Print fancy log report + run: | + find logs* + python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY + cleanup: strategy: fail-fast: false matrix: image: - - runner - - agent - - weblog-rack - - weblog-sinatra14 - - weblog-sinatra20 - - weblog-sinatra21 - - weblog-sinatra22 - - weblog-sinatra30 - - weblog-sinatra31 - - weblog-sinatra32 - - weblog-sinatra40 - - weblog-rails42 - - weblog-rails50 - - weblog-rails51 - - weblog-rails52 - - weblog-rails60 - - weblog-rails61 - - weblog-rails70 - - weblog-rails71 - - weblog-rails72 - - weblog-rails80 - - weblog-graphql23 + - runner + - agent + - weblog-rack + - weblog-sinatra14 + - weblog-sinatra20 + - weblog-sinatra21 + - weblog-sinatra22 + - weblog-sinatra30 + - weblog-sinatra31 + - weblog-sinatra32 + - weblog-sinatra40 + - weblog-rails42 + - weblog-rails50 + - weblog-rails51 + - weblog-rails52 + - weblog-rails60 + - weblog-rails61 + - weblog-rails70 + - weblog-rails71 + - weblog-rails72 + - weblog-rails80 + - weblog-graphql23 runs-on: ubuntu-latest needs: - - test - if: "${{ always() }}" + - test + if: ${{ always() }} name: Cleanup (${{ matrix.image }}) steps: - - name: Log in to the Container registry - run: 'echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u - ${{ github.actor }} --password-stdin - - ' - - uses: actions/delete-package-versions@v5 - with: - package-version-ids: gha${{ github.run_id }}-g${{ github.sha }} - package-name: system-tests/${{ matrix.image }} - package-type: container - continue-on-error: true + - name: Log in to the Container registry + run: | + echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin + - uses: actions/delete-package-versions@v5 + with: + package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}' + package-name: 'system-tests/${{ matrix.image }}' + package-type: 'container' + continue-on-error: true