ci: updated pipeline to filter src, tests and tests_aws folders

Signed-off-by: Cagri Yonca <[email protected]>

.circleci/config.yml

@@ -1,375 +1,30 @@
 version: 2.1
 
-# More about orbs: https://circleci.com/docs/2.0/using-orbs/
-# orbs:
-#   ruby: circleci/[email protected]
+setup: true
 
-commands:
-  pip-install-deps:
-    parameters:
-      requirements:
-        default: "tests/requirements.txt"
-        type: string
-    steps:
-      - run:
-          name: Install Python Dependencies
-          command: |
-            python -m venv venv
-            . venv/bin/activate
-            pip install --upgrade pip
-            pip install 'wheel>=0.29.0'
-            pip install -r requirements.txt
-            pip install -r <<parameters.requirements>>
-
-  run-tests-with-coverage-report:
-    parameters:
-      cassandra:
-        default: ""
-        type: string
-      gevent:
-        default: ""
-        type: string
-      tests:
-        default: "tests"
-        type: string
-    steps:
-      - run:
-          name: Run Tests With Coverage Report
-          environment:
-            CASSANDRA_TEST: "<<parameters.cassandra>>"
-            GEVENT_STARLETTE_TEST: "<<parameters.gevent>>"
-          command: |
-            . venv/bin/activate
-            coverage run --source=instana -m pytest -v --junitxml=test-results <<parameters.tests>>
-            coverage report -m
-            coverage html
-            mkdir coverage_results
-            cp -R .coverage coverage_results/.coverage.${CIRCLE_BUILD_NUM}
-            cd coverage_results
-      - persist_to_workspace:
-          root: .
-          paths:
-            - coverage_results
-
-  store-pytest-results:
-    steps:
-      - store_test_results:
-          path: test-results
-
-  run_sonarqube:
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: Install Java
-          command: |
-            sudo apt-get update
-            sudo apt-get install openjdk-11-jdk
-      - run:
-          name: Run SonarQube to report the coverage
-          command: |
-            . venv/bin/activate
-            coverage combine ./coverage_results
-            coverage xml -i
-            wget -O /tmp/sonar-scanner-cli.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.8.1.3023.zip
-            unzip -d /tmp /tmp/sonar-scanner-cli.zip
-            if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then
-              /tmp/sonar-scanner-4.8.1.3023/bin/sonar-scanner \
-                -Dsonar.host.url=${SONARQUBE_URL} \
-                -Dsonar.login="${SONARQUBE_LOGIN}" \
-                -Dsonar.pullrequest.key="${CIRCLE_PR_NUMBER}" \
-                -Dsonar.pullrequest.branch="${CIRCLE_BRANCH}"
-            else
-              /tmp/sonar-scanner-4.8.1.3023/bin/sonar-scanner \
-                -Dsonar.host.url=${SONARQUBE_URL} \
-                -Dsonar.login="${SONARQUBE_LOGIN}" \
-                -Dsonar.branch.name="${CIRCLE_BRANCH}"
-            fi
-      - store_artifacts:
-          path: htmlcov
-
-  store-coverage-report:
-    steps:
-      - store_artifacts:
-          path: htmlcov
+orbs:
+  path-filtering: circleci/[email protected]
+  continuation: circleci/[email protected]
 
 jobs:
-  python38:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.8
-      - image: public.ecr.aws/docker/library/postgres:16.2-bookworm
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: passw0rd
-          POSTGRES_DB: instana_test_db
-      - image: public.ecr.aws/docker/library/mariadb:11.3.2
-        environment:
-          MYSQL_ROOT_PASSWORD: passw0rd
-          MYSQL_DATABASE: instana_test_db
-      - image: public.ecr.aws/docker/library/redis:7.2.4-bookworm
-      - image: public.ecr.aws/docker/library/rabbitmq:3.13.0
-      - image: public.ecr.aws/docker/library/mongo:7.0.6
-      - image: quay.io/thekevjames/gcloud-pubsub-emulator:latest
-        environment:
-          PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
-          PUBSUB_PROJECT1: test-project,test-topic
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps
-      - run-tests-with-coverage-report
-      - store-pytest-results
-      - store-coverage-report
-
-  python39:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.9
-      - image: public.ecr.aws/docker/library/postgres:16.2-bookworm
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: passw0rd
-          POSTGRES_DB: instana_test_db
-      - image: public.ecr.aws/docker/library/mariadb:11.3.2
-        environment:
-          MYSQL_ROOT_PASSWORD: passw0rd
-          MYSQL_DATABASE: instana_test_db
-      - image: public.ecr.aws/docker/library/redis:7.2.4-bookworm
-      - image: public.ecr.aws/docker/library/rabbitmq:3.13.0
-      - image: public.ecr.aws/docker/library/mongo:7.0.6
-      - image: quay.io/thekevjames/gcloud-pubsub-emulator:latest
-        environment:
-          PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
-          PUBSUB_PROJECT1: test-project,test-topic
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps
-      - run-tests-with-coverage-report
-      - store-pytest-results
-      - store-coverage-report
-
-  python310:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.10
-      - image: public.ecr.aws/docker/library/postgres:16.2-bookworm
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: passw0rd
-          POSTGRES_DB: instana_test_db
-      - image: public.ecr.aws/docker/library/mariadb:11.3.2
-        environment:
-          MYSQL_ROOT_PASSWORD: passw0rd
-          MYSQL_DATABASE: instana_test_db
-      - image: public.ecr.aws/docker/library/redis:7.2.4-bookworm
-      - image: public.ecr.aws/docker/library/rabbitmq:3.13.0
-      - image: public.ecr.aws/docker/library/mongo:7.0.6
-      - image: quay.io/thekevjames/gcloud-pubsub-emulator:latest
-        environment:
-          PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
-          PUBSUB_PROJECT1: test-project,test-topic
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements.txt"
-      - run-tests-with-coverage-report
-      - store-pytest-results
-      - store-coverage-report
-
-  python311:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.11
-      - image: public.ecr.aws/docker/library/postgres:16.2-bookworm
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: passw0rd
-          POSTGRES_DB: instana_test_db
-      - image: public.ecr.aws/docker/library/mariadb:11.3.2
-        environment:
-          MYSQL_ROOT_PASSWORD: passw0rd
-          MYSQL_DATABASE: instana_test_db
-      - image: public.ecr.aws/docker/library/redis:7.2.4-bookworm
-      - image: public.ecr.aws/docker/library/rabbitmq:3.13.0
-      - image: public.ecr.aws/docker/library/mongo:7.0.6
-      - image: quay.io/thekevjames/gcloud-pubsub-emulator:latest
-        environment:
-          PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
-          PUBSUB_PROJECT1: test-project,test-topic
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements.txt"
-      - run-tests-with-coverage-report
-      - store-pytest-results
-      - store-coverage-report
-
-  python312:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.12
-      - image: public.ecr.aws/docker/library/postgres:16.2-bookworm
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: passw0rd
-          POSTGRES_DB: instana_test_db
-      - image: public.ecr.aws/docker/library/mariadb:11.3.2
-        environment:
-          MYSQL_ROOT_PASSWORD: passw0rd
-          MYSQL_DATABASE: instana_test_db
-      - image: public.ecr.aws/docker/library/redis:7.2.4-bookworm
-      - image: public.ecr.aws/docker/library/rabbitmq:3.13.0
-      - image: public.ecr.aws/docker/library/mongo:7.0.6
-      - image: quay.io/thekevjames/gcloud-pubsub-emulator:latest
-        environment:
-          PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
-          PUBSUB_PROJECT1: test-project,test-topic
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements.txt"
-      - run-tests-with-coverage-report
-      - store-pytest-results
-      - store-coverage-report
-
-  py312aws:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.12
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements.txt"
-      - run-tests-with-coverage-report:
-          tests: "tests_aws"
-      - store-pytest-results
-      - store-coverage-report
-
-  python313:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.13
-      - image: public.ecr.aws/docker/library/postgres:16.2-bookworm
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: passw0rd
-          POSTGRES_DB: instana_test_db
-      - image: public.ecr.aws/docker/library/mariadb:11.3.2
-        environment:
-          MYSQL_ROOT_PASSWORD: passw0rd
-          MYSQL_DATABASE: instana_test_db
-      - image: public.ecr.aws/docker/library/redis:7.2.4-bookworm
-      - image: public.ecr.aws/docker/library/rabbitmq:3.13.0
-      - image: public.ecr.aws/docker/library/mongo:7.0.6
-      - image: quay.io/thekevjames/gcloud-pubsub-emulator:latest
-        environment:
-          PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
-          PUBSUB_PROJECT1: test-project,test-topic
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements.txt"
-      - run-tests-with-coverage-report
-      - store-pytest-results
-      - store-coverage-report
-
-  python314:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.14.0a2
-      - image: public.ecr.aws/docker/library/postgres:16.2-bookworm
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_PASSWORD: passw0rd
-          POSTGRES_DB: instana_test_db
-      - image: public.ecr.aws/docker/library/mariadb:11.3.2
-        environment:
-          MYSQL_ROOT_PASSWORD: passw0rd
-          MYSQL_DATABASE: instana_test_db
-      - image: public.ecr.aws/docker/library/redis:7.2.4-bookworm
-      - image: public.ecr.aws/docker/library/rabbitmq:3.13.0
-      - image: public.ecr.aws/docker/library/mongo:7.0.6
-      - image: quay.io/thekevjames/gcloud-pubsub-emulator:latest
-        environment:
-          PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
-          PUBSUB_PROJECT1: test-project,test-topic
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements-pre314.txt"
-      - run-tests-with-coverage-report
-      - store-pytest-results
-      - store-coverage-report
-
-  py39cassandra:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.9
-      - image: public.ecr.aws/docker/library/cassandra:3.11.16-jammy
-        environment:
-          MAX_HEAP_SIZE: 2048m
-          HEAP_NEWSIZE: 512m
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements-cassandra.txt"
-      - run-tests-with-coverage-report:
-          cassandra: "true"
-          tests: "tests/clients/test_cassandra-driver.py"
-      - store-pytest-results
-      - store-coverage-report
-
-  final_job:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.9
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-            requirements: "tests/requirements.txt"
-      - store-pytest-results
-      # - run_sonarqube
-
-  py39gevent_starlette:
-    docker:
-      - image: public.ecr.aws/docker/library/python:3.9
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - pip-install-deps:
-          requirements: "tests/requirements-gevent-starlette.txt"
-      - run-tests-with-coverage-report:
-          # TODO: uncomment once gevent instrumentation is done
-          # gevent: "true"
-          # tests: "tests/frameworks/test_gevent.py tests/frameworks/test_starlette.py"
-          tests: "tests/frameworks/test_starlette.py"
-      - store-pytest-results
-      - store-coverage-report
+  filter:
+    executor: path-filtering/default
+    resource_class: small
+    steps:
+      - checkout
+      - path-filtering/set-parameters:
+          base-revision: main
+          config-path: .circleci/continue_config.yml
+          output-path: /tmp/pipeline-parameters.json
+          mapping: |
+            src/.* run-pipeline-flag true
+            tests/.* run-pipeline-flag true
+            tests_aws/.* run-pipeline-flag true
+      - continuation/continue:
+          parameters: /tmp/pipeline-parameters.json
+          configuration_path: .circleci/continue_config.yml
 
 workflows:
-  version: 2
-  build:
+  setup:
     jobs:
-      - python38
-      - python39
-      - python310
-      - python311
-      - python312
-      - python313
-      - python314
-      - py39cassandra
-      - py39gevent_starlette
-      - py312aws
-      - final_job:
-          requires:
-            - python38
-            - python39
-            - python310
-            - python311
-            - python312
-            - python313
-            # Uncomment the following when giving real support to 3.14
-            # - python314
-            - py39cassandra
-            - py39gevent_starlette
-            - py312aws
+      - filter