From 4265fab99723257efad8e4165b5fc641557af147 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Thu, 9 Jan 2025 15:03:05 -0800 Subject: [PATCH] [airbyte-ci/auto-merge] - Updates PR pipeline to bypass CI checks for promoted release candidate PRs (#49827) --- airbyte-ci/connectors/auto_merge/README.md | 3 ++ .../connectors/auto_merge/pyproject.toml | 2 +- .../auto_merge/src/auto_merge/consts.py | 1 + .../auto_merge/src/auto_merge/main.py | 27 ++++++++-- .../src/auto_merge/pr_validators.py | 25 ++++++++-- airbyte-ci/connectors/pipelines/README.md | 3 +- .../airbyte_ci/connectors/publish/pipeline.py | 5 +- .../pipelines/pipelines/helpers/utils.py | 4 +- airbyte-ci/connectors/pipelines/poetry.lock | 50 +++++++++++++------ .../connectors/pipelines/pyproject.toml | 7 +-- .../test_execution/test_run_steps.py | 10 +++- 11 files changed, 103 insertions(+), 34 deletions(-) diff --git a/airbyte-ci/connectors/auto_merge/README.md b/airbyte-ci/connectors/auto_merge/README.md index 90763cc08fd0..bbc93236cbbe 100644 --- a/airbyte-ci/connectors/auto_merge/README.md +++ b/airbyte-ci/connectors/auto_merge/README.md @@ -50,6 +50,9 @@ have been merged. ## Changelog +### 0.1.3 +Adds `auto-merge/bypass-ci-checks` label which does not require CI checks to pass to auto-merge PR. + ### 0.1.2 Set merge method to `squash`. diff --git a/airbyte-ci/connectors/auto_merge/pyproject.toml b/airbyte-ci/connectors/auto_merge/pyproject.toml index e73e19f4c3e2..347672ade245 100644 --- a/airbyte-ci/connectors/auto_merge/pyproject.toml +++ b/airbyte-ci/connectors/auto_merge/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "auto-merge" -version = "0.1.2" +version = "0.1.3" description = "" authors = ["Airbyte "] readme = "README.md" diff --git a/airbyte-ci/connectors/auto_merge/src/auto_merge/consts.py b/airbyte-ci/connectors/auto_merge/src/auto_merge/consts.py index 45d765a08cfc..dee7526b46cb 100644 --- a/airbyte-ci/connectors/auto_merge/src/auto_merge/consts.py +++ b/airbyte-ci/connectors/auto_merge/src/auto_merge/consts.py @@ -4,6 +4,7 @@ AIRBYTE_REPO = "airbytehq/airbyte" AUTO_MERGE_LABEL = "auto-merge" +AUTO_MERGE_BYPASS_CI_CHECKS_LABEL = "auto-merge/bypass-ci-checks" BASE_BRANCH = "master" CONNECTOR_PATH_PREFIXES = { "airbyte-integrations/connectors", diff --git a/airbyte-ci/connectors/auto_merge/src/auto_merge/main.py b/airbyte-ci/connectors/auto_merge/src/auto_merge/main.py index 5a489e3681b0..a82339018701 100644 --- a/airbyte-ci/connectors/auto_merge/src/auto_merge/main.py +++ b/airbyte-ci/connectors/auto_merge/src/auto_merge/main.py @@ -8,14 +8,14 @@ from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Callable, Optional from github import Auth, Github -from .consts import AIRBYTE_REPO, AUTO_MERGE_LABEL, BASE_BRANCH, MERGE_METHOD +from .consts import AIRBYTE_REPO, AUTO_MERGE_BYPASS_CI_CHECKS_LABEL, AUTO_MERGE_LABEL, BASE_BRANCH, MERGE_METHOD from .env import GITHUB_TOKEN, PRODUCTION from .helpers import generate_job_summary_as_markdown -from .pr_validators import ENABLED_VALIDATORS +from .pr_validators import VALIDATOR_MAPPING if TYPE_CHECKING: from github.Commit import Commit as GithubCommit @@ -49,7 +49,9 @@ def check_if_pr_is_auto_mergeable(head_commit: GithubCommit, pr: PullRequest, re Returns: bool: True if the PR is auto-mergeable, False otherwise """ - for validator in ENABLED_VALIDATORS: + + validators = get_pr_validators(pr) + for validator in validators: is_valid, error = validator(head_commit, pr, required_checks) if not is_valid: if error: @@ -58,6 +60,23 @@ def check_if_pr_is_auto_mergeable(head_commit: GithubCommit, pr: PullRequest, re return True +def get_pr_validators(pr: PullRequest) -> set[Callable]: + """ + Get the validator for a PR based on its labels + + Args: + pr (PullRequest): The PR to get the validator for + + Returns: + list[callable]: The validators + """ + + for label in pr.labels: + if label.name in VALIDATOR_MAPPING: + return VALIDATOR_MAPPING[label.name] + return VALIDATOR_MAPPING[AUTO_MERGE_LABEL] + + def merge_with_retries(pr: PullRequest, max_retries: int = 3, wait_time: int = 60) -> Optional[PullRequest]: """Merge a PR with retries diff --git a/airbyte-ci/connectors/auto_merge/src/auto_merge/pr_validators.py b/airbyte-ci/connectors/auto_merge/src/auto_merge/pr_validators.py index 5395d6d90a4c..025c4b7c8ba6 100644 --- a/airbyte-ci/connectors/auto_merge/src/auto_merge/pr_validators.py +++ b/airbyte-ci/connectors/auto_merge/src/auto_merge/pr_validators.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Callable, Optional, Tuple -from .consts import AUTO_MERGE_LABEL, BASE_BRANCH, CONNECTOR_PATH_PREFIXES +from .consts import AUTO_MERGE_BYPASS_CI_CHECKS_LABEL, AUTO_MERGE_LABEL, BASE_BRANCH, CONNECTOR_PATH_PREFIXES if TYPE_CHECKING: from github.Commit import Commit as GithubCommit @@ -18,6 +18,15 @@ def has_auto_merge_label(head_commit: GithubCommit, pr: PullRequest, required_ch return True, None +def has_auto_merge_bypass_ci_checks_label( + head_commit: GithubCommit, pr: PullRequest, required_checks: set[str] +) -> Tuple[bool, Optional[str]]: + has_auto_merge_bypass_ci_checks_label = any(label.name == AUTO_MERGE_BYPASS_CI_CHECKS_LABEL for label in pr.labels) + if not has_auto_merge_bypass_ci_checks_label: + return False, f"does not have the {AUTO_MERGE_BYPASS_CI_CHECKS_LABEL} label" + return True, None + + def targets_main_branch(head_commit: GithubCommit, pr: PullRequest, required_checks: set[str]) -> Tuple[bool, Optional[str]]: if not pr.base.ref == BASE_BRANCH: return False, f"does not target {BASE_BRANCH}" @@ -55,5 +64,13 @@ def head_commit_passes_all_required_checks( # - the head commit passes all required checks # PLEASE BE CAREFUL OF THE VALIDATOR ORDERING -# Let's declared faster checks first as the check_if_pr_is_auto_mergeable function fails fast. -ENABLED_VALIDATORS = [has_auto_merge_label, targets_main_branch, only_modifies_connectors, head_commit_passes_all_required_checks] +# Let's declare faster checks first as the check_if_pr_is_auto_mergeable function fails fast. +COMMON_VALIDATORS = { + targets_main_branch, + only_modifies_connectors, +} +# Let's declare faster checks first as the check_if_pr_is_auto_mergeable function fails fast. +VALIDATOR_MAPPING: dict[str, set[Callable]] = { + AUTO_MERGE_LABEL: COMMON_VALIDATORS | {has_auto_merge_label, head_commit_passes_all_required_checks}, + AUTO_MERGE_BYPASS_CI_CHECKS_LABEL: COMMON_VALIDATORS | {has_auto_merge_bypass_ci_checks_label}, +} diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 26212c25fc3b..77f8d9c7709c 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -853,7 +853,8 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only ## Changelog | Version | PR | Description | -|---------|------------------------------------------------------------| ---------------------------------------------------------------------------------------------------------------------------- | +| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.48.5 | [#49827](https://github.com/airbytehq/airbyte/pull/49827) | Bypasses CI checks for promoted release candidate PRs. | | 4.48.4 | [#51003](https://github.com/airbytehq/airbyte/pull/51003) | Install git in the build / test connector container when `--use-cdk-ref` is passed. | | 4.48.3 | [#50988](https://github.com/airbytehq/airbyte/pull/50988) | Remove deprecated `--no-update` flag from poetry commands | | 4.48.2 | [#50871](https://github.com/airbytehq/airbyte/pull/50871) | Speed up connector modification detection. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 4ced8a931b46..586b0e00a7a4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -13,6 +13,7 @@ import semver import yaml from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification # type: ignore +from auto_merge.consts import AUTO_MERGE_BYPASS_CI_CHECKS_LABEL # type: ignore from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage # type: ignore from dagger import Container, Directory, ExecError, File, ImageLayerCompression, Platform, QueryError from pydantic import BaseModel, ValidationError @@ -753,7 +754,9 @@ async def run_connector_promote_pipeline(context: PublishConnectorContext, semap all_modified_files.update( await add_changelog_entry.export_modified_files(context.connector.local_connector_documentation_directory) ) - post_changelog_pr_update = CreateOrUpdatePullRequest(context, skip_ci=False, labels=["auto-merge"]) + post_changelog_pr_update = CreateOrUpdatePullRequest( + context, skip_ci=True, labels=[AUTO_MERGE_BYPASS_CI_CHECKS_LABEL, "promoted-rc"] + ) pr_creation_args, pr_creation_kwargs = get_promotion_pr_creation_arguments( all_modified_files, context, results, current_version, promoted_version ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index 6160924f7a78..8e6f3eadf5e6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -19,10 +19,10 @@ from pathlib import Path from typing import TYPE_CHECKING -import anyio import asyncclick as click import asyncer from dagger import Client, Config, Container, Directory, ExecError, File, ImageLayerCompression, Platform, Secret +from exceptiongroup import ExceptionGroup from more_itertools import chunked if TYPE_CHECKING: @@ -115,7 +115,7 @@ async def get_file_contents(container: Container, path: str) -> Optional[str]: def catch_exec_error_group() -> Generator: try: yield - except anyio.ExceptionGroup as eg: + except ExceptionGroup as eg: for e in eg.exceptions: if isinstance(e, ExecError): raise e diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index 20f7205e16f3..a7a9b160e388 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1,8 +1,8 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "airbyte-connectors-base-images" -version = "1.3.0" +version = "1.5.0" description = "This package is used to generate and publish the base images for Airbyte Connectors." optional = false python-versions = "^3.10" @@ -61,24 +61,25 @@ files = [ [[package]] name = "anyio" -version = "3.7.1" +version = "4.8.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, - {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, + {file = "anyio-4.8.0-py3-none-any.whl", hash = "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a"}, + {file = "anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a"}, ] [package.dependencies] -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] -doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] -test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] -trio = ["trio (<0.22)"] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] +trio = ["trio (>=0.26.1)"] [[package]] name = "asyncclick" @@ -97,17 +98,17 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "asyncer" -version = "0.0.2" +version = "0.0.8" description = "Asyncer, async and await, focused on developer experience." optional = false -python-versions = ">=3.6.2,<4.0.0" +python-versions = ">=3.8" files = [ - {file = "asyncer-0.0.2-py3-none-any.whl", hash = "sha256:46e0e1423ce21588350ad425875e81795280b9e1f517e8a389de940b86c348bd"}, - {file = "asyncer-0.0.2.tar.gz", hash = "sha256:d546c85f3626ebbaf06bb4395db49761c902a61a6ac802b1a74133cab4f7f433"}, + {file = "asyncer-0.0.8-py3-none-any.whl", hash = "sha256:5920d48fc99c8f8f0f1576e1882f5022885589c5fcbc46ce4224ec3e53776eeb"}, + {file = "asyncer-0.0.8.tar.gz", hash = "sha256:a589d980f57e20efb07ed91d0dbe67f1d2fd343e7142c66d3a099f05c620739c"}, ] [package.dependencies] -anyio = ">=3.4.0,<4.0.0" +anyio = ">=3.4.0,<5.0" [[package]] name = "attrs" @@ -128,6 +129,23 @@ docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphi tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +[[package]] +name = "auto-merge" +version = "0.1.2" +description = "" +optional = false +python-versions = "^3.10" +files = [] +develop = true + +[package.dependencies] +anyio = "^4.3.0" +pygithub = "^2.3.0" + +[package.source] +type = "directory" +url = "../auto_merge" + [[package]] name = "backoff" version = "2.2.1" @@ -3223,4 +3241,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "57f62509d42f2ec8413b6f92531e9efaafd95ced1eb10d0331f79b9d3e368614" +content-hash = "94485e01e4d2d43cf9268c5a285997ab50de3e5c8c5128181fd13c59867b4942" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index fc5699aaaff2..d8571c69fd64 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.48.4" +version = "4.48.5" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] @@ -12,8 +12,8 @@ authors = ["Airbyte "] python = "~3.10" dagger-io = "==0.13.3" beartype = ">=0.18.2" -asyncer = "^0.0.2" -anyio = "^3.4.1" +asyncer = "^0" +anyio = "^4" more-itertools = "^8.11.0" docker = "^7" semver = "^3" @@ -40,6 +40,7 @@ pygithub = "^2.3.0" pydash = "6.0.2" python-slugify = ">=8.0.4" deepdiff = "^7.0.1" +auto-merge = { path = "../auto_merge", develop = true } [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2" diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py index 069702081d39..93cb3ace8ace 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py @@ -4,6 +4,7 @@ import anyio import pytest +from exceptiongroup import ExceptionGroup from pipelines.helpers.execution.run_steps import InvalidStepConfiguration, RunStepOptions, StepToRun, run_steps from pipelines.models.contexts.pipeline_context import PipelineContext @@ -353,17 +354,22 @@ async def test_run_steps_throws_on_invalid_args(invalid_args): [StepToRun(id="step1", step=TestStep(test_context), args=invalid_args)], ] - with pytest.raises(TypeError): + with pytest.raises(ExceptionGroup) as exc: await run_steps(steps) + assert len(exc.value.exceptions) == 1 + assert isinstance(exc.value.exceptions[0], TypeError) + @pytest.mark.anyio async def test_run_steps_with_params(): steps = [StepToRun(id="step1", step=TestStep(test_context))] options = RunStepOptions(fail_fast=True, step_params={"step1": {"--param1": ["value1"]}}) TestStep.accept_extra_params = False - with pytest.raises(ValueError): + with pytest.raises(ExceptionGroup) as exc: await run_steps(steps, options=options) + assert len(exc.value.exceptions) == 1 + assert isinstance(exc.value.exceptions[0], ValueError) assert steps[0].step.params_as_cli_options == [] TestStep.accept_extra_params = True await run_steps(steps, options=options)