From c050055c6f95f532423f59d1ce28baa72418dc2e Mon Sep 17 00:00:00 2001 From: sivhari Date: Fri, 10 Jul 2026 12:29:11 +0530 Subject: [PATCH] fix(executors): discover connector console script from entry points VenvExecutor no longer assumes the installed CLI name matches the connector name. When they differ, resolve the executable via console_scripts metadata instead of triggering a useless reinstall loop. Closes #290 Co-authored-by: Cursor --- airbyte/_executors/python.py | 86 ++++++++++++++++--- scripts/reproduce_issue_290.py | 51 +++++++++++ .../fixtures/source-wrong-exe/setup.py | 21 +++++ .../source_wrong_exe/__init__.py | 1 + .../source-wrong-exe/source_wrong_exe/run.py | 24 ++++++ .../test_issue_290_wrong_executable.py | 35 ++++++++ 6 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 scripts/reproduce_issue_290.py create mode 100644 tests/integration_tests/fixtures/source-wrong-exe/setup.py create mode 100644 tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/__init__.py create mode 100644 tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/run.py create mode 100644 tests/unit_tests/test_issue_290_wrong_executable.py diff --git a/airbyte/_executors/python.py b/airbyte/_executors/python.py index 57a5e0094..d9a9add58 100644 --- a/airbyte/_executors/python.py +++ b/airbyte/_executors/python.py @@ -69,6 +69,7 @@ def __init__( with suppress(Exception): self.install_root.mkdir(parents=True, exist_ok=True) self.use_python = use_python + self._console_script_name: str | None = None def _get_venv_name(self) -> str: return f".venv-{self.name}" @@ -76,9 +77,73 @@ def _get_venv_name(self) -> str: def _get_venv_path(self) -> Path: return self.install_root / self._get_venv_name() + def _get_pypi_package_name(self) -> str: + if self.metadata and self.metadata.pypi_package_name: + return self.metadata.pypi_package_name + return f"airbyte-{self.name}" + + def _discover_console_script_name(self) -> str | None: + """Return the installed package's console script name, if discoverable.""" + if not self.interpreter_path.exists(): + return None + + package_name = self._get_pypi_package_name() + connector_name = self.name + discovery_script = f""" +import importlib.metadata as metadata + +package_name = {package_name!r} +connector_name = {connector_name!r} +entry_points = [ + ep + for ep in metadata.entry_points(group="console_scripts") + if ep.dist.name == package_name +] +if connector_name in {{ep.name for ep in entry_points}}: + print(connector_name) +elif len(entry_points) == 1: + print(entry_points[0].name) +elif entry_points: + print(entry_points[0].name) +else: + print("") +""".strip() + try: + result = subprocess.check_output( + [str(self.interpreter_path), "-c", discovery_script], + universal_newlines=True, + stderr=subprocess.PIPE, + ).strip() + except Exception: + return None + + return result or None + + def _resolve_console_script_name(self) -> str | None: + """Resolve the connector CLI executable name within the virtual environment.""" + if self._console_script_name: + return self._console_script_name + + suffix: Literal[".exe", ""] = ".exe" if is_windows() else "" + default_name = self.name + suffix + default_path = get_bin_dir(self._get_venv_path()) / default_name + if default_path.exists(): + self._console_script_name = self.name + return self._console_script_name + + discovered_name = self._discover_console_script_name() + if discovered_name: + discovered_path = get_bin_dir(self._get_venv_path()) / (discovered_name + suffix) + if discovered_path.exists(): + self._console_script_name = discovered_name + return self._console_script_name + + return None + def _get_connector_path(self) -> Path: suffix: Literal[".exe", ""] = ".exe" if is_windows() else "" - return get_bin_dir(self._get_venv_path()) / (self.name + suffix) + script_name = self._resolve_console_script_name() or self.name + return get_bin_dir(self._get_venv_path()) / (script_name + suffix) @property def interpreter_path(self) -> Path: @@ -103,6 +168,7 @@ def uninstall(self) -> None: rmtree(str(self._get_venv_path())) self.reported_version = None # Reset the reported version from the previous installation + self._console_script_name = None @property def docs_url(self) -> str: @@ -182,6 +248,7 @@ def install(self) -> None: raise exc.AirbyteConnectorInstallationError from ex # Assuming the installation succeeded, store the installed version + self._console_script_name = None self.reported_version = self.get_installed_version(raise_on_error=False, recheck=True) log_install_state(self.name, state=EventState.SUCCEEDED) print( @@ -212,7 +279,6 @@ def get_installed_version( if not recheck and self.reported_version: return self.reported_version - connector_name = self.name if not self.interpreter_path.exists(): # No point in trying to detect the version if the interpreter does not exist if raise_on_error: @@ -225,11 +291,7 @@ def get_installed_version( return None try: - package_name = ( - self.metadata.pypi_package_name - if self.metadata and self.metadata.pypi_package_name - else f"airbyte-{connector_name}" - ) + package_name = self._get_pypi_package_name() return subprocess.check_output( [ self.interpreter_path, @@ -281,7 +343,7 @@ def ensure_installation( self.install() reinstalled = True - elif not self._get_connector_path().exists(): + elif not self._resolve_console_script_name(): if not auto_fix: raise exc.AirbyteConnectorInstallationError( message="Could not locate connector executable within the virtual environment.", @@ -295,7 +357,7 @@ def ensure_installation( # This is sometimes caused by a failed or partial installation. print( "Connector executable not found within the virtual environment " - f"at {self._get_connector_path()!s}.\nReinstalling...", + f"at {get_bin_dir(self._get_venv_path()) / self.name!s}.\nReinstalling...", file=sys.stderr, ) self.uninstall() @@ -304,15 +366,15 @@ def ensure_installation( # By now, everything should be installed. Raise an exception if not. - connector_path = self._get_connector_path() - if not connector_path.exists(): + if not self._resolve_console_script_name(): raise exc.AirbyteConnectorInstallationError( message="Connector's executable could not be found within the virtual environment.", connector_name=self.name, context={ "connector_path": self._get_connector_path(), + "discovered_console_scripts": self._discover_console_script_name(), }, - ) from FileNotFoundError(connector_path) + ) from FileNotFoundError(self._get_connector_path()) if self.enforce_version: version_after_reinstall: str | None = None diff --git a/scripts/reproduce_issue_290.py b/scripts/reproduce_issue_290.py new file mode 100644 index 000000000..e7b12596d --- /dev/null +++ b/scripts/reproduce_issue_290.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Reproduce PyAirbyte issue #290 before/after the executable discovery fix.""" +from __future__ import annotations + +import os + +os.environ["AIRBYTE_NO_UV"] = "true" + +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +FIXTURE_DIR = REPO_ROOT / "tests/integration_tests/fixtures/source-wrong-exe" +CONNECTOR_NAME = "source-wrong-exe" + + +def main() -> int: + os.chdir(REPO_ROOT) + sys.path.insert(0, str(REPO_ROOT)) + + from airbyte._executors.python import VenvExecutor # noqa: PLC0415 + + install_root = Path(tempfile.mkdtemp(prefix="pyairbyte-issue-290-")) + executor = VenvExecutor( + name=CONNECTOR_NAME, + pip_url=str(FIXTURE_DIR), + install_root=install_root, + ) + + print("Installing connector with mismatched console script name...") + executor.install() + executor.ensure_installation() + + script_name = executor._resolve_console_script_name() # noqa: SLF001 + connector_path = executor._get_connector_path() # noqa: SLF001 + + print(f"Discovered console script: {script_name}") + print(f"Connector executable path: {connector_path}") + print(f"Executable exists: {connector_path.exists()}") + + if script_name == "wrong-script-name" and connector_path.exists(): + print("\nIssue #290 fix verified.") + return 0 + + print("\nUnexpected state — executable discovery may still be broken.") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration_tests/fixtures/source-wrong-exe/setup.py b/tests/integration_tests/fixtures/source-wrong-exe/setup.py new file mode 100644 index 000000000..87a370024 --- /dev/null +++ b/tests/integration_tests/fixtures/source-wrong-exe/setup.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +from setuptools import setup + +# Intentionally mismatched console script name — regression fixture for issue #290. +setup( + name="airbyte-source-wrong-exe", + version="0.0.1", + description="Test Source with mismatched executable name", + author="Airbyte", + author_email="contact@airbyte.io", + packages=["source_wrong_exe"], + entry_points={ + "console_scripts": [ + "wrong-script-name=source_wrong_exe.run:run", + ], + }, +) diff --git a/tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/__init__.py b/tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/__init__.py new file mode 100644 index 000000000..f70ecfc3a --- /dev/null +++ b/tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/run.py b/tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/run.py new file mode 100644 index 000000000..93a8572f2 --- /dev/null +++ b/tests/integration_tests/fixtures/source-wrong-exe/source_wrong_exe/run.py @@ -0,0 +1,24 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import json +import sys + +sample_spec = { + "type": "SPEC", + "spec": { + "documentationUrl": "https://example.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": {"type": "string"}, + }, + }, + }, +} + + +def run() -> None: + if sys.argv[1] == "spec": + print(json.dumps(sample_spec)) diff --git a/tests/unit_tests/test_issue_290_wrong_executable.py b/tests/unit_tests/test_issue_290_wrong_executable.py new file mode 100644 index 000000000..c04a2d29e --- /dev/null +++ b/tests/unit_tests/test_issue_290_wrong_executable.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""Regression tests for https://github.com/airbytehq/PyAirbyte/issues/290.""" +from __future__ import annotations + +import tempfile +from pathlib import Path + +import pytest + +from airbyte._executors.python import VenvExecutor + +REPO_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DIR = REPO_ROOT / "tests/integration_tests/fixtures/source-wrong-exe" + + +@pytest.fixture(autouse=True) +def _use_uv_for_install(monkeypatch: pytest.MonkeyPatch) -> None: + """Local connector installs require uv in this environment.""" + monkeypatch.setattr("airbyte._executors.python.NO_UV", False) + + +def test_discovers_console_script_when_name_differs_from_connector() -> None: + install_root = Path(tempfile.mkdtemp(prefix="pyairbyte-issue-290-test-")) + executor = VenvExecutor( + name="source-wrong-exe", + pip_url=str(FIXTURE_DIR), + install_root=install_root, + ) + + executor.install() + executor.ensure_installation() + + assert executor._resolve_console_script_name() == "wrong-script-name" # noqa: SLF001 + assert executor._get_connector_path().name == "wrong-script-name" # noqa: SLF001 + assert executor.pip_url == str(FIXTURE_DIR)