diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 58d57b1c7..a5ae56bb8 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -32,10 +32,10 @@ body: description: Version of Python you are using options: - "3.6 (deprecated)" - - "3.7" - "3.8" - "3.9" - "3.10" + - "3.11" - "NA" validations: required: true @@ -75,6 +75,6 @@ body: attributes: label: Code description: Paste the failing code and/or traceback, if applicable - render: python + render: Python validations: required: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8ee9ed957..e59fbdc54 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: matrix: session: [tests] os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] sqlalchemy: ["2.*"] include: - { session: tests, python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "1.*" } diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} index 0ea2f9ae7..2db186c11 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} @@ -12,7 +12,7 @@ jobs: GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }} strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python {{ '${{ matrix.python-version }}' }} diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index a1dcd7933..23213fefd 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -12,6 +12,14 @@ keywords = [ "ELT", "{{cookiecutter.source_name}}", ] +classifiers = [ + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] license = "Apache-2.0" {%- if cookiecutter.variant != "None (Skip)" %} packages = [ @@ -51,7 +59,7 @@ ignore = [ ] select = ["ALL"] src = ["{{cookiecutter.library_name}}"] -target-version = "py37" +target-version = "py38" [tool.ruff.flake8-annotations] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini index 70b9e4ac7..8de2dabff 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini @@ -1,7 +1,7 @@ # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy [tox] -envlist = py37, py38, py39, py310, py311 +envlist = py38, py39, py310, py311 isolated_build = true [testenv] @@ -13,7 +13,7 @@ commands = [testenv:pytest] # Run the python tests. # To execute, run `tox -e pytest` -envlist = py37, py38, py39, py310, py311 +envlist = py38, py39, py310, py311 commands = poetry install -v poetry run pytest diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index dae2269df..22955a86e 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -41,14 +41,6 @@ from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}A {% endif -%} -{%- if cookiecutter.auth_method in ("OAuth2", "JWT") -%} -if sys.version_info >= (3, 8): - from functools import cached_property -else: - from cached_property import cached_property - -{% endif -%} - _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} index 4544911a6..51501411d 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/{% if cookiecutter.include_ci_files == 'GitHub' %}test.yml{%endif%} @@ -12,7 +12,7 @@ jobs: GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }} strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python {{ '${{ matrix.python-version }}' }} diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index f88dca540..4fd948f14 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -12,6 +12,14 @@ keywords = [ "ELT", "{{cookiecutter.destination_name}}", ] +classifiers = [ + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] license = "Apache-2.0" {%- if cookiecutter.variant != "None (Skip)" %} packages = [ @@ -41,7 +49,7 @@ ignore = [ ] select = ["ALL"] src = ["{{cookiecutter.library_name}}"] -target-version = "py37" +target-version = "py38" [tool.ruff.flake8-annotations] allow-star-arg-any = true diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini b/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini index 70b9e4ac7..8de2dabff 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini @@ -1,7 +1,7 @@ # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy [tox] -envlist = py37, py38, py39, py310, py311 +envlist = py38, py39, py310, py311 isolated_build = true [testenv] @@ -13,7 +13,7 @@ commands = [testenv:pytest] # Run the python tests. # To execute, run `tox -e pytest` -envlist = py37, py38, py39, py310, py311 +envlist = py38, py39, py310, py311 commands = poetry install -v poetry run pytest diff --git a/noxfile.py b/noxfile.py index 6770641d4..c4c5fe50b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -28,7 +28,7 @@ COOKIECUTTER_REPLAY_FILES = list(Path("./e2e-tests/cookiecutters").glob("*.json")) package = "singer_sdk" -python_versions = ["3.11", "3.10", "3.9", "3.8", "3.7"] +python_versions = ["3.11", "3.10", "3.9", "3.8"] main_python_version = "3.10" locations = "singer_sdk", "tests", "noxfile.py", "docs/conf.py" nox.options.sessions = ( diff --git a/poetry.lock b/poetry.lock index 793a30e5a..69ff3de31 100644 --- a/poetry.lock +++ b/poetry.lock @@ -771,7 +771,7 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" -optional = false +optional = true python-versions = ">=3.8" files = [ {file = "importlib_metadata-6.8.0-py3-none-any.whl", hash = "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb"}, @@ -2538,5 +2538,5 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" -python-versions = ">=3.8,<4" -content-hash = "8f9c1238bd8810ffa2abbd147fb099ca5b51af433eda954865a38f0567cc010f" +python-versions = "<4,>=3.8" +content-hash = "5c7e77fab78c4b3eca751c17f5f70039c36a3ebfba628ca9611bd3d572f3691b" diff --git a/pyproject.toml b/pyproject.toml index 710976b3f..d56bacfaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,12 +37,11 @@ license = "Apache-2.0" "Youtube" = "https://www.youtube.com/meltano" [tool.poetry.dependencies] -python = ">=3.8,<4" +python = "<4,>=3.8" backoff = ">=2.0.0" click = "~=8.0" cryptography = ">=3.4.6,<42.0.0" fs = ">=2.4.16" -importlib-metadata = {version = "<7.0.0", markers = "python_version < \"3.8\""} importlib-resources = {version = ">=5.12.0", markers = "python_version < \"3.9\""} inflection = ">=0.5.1" joblib = ">=1.0.1" @@ -71,7 +70,7 @@ sphinx-copybutton = {version = ">=0.3.1", optional = true} myst-parser = {version = ">=1", optional = true} sphinx-autobuild = {version = ">=2021.3.14", optional = true} sphinx-reredirects = {version = ">=0.1.1", optional = true} -sphinx-inline-tabs = {version = ">=2023.4.21", optional = true, markers = "python_version >= \"3.8\""} +sphinx-inline-tabs = {version = ">=2023.4.21", optional = true} # File storage dependencies installed as optional 'filesystem' extras fs-s3fs = {version = ">=1.1.1", optional = true} @@ -104,10 +103,7 @@ coverage = {extras = ["toml"], version = ">=7.2"} duckdb = ">=0.8.0" duckdb-engine = ">=0.9.2" mypy = ">=1.0" -numpy = [ - { version = "<1.22", python = "<3.8" }, - { version = ">=1.22", python = ">=3.8" }, -] +numpy = ">=1.22" pyarrow = ">=11,<13" pytest-benchmark = "^4.0.0" pytest-snapshot = ">=0.9.0" @@ -285,8 +281,6 @@ unfixable = [ "tests/*" = ["ANN", "D1", "D2", "FBT001", "FBT003", "PLR2004", "S101"] # Disabled some checks in samples code "samples/*" = ["ANN", "D"] -# Don't require docstrings conventions or type annotations in private modules -"singer_sdk/helpers/_*.py" = ["ANN", "D105"] # Templates support a generic resource of type Any. "singer_sdk/testing/*.py" = ["S101"] "singer_sdk/testing/templates.py" = ["ANN401"] diff --git a/singer_sdk/helpers/_batch.py b/singer_sdk/helpers/_batch.py index 62447ddb3..7c1f142b1 100644 --- a/singer_sdk/helpers/_batch.py +++ b/singer_sdk/helpers/_batch.py @@ -82,7 +82,7 @@ class SDKBatchMessage(Message): manifest: list[str] = field(default_factory=list) """The manifest of files in the batch.""" - def __post_init__(self): + def __post_init__(self) -> None: if isinstance(self.encoding, dict): self.encoding = BaseBatchFileEncoding.from_dict(self.encoding) @@ -102,7 +102,7 @@ class StorageTarget: params: dict = field(default_factory=dict) """"The storage parameters.""" - def asdict(self): + def asdict(self) -> dict[str, t.Any]: """Return a dictionary representation of the message. Returns: @@ -134,7 +134,7 @@ def split_url(url: str) -> tuple[str, str]: """ if platform.system() == "Windows" and "\\" in url: # Original code from pyFileSystem split - # Augemnted slitly to properly Windows paths + # Augmented slightly to properly handle Windows path split = url.rsplit("\\", 1) return (split[0] or "\\", split[1]) @@ -214,7 +214,7 @@ class BatchConfig: batch_size: int = DEFAULT_BATCH_SIZE """The max number of records in a batch.""" - def __post_init__(self): + def __post_init__(self) -> None: if isinstance(self.encoding, dict): self.encoding = BaseBatchFileEncoding.from_dict(self.encoding) @@ -224,7 +224,7 @@ def __post_init__(self): if self.batch_size is None: self.batch_size = DEFAULT_BATCH_SIZE - def asdict(self): + def asdict(self) -> dict[str, t.Any]: """Return a dictionary representation of the message. Returns: diff --git a/singer_sdk/helpers/_flattening.py b/singer_sdk/helpers/_flattening.py index 32288fb3c..edfe65e60 100644 --- a/singer_sdk/helpers/_flattening.py +++ b/singer_sdk/helpers/_flattening.py @@ -70,7 +70,7 @@ def flatten_key(key_name: str, parent_keys: list[str], separator: str = "__") -> inflection.camelize(inflected_key[reducer_index]), ) inflected_key[reducer_index] = ( - reduced_key if len(reduced_key) > 1 else inflected_key[reducer_index][0:3] + reduced_key if len(reduced_key) > 1 else inflected_key[reducer_index][:3] ).lower() reducer_index += 1 @@ -358,8 +358,8 @@ def _flatten_schema( # noqa: C901, PLR0912 items.append((new_key, next(iter(field_schema.values()))[0])) # Sort and check for duplicates - def _key_func(item): - return item[0] # first item is tuple is the key name. + def _key_func(item: tuple[str, dict]) -> str: + return item[0] # first item in tuple is the key name. sorted_items = sorted(items, key=_key_func) for field_name, g in itertools.groupby(sorted_items, key=_key_func): @@ -451,7 +451,11 @@ def _flatten_record( return dict(items) -def _should_jsondump_value(key: str, value: t.Any, flattened_schema=None) -> bool: +def _should_jsondump_value( + key: str, + value: t.Any, # noqa: ANN401 + flattened_schema: dict[str, t.Any] | None = None, +) -> bool: """Return True if json.dump() should be used to serialize the value. Args: diff --git a/singer_sdk/helpers/_secrets.py b/singer_sdk/helpers/_secrets.py index ad7d05032..bbababa70 100644 --- a/singer_sdk/helpers/_secrets.py +++ b/singer_sdk/helpers/_secrets.py @@ -28,7 +28,7 @@ def is_common_secret_key(key_name: str) -> bool: class SecretString(str): """For now, this class wraps a sensitive string to be identified as such later.""" - def __init__(self, contents): + def __init__(self, contents: str) -> None: """Initialize secret string.""" self.contents = contents diff --git a/singer_sdk/helpers/_state.py b/singer_sdk/helpers/_state.py index 9d0102186..c40ba072f 100644 --- a/singer_sdk/helpers/_state.py +++ b/singer_sdk/helpers/_state.py @@ -18,12 +18,12 @@ STARTING_MARKER = "starting_replication_value" -def get_state_if_exists( # noqa: PLR0911 +def get_state_if_exists( tap_state: dict, tap_stream_id: str, state_partition_context: dict | None = None, key: str | None = None, -) -> t.Any | None: +) -> t.Any | None: # noqa: ANN401 """Return the stream or partition state, creating a new one if it does not exist. Args: @@ -47,9 +47,7 @@ def get_state_if_exists( # noqa: PLR0911 stream_state = tap_state["bookmarks"][tap_stream_id] if not state_partition_context: - if key: - return stream_state.get(key, None) - return stream_state + return stream_state.get(key, None) if key else stream_state if "partitions" not in stream_state: return None # No partitions defined @@ -59,9 +57,7 @@ def get_state_if_exists( # noqa: PLR0911 ) if matched_partition is None: return None # Partition definition not present - if key: - return matched_partition.get(key, None) - return matched_partition + return matched_partition.get(key, None) if key else matched_partition def get_state_partitions_list(tap_state: dict, tap_stream_id: str) -> list[dict] | None: @@ -84,10 +80,7 @@ def _find_in_partitions_list( f"{{state_partition_context}}.\nMatching state values were: {found!s}" ) raise ValueError(msg) - if found: - return t.cast(dict, found[0]) - - return None + return t.cast(dict, found[0]) if found else None def _create_in_partitions_list( @@ -134,18 +127,20 @@ def get_writeable_state_dict( if "partitions" not in stream_state: stream_state["partitions"] = [] stream_state_partitions: list[dict] = stream_state["partitions"] - found = _find_in_partitions_list(stream_state_partitions, state_partition_context) - if found: + if found := _find_in_partitions_list( + stream_state_partitions, + state_partition_context, + ): return found return _create_in_partitions_list(stream_state_partitions, state_partition_context) def write_stream_state( - tap_state, + tap_state: dict, tap_stream_id: str, - key, - val, + key: str, + val: t.Any, # noqa: ANN401 *, state_partition_context: dict | None = None, ) -> None: @@ -172,7 +167,7 @@ def reset_state_progress_markers(stream_or_partition_state: dict) -> dict | None def write_replication_key_signpost( stream_or_partition_state: dict, - new_signpost_value: t.Any, + new_signpost_value: t.Any, # noqa: ANN401, ) -> None: """Write signpost value.""" stream_or_partition_state[SIGNPOST_MARKER] = to_json_compatible(new_signpost_value) @@ -180,13 +175,15 @@ def write_replication_key_signpost( def write_starting_replication_value( stream_or_partition_state: dict, - initial_value: t.Any, + initial_value: t.Any, # noqa: ANN401 ) -> None: """Write initial replication value to state.""" stream_or_partition_state[STARTING_MARKER] = to_json_compatible(initial_value) -def get_starting_replication_value(stream_or_partition_state: dict): +def get_starting_replication_value( + stream_or_partition_state: dict, +) -> t.Any | None: # noqa: ANN401 """Retrieve initial replication marker value from state.""" if not stream_or_partition_state: return None diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 0ab018e51..1223823e7 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -39,7 +39,7 @@ def __init__(self, *args: object) -> None: super().__init__(msg, *args) -def to_json_compatible(val: t.Any) -> t.Any: +def to_json_compatible(val: t.Any) -> t.Any: # noqa: ANN401 """Return as string if datetime. JSON does not support proper datetime types. If given a naive datetime object, pendulum automatically makes it utc @@ -185,7 +185,7 @@ def get_datelike_property_type(property_schema: dict) -> str | None: return None -def _is_string_with_format(type_dict): +def _is_string_with_format(type_dict: dict[str, t.Any]) -> bool | None: if "string" in type_dict.get("type", []) and type_dict.get("format") in { "date-time", "time", @@ -196,14 +196,14 @@ def _is_string_with_format(type_dict): def handle_invalid_timestamp_in_record( - record, # noqa: ARG001 + record: dict[str, t.Any], # noqa: ARG001 key_breadcrumb: list[str], invalid_value: str, datelike_typename: str, ex: Exception, treatment: DatetimeErrorTreatmentEnum | None, logger: logging.Logger, -) -> t.Any: +) -> t.Any: # noqa: ANN401 """Apply treatment or raise an error for invalid time values.""" treatment = treatment or DatetimeErrorTreatmentEnum.ERROR msg = ( @@ -331,7 +331,7 @@ def _warn_unmapped_properties( stream_name: str, property_names: tuple[str], logger: logging.Logger, -): +) -> None: logger.warning( "Properties %s were present in the '%s' stream but " "not found in catalog schema. Ignoring.", @@ -469,9 +469,9 @@ def _conform_record_data_types( # noqa: PLR0912 def _conform_primitive_property( # noqa: PLR0911 - elem: t.Any, + elem: t.Any, # noqa: ANN401 property_schema: dict, -) -> t.Any: +) -> t.Any: # noqa: ANN401 """Converts a primitive (i.e. not object or array) to a json compatible type.""" if isinstance(elem, (datetime.datetime, pendulum.DateTime)): return to_json_compatible(elem) diff --git a/singer_sdk/pagination.py b/singer_sdk/pagination.py index f00bb0920..43b01dfdf 100644 --- a/singer_sdk/pagination.py +++ b/singer_sdk/pagination.py @@ -2,18 +2,12 @@ from __future__ import annotations -import sys import typing as t from abc import ABCMeta, abstractmethod from urllib.parse import ParseResult, urlparse from singer_sdk.helpers.jsonpath import extract_jsonpath -if sys.version_info >= (3, 8): - from typing import Protocol # noqa: ICN003 -else: - from typing_extensions import Protocol - if t.TYPE_CHECKING: from requests import Response @@ -406,7 +400,7 @@ def get_next(self, response: Response) -> int | None: # noqa: ARG002 return self._value + self._page_size -class LegacyPaginatedStreamProtocol(Protocol[TPageToken]): +class LegacyPaginatedStreamProtocol(t.Protocol[TPageToken]): """Protocol for legacy paginated streams classes.""" def get_next_page_token( diff --git a/singer_sdk/streams/sql.py b/singer_sdk/streams/sql.py index 18d2d8862..bbbb77625 100644 --- a/singer_sdk/streams/sql.py +++ b/singer_sdk/streams/sql.py @@ -4,6 +4,7 @@ import abc import typing as t +from functools import cached_property import singer_sdk.helpers._catalog as catalog from singer_sdk._singerlib import CatalogEntry, MetadataMapping @@ -73,7 +74,7 @@ def metadata(self) -> MetadataMapping: """ return self._singer_catalog_entry.metadata - @property # TODO: Investigate @cached_property after py > 3.7 + @cached_property def schema(self) -> dict: """Return metadata object (dict) as specified in the Singer spec. @@ -82,13 +83,7 @@ def schema(self) -> dict: Returns: The schema object. """ - if not self._cached_schema: - self._cached_schema = t.cast( - dict, - self._singer_catalog_entry.schema.to_dict(), - ) - - return self._cached_schema + return self._singer_catalog_entry.schema.to_dict() @property def tap_stream_id(self) -> str: