From 65cb845de2912f20652a4ce5e61670261f5df3bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Fri, 26 Jan 2024 10:47:29 -0600 Subject: [PATCH 1/3] chore: Replace isort, flake8, black and pyupgrade with Ruff --- .pre-commit-config.yaml | 45 +++--- README.md | 2 +- log_based/init.sql | 2 +- poetry.lock | 209 ++++++++++++---------------- pyproject.toml | 60 ++++++-- tap_postgres/client.py | 79 ++++------- tap_postgres/tap.py | 65 +++++---- tests/test_core.py | 109 ++++++++------- tests/test_log_based.py | 16 +-- tests/test_replication_key.py | 31 +++-- tests/test_selected_columns_only.py | 14 +- tox.ini | 31 +---- 12 files changed, 319 insertions(+), 344 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cdf38d..be77e27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,32 +1,25 @@ +ci: + autofix_prs: false + autoupdate_schedule: weekly + autoupdate_commit_msg: 'chore: pre-commit autoupdate' + repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v4.5.0 hooks: - - id: check-yaml + - id: check-json + - id: check-toml + exclude: | + (?x)^( + copier_template/.*/pyproject.toml + )$ - id: end-of-file-fixer + exclude: (copier_template/.*|docs/.*|samples/.*\.json) - id: trailing-whitespace -- repo: https://github.com/psf/black - rev: 22.10.0 + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.14 hooks: - - id: black -- repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort -- repo: https://github.com/python-poetry/poetry - rev: 1.3.2 - hooks: - - id: poetry-check - - id: poetry-lock - args: [--no-update] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.991' - hooks: - - id: mypy - exclude: tests - additional_dependencies: - - types-paramiko -- repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 \ No newline at end of file + - id: ruff + args: [--fix] + - id: ruff-format diff --git a/README.md b/README.md index 7f538dc..226d317 100644 --- a/README.md +++ b/README.md @@ -261,4 +261,4 @@ Note also that using log-based replication will cause the replication key for al "*": replication_method: LOG_BASED replication_key: _sdc_lsn - ``` \ No newline at end of file + ``` diff --git a/log_based/init.sql b/log_based/init.sql index 4cd8790..a700552 100644 --- a/log_based/init.sql +++ b/log_based/init.sql @@ -1 +1 @@ -SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json'); \ No newline at end of file +SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json'); diff --git a/poetry.lock b/poetry.lock index 40cfc6e..1653a05 100644 --- a/poetry.lock +++ b/poetry.lock @@ -149,52 +149,6 @@ files = [ tests = ["pytest (>=3.2.1,!=3.3.0)"] typecheck = ["mypy"] -[[package]] -name = "black" -version = "23.12.1" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.8" -files = [ - {file = "black-23.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2"}, - {file = "black-23.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba"}, - {file = "black-23.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0"}, - {file = "black-23.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3"}, - {file = "black-23.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba"}, - {file = "black-23.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b"}, - {file = "black-23.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59"}, - {file = "black-23.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50"}, - {file = "black-23.12.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e"}, - {file = "black-23.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec"}, - {file = "black-23.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e"}, - {file = "black-23.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9"}, - {file = "black-23.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f"}, - {file = "black-23.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d"}, - {file = "black-23.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a"}, - {file = "black-23.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e"}, - {file = "black-23.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055"}, - {file = "black-23.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54"}, - {file = "black-23.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea"}, - {file = "black-23.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2"}, - {file = "black-23.12.1-py3-none-any.whl", hash = "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e"}, - {file = "black-23.12.1.tar.gz", hash = "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -packaging = ">=22.0" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - [[package]] name = "cachetools" version = "5.3.2" @@ -528,22 +482,6 @@ docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1 testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] typing = ["typing-extensions (>=4.8)"] -[[package]] -name = "flake8" -version = "7.0.0" -description = "the modular source code checker: pep8 pyflakes and co" -optional = false -python-versions = ">=3.8.1" -files = [ - {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, - {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, -] - -[package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.11.0,<2.12.0" -pyflakes = ">=3.2.0,<3.3.0" - [[package]] name = "fs" version = "2.4.16" @@ -718,20 +656,6 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "isort" -version = "5.13.2" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, - {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, -] - -[package.extras] -colors = ["colorama (>=0.4.6)"] - [[package]] name = "joblib" version = "1.3.2" @@ -795,17 +719,6 @@ files = [ importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} referencing = ">=0.31.0" -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - [[package]] name = "memoization" version = "0.4.0" @@ -920,17 +833,6 @@ all = ["gssapi (>=1.4.1)", "invoke (>=2.0)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1 gssapi = ["gssapi (>=1.4.1)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] invoke = ["invoke (>=2.0)"] -[[package]] -name = "pathspec" -version = "0.12.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, - {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, -] - [[package]] name = "pendulum" version = "3.0.0" @@ -1183,17 +1085,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, ] -[[package]] -name = "pycodestyle" -version = "2.11.1" -description = "Python style guide checker" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, -] - [[package]] name = "pycparser" version = "2.21" @@ -1222,17 +1113,6 @@ snowballstemmer = ">=2.2.0" [package.extras] toml = ["tomli (>=1.2.3)"] -[[package]] -name = "pyflakes" -version = "3.2.0" -description = "passive checker of Python programs" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, - {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, -] - [[package]] name = "pyjwt" version = "2.8.0" @@ -1574,6 +1454,32 @@ files = [ {file = "rpds_py-0.16.2.tar.gz", hash = "sha256:781ef8bfc091b19960fc0142a23aedadafa826bc32b433fdfe6fd7f964d7ef44"}, ] +[[package]] +name = "ruff" +version = "0.1.14" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.14-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:96f76536df9b26622755c12ed8680f159817be2f725c17ed9305b472a757cdbb"}, + {file = "ruff-0.1.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ab3f71f64498c7241123bb5a768544cf42821d2a537f894b22457a543d3ca7a9"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7060156ecc572b8f984fd20fd8b0fcb692dd5d837b7606e968334ab7ff0090ab"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a53d8e35313d7b67eb3db15a66c08434809107659226a90dcd7acb2afa55faea"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bea9be712b8f5b4ebed40e1949379cfb2a7d907f42921cf9ab3aae07e6fba9eb"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2270504d629a0b064247983cbc495bed277f372fb9eaba41e5cf51f7ba705a6a"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80258bb3b8909b1700610dfabef7876423eed1bc930fe177c71c414921898efa"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:653230dd00aaf449eb5ff25d10a6e03bc3006813e2cb99799e568f55482e5cae"}, + {file = "ruff-0.1.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b3acc6c4e6928459ba9eb7459dd4f0c4bf266a053c863d72a44c33246bfdbf"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6b3dadc9522d0eccc060699a9816e8127b27addbb4697fc0c08611e4e6aeb8b5"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1c8eca1a47b4150dc0fbec7fe68fc91c695aed798532a18dbb1424e61e9b721f"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:62ce2ae46303ee896fc6811f63d6dabf8d9c389da0f3e3f2bce8bc7f15ef5488"}, + {file = "ruff-0.1.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b2027dde79d217b211d725fc833e8965dc90a16d0d3213f1298f97465956661b"}, + {file = "ruff-0.1.14-py3-none-win32.whl", hash = "sha256:722bafc299145575a63bbd6b5069cb643eaa62546a5b6398f82b3e4403329cab"}, + {file = "ruff-0.1.14-py3-none-win_amd64.whl", hash = "sha256:e3d241aa61f92b0805a7082bd89a9990826448e4d0398f0e2bc8f05c75c63d99"}, + {file = "ruff-0.1.14-py3-none-win_arm64.whl", hash = "sha256:269302b31ade4cde6cf6f9dd58ea593773a37ed3f7b97e793c8594b262466b67"}, + {file = "ruff-0.1.14.tar.gz", hash = "sha256:ad3f8088b2dfd884820289a06ab718cde7d38b94972212cc4ba90d5fbc9955f3"}, +] + [[package]] name = "setuptools" version = "69.0.3" @@ -1912,6 +1818,67 @@ virtualenv = ">=20.25" docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-argparse-cli (>=1.11.1)", "sphinx-autodoc-typehints (>=1.25.2)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.11)"] testing = ["build[virtualenv] (>=1.0.3)", "covdefaults (>=2.3)", "detect-test-pollution (>=1.2)", "devpi-process (>=1)", "diff-cover (>=8.0.2)", "distlib (>=0.3.8)", "flaky (>=3.7)", "hatch-vcs (>=0.4)", "hatchling (>=1.21)", "psutil (>=5.9.7)", "pytest (>=7.4.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-xdist (>=3.5)", "re-assert (>=1.1)", "time-machine (>=2.13)", "wheel (>=0.42)"] +[[package]] +name = "types-jsonschema" +version = "4.21.0.20240118" +description = "Typing stubs for jsonschema" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-jsonschema-4.21.0.20240118.tar.gz", hash = "sha256:31aae1b5adc0176c1155c2d4f58348b22d92ae64315e9cc83bd6902168839232"}, + {file = "types_jsonschema-4.21.0.20240118-py3-none-any.whl", hash = "sha256:77a4ac36b0be4f24274d5b9bf0b66208ee771c05f80e34c4641de7d63e8a872d"}, +] + +[package.dependencies] +referencing = "*" + +[[package]] +name = "types-paramiko" +version = "3.4.0.20240120" +description = "Typing stubs for paramiko" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-paramiko-3.4.0.20240120.tar.gz", hash = "sha256:07e5992e995c2266a0803a332a69d912036be7664ca36a1ca01dac1b67643132"}, + {file = "types_paramiko-3.4.0.20240120-py3-none-any.whl", hash = "sha256:b0b44764c6c61a3e7629ee5923dc78b1a22b1d029522df321997697501e30d97"}, +] + +[package.dependencies] +cryptography = ">=37.0.0" + +[[package]] +name = "types-psycopg2" +version = "2.9.21.20240118" +description = "Typing stubs for psycopg2" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-psycopg2-2.9.21.20240118.tar.gz", hash = "sha256:e4a06316e7c9690255175c3ee5dffa5b47c5057f17181f5e34c6dcdb34066f35"}, + {file = "types_psycopg2-2.9.21.20240118-py3-none-any.whl", hash = "sha256:08c024f7da3a78c2c0404305f96c2b6067185d690cc4e9d14fc6ea595879ff8a"}, +] + +[[package]] +name = "types-simplejson" +version = "3.19.0.2" +description = "Typing stubs for simplejson" +optional = false +python-versions = "*" +files = [ + {file = "types-simplejson-3.19.0.2.tar.gz", hash = "sha256:ebc81f886f89d99d6b80c726518aa2228bc77c26438f18fd81455e4f79f8ee1b"}, + {file = "types_simplejson-3.19.0.2-py3-none-any.whl", hash = "sha256:8ba093dc7884f59b3e62aed217144085e675a269debc32678fd80e0b43b2b86f"}, +] + +[[package]] +name = "types-sqlalchemy" +version = "1.4.53.38" +description = "Typing stubs for SQLAlchemy" +optional = false +python-versions = "*" +files = [ + {file = "types-SQLAlchemy-1.4.53.38.tar.gz", hash = "sha256:5bb7463537e04e1aa5a3557eb725930df99226dcfd3c9bf93008025bfe5c169e"}, + {file = "types_SQLAlchemy-1.4.53.38-py3-none-any.whl", hash = "sha256:7e60e74f823931cc9a9e8adb0a4c05e5533e6708b8a266807893a739faf4eaaa"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -1988,4 +1955,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "<3.13,>=3.8.1" -content-hash = "be7623681c33f3fbcb99780e77d59c583442841d5c413d79243404166c18edcc" +content-hash = "58486f54a50cff0a34e823b8c141abf19e0700768b17ca88ae1001d8f33529b7" diff --git a/pyproject.toml b/pyproject.toml index 00a4bb0..be71545 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,25 +39,23 @@ sqlalchemy = "<2" sshtunnel = "0.4.0" [tool.poetry.group.dev.dependencies] -black = "^23.1.0" faker = ">=18.5.1,<23.0.0" -flake8 = "^7.0.0" -isort = "^5.10.1" mypy = "1.8.0" pendulum = "~=3.0" pre-commit = "^3.0.4" +ruff = "^0.1.14" pydocstyle = "^6.1.1" singer-sdk = {version = "*", extras = ["testing"]} tox = "^4" - -[tool.isort] -profile = "black" -multi_line_output = 3 # Vertical Hanging Indent -src_paths = "tap_postgres" +types-paramiko = "^3.3.0.0" +types-simplejson = "^3.19.0.2" +types-sqlalchemy = "^1.4.53.38" +types-jsonschema = "^4.19.0.3" +types-psycopg2 = "^2.9.21.20240118" [tool.mypy] exclude = "tests" -python_version = "3.9" +python_version = "3.11" warn_unused_configs = true warn_unused_ignores = true @@ -68,9 +66,51 @@ module = [ ] [build-system] -requires = ["poetry-core>=1.0.8", "poetry-dynamic-versioning"] +requires = ["poetry-core==1.8.1", "poetry-dynamic-versioning==1.2.0"] build-backend = "poetry_dynamic_versioning.backend" [tool.poetry.scripts] # CLI declaration tap-postgres = 'tap_postgres.tap:TapPostgres.cli' + +[tool.poetry-dynamic-versioning] +enable = true +vcs = "git" +style = "pep440" + +[tool.ruff] +target-version = "py38" + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "W", # pycodestyle warnings + "E", # pycodestyle errors + "I", # isort + "N", # pep8-naming + "D", # pydocsyle + "UP", # pyupgrade + "ICN", # flake8-import-conventions + "RET", # flake8-return + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + "ERA", # eradicate + "PGH", # pygrep-hooks + "PL", # Pylint + "PERF", # Perflint + "RUF", # ruff +] + +[tool.ruff.lint.flake8-import-conventions] +banned-from = ["sqlalchemy"] + +[tool.ruff.lint.flake8-import-conventions.extend-aliases] +sqlalchemy = "sa" + +[tool.ruff.per-file-ignores] +"tests/*" = [ + "D", +] + +[tool.ruff.lint.pydocstyle] +convention = "google" diff --git a/tap_postgres/client.py b/tap_postgres/client.py index f09ea74..86c83b0 100644 --- a/tap_postgres/client.py +++ b/tap_postgres/client.py @@ -11,24 +11,24 @@ import typing from functools import cached_property from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Dict, Iterable, Mapping, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Iterable, Mapping import pendulum import psycopg2 import singer_sdk.helpers._typing -import sqlalchemy +import sqlalchemy as sa from psycopg2 import extras from singer_sdk import SQLConnector, SQLStream from singer_sdk import typing as th from singer_sdk.helpers._state import increment_state from singer_sdk.helpers._typing import TypeConformanceLevel from singer_sdk.streams.core import REPLICATION_INCREMENTAL -from sqlalchemy import nullsfirst -from sqlalchemy.engine import Engine -from sqlalchemy.engine.reflection import Inspector if TYPE_CHECKING: from sqlalchemy.dialects import postgresql + from sqlalchemy.engine import Engine + from sqlalchemy.engine.reflection import Inspector + from sqlalchemy.types import TypeEngine def patched_conform( @@ -117,19 +117,13 @@ def __init__( # Note super is static, we can get away with this because this is called once # and is luckily referenced via the instance of the class - def to_jsonschema_type( + def to_jsonschema_type( # type: ignore[override] self, - sql_type: Union[ - str, - sqlalchemy.types.TypeEngine, - Type[sqlalchemy.types.TypeEngine], - postgresql.ARRAY, - Any, - ], + sql_type: str | TypeEngine | type[TypeEngine] | postgresql.ARRAY | Any, ) -> dict: """Return a JSON Schema representation of the provided type. - Overidden from SQLConnector to correctly handle JSONB and Arrays. + Overridden from SQLConnector to correctly handle JSONB and Arrays. Also Overridden in order to call our instance method `sdk_typing_object()` instead of the static version @@ -137,29 +131,26 @@ def to_jsonschema_type( By default will call `typing.to_jsonschema_type()` for strings and SQLAlchemy types. - Args - ---- + Args: sql_type: The string representation of the SQL type, a SQLAlchemy TypeEngine class or object, or a custom-specified object. - Raises - ------ + Raises: ValueError: If the type received could not be translated to jsonschema. - Returns - ------- + Returns: The JSON Schema representation of the provided type. """ type_name = None if isinstance(sql_type, str): type_name = sql_type - elif isinstance(sql_type, sqlalchemy.types.TypeEngine): + elif isinstance(sql_type, sa.types.TypeEngine): type_name = type(sql_type).__name__ if ( type_name is not None - and isinstance(sql_type, sqlalchemy.dialects.postgresql.ARRAY) + and isinstance(sql_type, sa.dialects.postgresql.ARRAY) and type_name == "ARRAY" ): array_type = self.sdk_typing_object(sql_type.item_type) @@ -168,9 +159,7 @@ def to_jsonschema_type( def sdk_typing_object( self, - from_type: str - | sqlalchemy.types.TypeEngine - | type[sqlalchemy.types.TypeEngine], + from_type: str | TypeEngine | type[TypeEngine], ) -> ( th.DateTimeType | th.NumberType @@ -178,22 +167,19 @@ def sdk_typing_object( | th.DateType | th.StringType | th.BooleanType + | th.CustomType ): """Return the JSON Schema dict that describes the sql type. - Args - ---- + Args: from_type: The SQL type as a string or as a TypeEngine. If a TypeEngine is provided, it may be provided as a class or a specific object instance. - Raises - ------ + Raises: ValueError: If the `from_type` value is not of type `str` or `TypeEngine`. - Returns - ------- + Returns: A compatible JSON Schema type definition. - """ # NOTE: This is an ordered mapping, with earlier mappings taking precedence. If # the SQL-provided type contains the type name on the left, the mapping will @@ -209,7 +195,8 @@ def sdk_typing_object( | th.IntegerType | th.DateType | th.StringType - | th.BooleanType, + | th.BooleanType + | th.CustomType, ] = { "jsonb": th.CustomType( {"type": ["string", "number", "integer", "array", "object", "boolean"]} @@ -236,11 +223,9 @@ def sdk_typing_object( sqltype_lookup["datetime"] = th.StringType() if isinstance(from_type, str): type_name = from_type - elif isinstance(from_type, sqlalchemy.types.TypeEngine): + elif isinstance(from_type, sa.types.TypeEngine): type_name = type(from_type).__name__ - elif isinstance(from_type, type) and issubclass( - from_type, sqlalchemy.types.TypeEngine - ): + elif isinstance(from_type, type) and issubclass(from_type, sa.types.TypeEngine): type_name = from_type.__name__ else: raise ValueError( @@ -274,30 +259,26 @@ class PostgresStream(SQLStream): connector_class = PostgresConnector - # JSONB Objects won't be selected without type_confomance_level to ROOT_ONLY + # JSONB Objects won't be selected without type_conformance_level to ROOT_ONLY TYPE_CONFORMANCE_LEVEL = TypeConformanceLevel.ROOT_ONLY - def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: + def get_records(self, context: dict | None) -> Iterable[dict[str, Any]]: """Return a generator of row-type dictionary objects. If the stream has a replication_key value defined, records will be sorted by the incremental key. If the stream also has an available starting bookmark, the records will be filtered for values greater than or equal to the bookmark value. - Args - ---- + Args: context: If partition context is provided, will read specifically from this data slice. - Yields - ------ + Yields: One dict per record. - Raises - ------ + Raises: NotImplementedError: If partition is passed in context and the stream does not support partitioning. - """ if context: raise NotImplementedError( @@ -315,7 +296,7 @@ def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: # Nulls first because the default is to have nulls as the "highest" value # which incorrectly causes the tap to attempt to store null state. - query = query.order_by(nullsfirst(replication_key_col)) + query = query.order_by(sa.nullsfirst(replication_key_col)) start_val = self.get_starting_replication_key_value(context) if start_val: @@ -391,7 +372,7 @@ def _increment_stream_state( check_sorted=self.check_sorted, ) - def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: + def get_records(self, context: dict | None) -> Iterable[dict[str, Any]]: """Return a generator of row-type dictionary objects.""" status_interval = 5.0 # if no records in 5 seconds the tap can exit start_lsn = self.get_starting_replication_key_value(context=context) @@ -529,6 +510,6 @@ def logical_replication_connection(self): # TODO: Make this change upstream in the SDK? # I'm not sure if in general SQL databases don't guarantee order of records log # replication, but at least Postgres does not. - def is_sorted(self) -> bool: + def is_sorted(self) -> bool: # type: ignore[override] """Return True if the stream is sorted by the replication key.""" return self.replication_method == REPLICATION_INCREMENTAL diff --git a/tap_postgres/tap.py b/tap_postgres/tap.py index 0aebab2..a603c34 100644 --- a/tap_postgres/tap.py +++ b/tap_postgres/tap.py @@ -5,12 +5,13 @@ import copy import io import signal +import sys from functools import cached_property from os import chmod, path -from typing import Any, Dict, cast +from typing import TYPE_CHECKING, Any, Sequence, cast import paramiko -from singer_sdk import SQLTap, Stream +from singer_sdk import SQLStream, SQLTap, Stream from singer_sdk import typing as th from singer_sdk._singerlib import ( # JSON schema typing helpers Catalog, @@ -27,6 +28,9 @@ PostgresStream, ) +if TYPE_CHECKING: + from collections.abc import Mapping + class TapPostgres(SQLTap): """Singer tap for Postgres.""" @@ -247,7 +251,7 @@ def __init__( th.StringType, default="verify-full", description=( - "SSL Protection method, see [postgres documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION)" # noqa: E501 + "SSL Protection method, see [postgres documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION)" + " for more information. Must be one of disable, allow, prefer," + " require, verify-ca, or verify-full." + " Note if sqlalchemy_url is set this will be ignored." @@ -309,7 +313,7 @@ def __init__( ), ).to_dict() - def get_sqlalchemy_url(self, config: Dict[Any, Any]) -> str: + def get_sqlalchemy_url(self, config: Mapping[str, Any]) -> str: """Generate a SQLAlchemy URL. Args: @@ -318,19 +322,18 @@ def get_sqlalchemy_url(self, config: Dict[Any, Any]) -> str: if config.get("sqlalchemy_url"): return cast(str, config["sqlalchemy_url"]) - else: - sqlalchemy_url = URL.create( - drivername="postgresql+psycopg2", - username=config["user"], - password=config["password"], - host=config["host"], - port=config["port"], - database=config["database"], - query=self.get_sqlalchemy_query(config=config), - ) - return cast(str, sqlalchemy_url) + sqlalchemy_url = URL.create( + drivername="postgresql+psycopg2", + username=config["user"], + password=config["password"], + host=config["host"], + port=config["port"], + database=config["database"], + query=self.get_sqlalchemy_query(config=config), + ) + return cast(str, sqlalchemy_url) - def get_sqlalchemy_query(self, config: dict) -> dict: + def get_sqlalchemy_query(self, config: Mapping[str, Any]) -> dict: """Get query values to be used for sqlalchemy URL creation. Args: @@ -390,12 +393,13 @@ def filepath_or_certificate( """ if path.isfile(value): return value - else: - with open(alternative_name, "wb") as alternative_file: - alternative_file.write(value.encode("utf-8")) - if restrict_permissions: - chmod(alternative_name, 0o600) - return alternative_name + + with open(alternative_name, "wb") as alternative_file: + alternative_file.write(value.encode("utf-8")) + if restrict_permissions: + chmod(alternative_name, 0o600) + + return alternative_name @cached_property def connector(self) -> PostgresConnector: @@ -439,8 +443,8 @@ def guess_key_type(self, key_data: str) -> paramiko.PKey: paramiko.Ed25519Key, ): try: - key = key_class.from_private_key(io.StringIO(key_data)) # type: ignore[attr-defined] # noqa: E501 - except paramiko.SSHException: + key = key_class.from_private_key(io.StringIO(key_data)) # type: ignore[attr-defined] + except paramiko.SSHException: # noqa: PERF203 continue else: return key @@ -491,7 +495,7 @@ def catch_signal(self, signum, frame) -> None: signum: The signal number frame: The current stack frame """ - exit(1) # Calling this to be sure atexit is called, so clean_up gets called + sys.exit(1) # Calling this to be sure atexit is called, so clean_up gets called @property def catalog_dict(self) -> dict: @@ -515,7 +519,7 @@ def catalog_dict(self) -> dict: return self._catalog_dict @property - def catalog(self) -> Catalog: # noqa: C901 + def catalog(self) -> Catalog: """Get the tap's working catalog. Override to do LOG_BASED modifications. @@ -528,7 +532,10 @@ def catalog(self) -> Catalog: # noqa: C901 for stream in super().catalog.streams: stream_modified = False new_stream = copy.deepcopy(stream) - if new_stream.replication_method == "LOG_BASED": + if ( + new_stream.replication_method == "LOG_BASED" + and new_stream.schema.properties + ): for property in new_stream.schema.properties.values(): if "null" not in property.type: if isinstance(property.type, list): @@ -573,13 +580,13 @@ def catalog(self) -> Catalog: # noqa: C901 ) return new_catalog - def discover_streams(self) -> list[Stream]: + def discover_streams(self) -> Sequence[Stream]: # type: ignore[override] """Initialize all available streams and return them as a list. Returns: List of discovered Stream objects. """ - streams = [] + streams: list[SQLStream] = [] for catalog_entry in self.catalog_dict["streams"]: if catalog_entry["replication_method"] == "LOG_BASED": streams.append( diff --git a/tests/test_core.py b/tests/test_core.py index 1bb8614..c794963 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5,20 +5,21 @@ import pendulum import pytest -import sqlalchemy +import sqlalchemy as sa from faker import Faker from singer_sdk.testing import get_tap_test_class, suites from singer_sdk.testing.runners import TapTestRunner -from sqlalchemy import Column, DateTime, Integer, MetaData, Numeric, String, Table, text from sqlalchemy.dialects.postgresql import ( + ARRAY, BIGINT, DATE, JSON, JSONB, TIME, TIMESTAMP, - ARRAY, ) + +from tap_postgres.tap import TapPostgres from tests.settings import DB_SCHEMA_NAME, DB_SQLALCHEMY_URL from tests.test_replication_key import TABLE_NAME, TapTestReplicationKey from tests.test_selected_columns_only import ( @@ -26,8 +27,6 @@ TapTestSelectedColumnsOnly, ) -from tap_postgres.tap import TapPostgres - SAMPLE_CONFIG = { "start_date": pendulum.datetime(2022, 11, 1).to_iso8601_string(), "sqlalchemy_url": DB_SQLALCHEMY_URL, @@ -45,22 +44,22 @@ def setup_test_table(table_name, sqlalchemy_url): """setup any state specific to the execution of the given module.""" - engine = sqlalchemy.create_engine(sqlalchemy_url, future=True) + engine = sa.create_engine(sqlalchemy_url, future=True) fake = Faker() date1 = datetime.date(2022, 11, 1) date2 = datetime.date(2022, 11, 30) - metadata_obj = MetaData() - test_replication_key_table = Table( + metadata_obj = sa.MetaData() + test_replication_key_table = sa.Table( table_name, metadata_obj, - Column("id", Integer, primary_key=True), - Column("updated_at", DateTime(), nullable=False), - Column("name", String()), + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.Column("name", sa.String()), ) with engine.begin() as conn: metadata_obj.create_all(conn) - conn.execute(text(f"TRUNCATE TABLE {table_name}")) + conn.execute(sa.text(f"TRUNCATE TABLE {table_name}")) for _ in range(1000): insert = test_replication_key_table.insert().values( updated_at=fake.date_between(date1, date2), name=fake.name() @@ -69,9 +68,9 @@ def setup_test_table(table_name, sqlalchemy_url): def teardown_test_table(table_name, sqlalchemy_url): - engine = sqlalchemy.create_engine(sqlalchemy_url, future=True) + engine = sa.create_engine(sqlalchemy_url, future=True) with engine.begin() as conn: - conn.execute(text(f"DROP TABLE {table_name}")) + conn.execute(sa.text(f"DROP TABLE {table_name}")) custom_test_replication_key = suites.TestSuite( @@ -117,7 +116,7 @@ def resource(self): teardown_test_table(self.table_name, self.sqlalchemy_url) -class TestTapPostgres_NOSQLALCHMY(TapPostgresTestNOSQLALCHEMY): +class TestTapPostgres_NOSQLALCHMY(TapPostgresTestNOSQLALCHEMY): # noqa: N801 table_name = TABLE_NAME sqlalchemy_url = SAMPLE_CONFIG["sqlalchemy_url"] @@ -146,15 +145,15 @@ def test_temporal_datatypes(): schema checks, and performs similar tests on times and timestamps. """ table_name = "test_temporal_datatypes" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) + engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("column_date", DATE), - Column("column_time", TIME), - Column("column_timestamp", TIMESTAMP), + sa.Column("column_date", DATE), + sa.Column("column_time", TIME), + sa.Column("column_timestamp", TIMESTAMP), ) with engine.begin() as conn: table.drop(conn, checkfirst=True) @@ -188,12 +187,12 @@ def test_temporal_datatypes(): and schema_message["stream"] == altered_table_name ): assert ( - "date" - == schema_message["schema"]["properties"]["column_date"]["format"] + schema_message["schema"]["properties"]["column_date"]["format"] + == "date" ) assert ( - "date-time" - == schema_message["schema"]["properties"]["column_timestamp"]["format"] + schema_message["schema"]["properties"]["column_timestamp"]["format"] + == "date-time" ) assert test_runner.records[altered_table_name][0] == { "column_date": "2022-03-19", @@ -203,16 +202,16 @@ def test_temporal_datatypes(): def test_jsonb_json(): - """JSONB and JSON Objects weren't being selected, make sure they are now""" + """JSONB and JSON Objects weren't being selected, make sure they are now.""" table_name = "test_jsonb_json" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) + engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("column_jsonb", JSONB), - Column("column_json", JSON), + sa.Column("column_jsonb", JSONB), + sa.Column("column_json", JSON), ) rows = [ @@ -280,13 +279,13 @@ def test_jsonb_json(): def test_jsonb_array(): """ARRAYS of JSONB objects had incorrect schemas. See issue #331.""" table_name = "test_jsonb_array" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) + engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("column_jsonb_array", ARRAY(JSONB)), + sa.Column("column_jsonb_array", ARRAY(JSONB)), ) rows = [ @@ -343,13 +342,13 @@ def test_jsonb_array(): def test_decimal(): """Schema was wrong for Decimal objects. Check they are correctly selected.""" table_name = "test_decimal" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) + engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("column", Numeric()), + sa.Column("column", sa.Numeric()), ) with engine.begin() as conn: table.drop(conn, checkfirst=True) @@ -388,13 +387,18 @@ def test_decimal(): def test_filter_schemas(): """Only return tables from a given schema""" table_name = "test_filter_schemas" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) + engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table(table_name, metadata_obj, Column("id", BIGINT), schema="new_schema") + metadata_obj = sa.MetaData() + table = sa.Table( + table_name, + metadata_obj, + sa.Column("id", BIGINT), + schema="new_schema", + ) with engine.begin() as conn: - conn.execute(text("CREATE SCHEMA IF NOT EXISTS new_schema")) + conn.execute(sa.text("CREATE SCHEMA IF NOT EXISTS new_schema")) table.drop(conn, checkfirst=True) metadata_obj.create_all(conn) filter_schemas_config = copy.deepcopy(SAMPLE_CONFIG) @@ -409,8 +413,8 @@ def test_filter_schemas(): class PostgresTestRunner(TapTestRunner): def run_sync_dry_run(self) -> bool: - """ - Dislike this function and how TestRunner does this so just hacking it here. + """Dislike this function and how TestRunner does this so just hacking it here. + Want to be able to run exactly the catalog given """ new_tap = self.new_tap() @@ -418,22 +422,21 @@ def run_sync_dry_run(self) -> bool: return True -def test_invalid_python_dates(): - """Some dates are invalid in python, but valid in Postgres +def test_invalid_python_dates(): # noqa: PLR0912 + """Some dates are invalid in python, but valid in Postgres. Check out https://www.psycopg.org/psycopg3/docs/advanced/adapt.html#example-handling-infinity-date for more information. - """ table_name = "test_invalid_python_dates" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) + engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("date", DATE), - Column("datetime", DateTime), + sa.Column("date", DATE), + sa.Column("datetime", sa.DateTime), ) with engine.begin() as conn: table.drop(conn, checkfirst=True) diff --git a/tests/test_log_based.py b/tests/test_log_based.py index 0048c0a..51075a7 100644 --- a/tests/test_log_based.py +++ b/tests/test_log_based.py @@ -1,12 +1,11 @@ import json -import sqlalchemy -from sqlalchemy import Column, MetaData, Table +import sqlalchemy as sa from sqlalchemy.dialects.postgresql import BIGINT, TEXT + from tap_postgres.tap import TapPostgres from tests.test_core import PostgresTestRunner - LOG_BASED_CONFIG = { "host": "localhost", "port": 5434, @@ -15,6 +14,7 @@ "database": "postgres", } + def test_null_append(): """LOG_BASED syncs failed with string property types. (issue #294). @@ -23,14 +23,14 @@ def test_null_append(): LOG_BASED replication can still append the "null" option to a property's type. """ table_name = "test_null_append" - engine = sqlalchemy.create_engine("postgresql://postgres:postgres@localhost:5434/postgres") + engine = sa.create_engine("postgresql://postgres:postgres@localhost:5434/postgres") - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("id", BIGINT, primary_key = True), - Column("data", TEXT, nullable = True) + sa.Column("id", BIGINT, primary_key=True), + sa.Column("data", TEXT, nullable=True), ) with engine.connect() as conn: table.drop(conn, checkfirst=True) diff --git a/tests/test_replication_key.py b/tests/test_replication_key.py index eb533b4..7f46f3b 100644 --- a/tests/test_replication_key.py +++ b/tests/test_replication_key.py @@ -4,14 +4,13 @@ import json import pendulum -import sqlalchemy +import sqlalchemy as sa from singer_sdk.testing.runners import TapTestRunner from singer_sdk.testing.templates import TapTestTemplate -from sqlalchemy import Column, MetaData, String, Table from sqlalchemy.dialects.postgresql import TIMESTAMP -from tests.settings import DB_SCHEMA_NAME, DB_SQLALCHEMY_URL from tap_postgres.tap import TapPostgres +from tests.settings import DB_SCHEMA_NAME, DB_SQLALCHEMY_URL TABLE_NAME = "test_replication_key" SAMPLE_CONFIG = { @@ -43,7 +42,7 @@ def replication_key_test(tap, table_name): # Handy for debugging # with open('data.json', 'w', encoding='utf-8') as f: - # json.dump(tap_catalog, f, indent=4) + # json.dump(tap_catalog, f, indent=4) # noqa: ERA001 tap = TapPostgres(config=SAMPLE_CONFIG, catalog=tap_catalog) tap.sync_all() @@ -56,14 +55,14 @@ def test_null_replication_key_with_start_date(): greater than the start date should be synced. """ table_name = "test_null_replication_key_with_start_date" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) + engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("data", String()), - Column("updated_at", TIMESTAMP), + sa.Column("data", sa.String()), + sa.Column("updated_at", TIMESTAMP), ) with engine.begin() as conn: table.drop(conn, checkfirst=True) @@ -112,14 +111,14 @@ def test_null_replication_key_without_start_date(): modified_config = copy.deepcopy(SAMPLE_CONFIG) modified_config["start_date"] = None - engine = sqlalchemy.create_engine(modified_config["sqlalchemy_url"], future=True) + engine = sa.create_engine(modified_config["sqlalchemy_url"], future=True) - metadata_obj = MetaData() - table = Table( + metadata_obj = sa.MetaData() + table = sa.Table( table_name, metadata_obj, - Column("data", String()), - Column("updated_at", TIMESTAMP), + sa.Column("data", sa.String()), + sa.Column("updated_at", TIMESTAMP), ) with engine.begin() as conn: table.drop(conn, checkfirst=True) @@ -155,7 +154,9 @@ def test_null_replication_key_without_start_date(): catalog=tap_catalog, ) test_runner.sync_all() - assert len(test_runner.records[altered_table_name]) == 3 # All three records. + assert ( + len(test_runner.records[altered_table_name]) == 3 # noqa: PLR2004 + ) # All three records. class TapTestReplicationKey(TapTestTemplate): diff --git a/tests/test_selected_columns_only.py b/tests/test_selected_columns_only.py index 0f964b8..c288656 100644 --- a/tests/test_selected_columns_only.py +++ b/tests/test_selected_columns_only.py @@ -2,9 +2,9 @@ import json from singer_sdk.testing.templates import TapTestTemplate -from tests.settings import DB_SQLALCHEMY_URL from tap_postgres.tap import TapPostgres +from tests.settings import DB_SQLALCHEMY_URL TABLE_NAME_SELECTED_COLUMNS_ONLY = "test_selected_columns_only" SAMPLE_CONFIG = { @@ -24,16 +24,18 @@ def selected_columns_only_test(tap, table_name): else: for metadata in stream["metadata"]: metadata["metadata"]["selected"] = True - if metadata["breadcrumb"] != []: - if metadata["breadcrumb"][1] == column_to_exclude: - metadata["metadata"]["selected"] = False + if ( + metadata["breadcrumb"] != [] + and metadata["breadcrumb"][1] == column_to_exclude + ): + metadata["metadata"]["selected"] = False tap = TapPostgres(config=SAMPLE_CONFIG, catalog=tap_catalog) streams = tap.discover_streams() - selected_stream = [s for s in streams if s.selected is True][0] + selected_stream = next(s for s in streams if s.selected is True) for row in selected_stream.get_records(context=None): - assert column_to_exclude not in row.keys() + assert column_to_exclude not in row class TapTestSelectedColumnsOnly(TapTestTemplate): diff --git a/tox.ini b/tox.ini index ccbf3b2..75c4972 100644 --- a/tox.ini +++ b/tox.ini @@ -8,18 +8,10 @@ isolated_build = true [testenv] allowlist_externals = poetry -commands = - poetry install -v - poetry run pytest - poetry run black --check tap_postgres/ - poetry run flake8 tap_postgres - poetry run pydocstyle tap_postgres - poetry run mypy tap_postgres --exclude='tap_postgres/tests' - [testenv:pytest] # Run the python tests. # To execute, run `tox -e pytest` -envlist = py37, py38, py39 +envlist = py38, py39, py310, py311, py312 commands = poetry install -v poetry run pytest @@ -29,25 +21,14 @@ commands = # To execute, run `tox -e format` commands = poetry install -v - poetry run black tap_postgres/ - poetry run isort tap_postgres + poetry run ruff check target_postgres/ + poetry run ruff format target_postgres/ [testenv:lint] # Raise an error if lint and style standards are not met. # To execute, run `tox -e lint` commands = poetry install -v - poetry run black --check --diff tap_postgres/ - poetry run isort --check tap_postgres - poetry run flake8 tap_postgres - poetry run pydocstyle tap_postgres - # refer to pyproject.toml for specific settings - # poetry run mypy . - -[flake8] -ignore = W503 -max-line-length = 88 -max-complexity = 10 - -[pydocstyle] -convention = google + poetry run ruff check --diff target_postgres/ + poetry run ruff format --check target_postgres/ + poetry run mypy . From b2ec55835fa001efb6cd363328431360eacde196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Fri, 26 Jan 2024 10:49:04 -0600 Subject: [PATCH 2/3] target_postgres -> tap_postgres --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 75c4972..fdf5efa 100644 --- a/tox.ini +++ b/tox.ini @@ -21,14 +21,14 @@ commands = # To execute, run `tox -e format` commands = poetry install -v - poetry run ruff check target_postgres/ - poetry run ruff format target_postgres/ + poetry run ruff check tap_postgres/ + poetry run ruff format tap_postgres/ [testenv:lint] # Raise an error if lint and style standards are not met. # To execute, run `tox -e lint` commands = poetry install -v - poetry run ruff check --diff target_postgres/ - poetry run ruff format --check target_postgres/ + poetry run ruff check --diff tap_postgres/ + poetry run ruff format --check tap_postgres/ poetry run mypy . From 3dfe717595b5083b841e0d35c9e14e508bffbb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Fri, 26 Jan 2024 10:49:57 -0600 Subject: [PATCH 3/3] Add job concurrency --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8d1d5e..aead0a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,10 @@ on: workflow_dispatch: inputs: {} +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: tests: runs-on: ubuntu-latest