diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
index ee12485bf..ffd2e5d7d 100644
--- a/.github/ISSUE_TEMPLATE/bug.yml
+++ b/.github/ISSUE_TEMPLATE/bug.yml
@@ -1,7 +1,9 @@
name: Bug Report
description: File a bug report
title: "bug:
"
-labels: ["kind/Bug", "valuestream/SDK"]
+type: Fix
+assignees:
+ - edgarrmondragon
body:
- type: markdown
@@ -13,7 +15,7 @@ body:
attributes:
label: Singer SDK Version
description: Version of the library you are using
- placeholder: "0.39.1"
+ placeholder: "0.42.1"
validations:
required: true
- type: checkboxes
@@ -29,14 +31,13 @@ body:
label: Python Version
description: Version of Python you are using
options:
- - "3.6 (EOL)"
- - "3.7 (EOL)"
- - "3.8"
- - "3.9"
- - "3.10"
- - "3.11"
- - "3.12"
- "NA"
+ - "3.13"
+ - "3.12"
+ - "3.11"
+ - "3.10"
+ - "3.9"
+ - "3.8 or earlier"
validations:
required: true
- type: dropdown
@@ -61,7 +62,7 @@ body:
description: What operating system you are using
placeholder: "Windows"
validations:
- required: true
+ required: false
- type: textarea
id: what-happened
attributes:
@@ -78,3 +79,9 @@ body:
render: Python
validations:
required: false
+ - type: input
+ id: slack_or_linen
+ attributes:
+ label: Link to Slack/Linen
+ description: Provide a link to the Slack or Linen conversation, if applicable
+ placeholder: "https://..."
diff --git a/.github/ISSUE_TEMPLATE/docs.yml b/.github/ISSUE_TEMPLATE/docs.yml
index ca0d9c673..b47a9909d 100644
--- a/.github/ISSUE_TEMPLATE/docs.yml
+++ b/.github/ISSUE_TEMPLATE/docs.yml
@@ -1,7 +1,10 @@
name: Documentation change
description: Request a documentation change
title: "docs: "
+type: Docs
labels: ["Documentation", "valuestream/SDK"]
+assignees:
+ - edgarrmondragon
body:
- type: markdown
diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml
index a5afd5974..bf435a43d 100644
--- a/.github/ISSUE_TEMPLATE/feature.yml
+++ b/.github/ISSUE_TEMPLATE/feature.yml
@@ -2,8 +2,9 @@ name: Feature request
description: Request a new feature
title: "feat: "
labels: ["kind/Feature", "valuestream/SDK"]
+type: Feat
assignees:
- - meltano/engineering
+ - edgarrmondragon
body:
- type: markdown
@@ -16,7 +17,8 @@ body:
label: Feature scope
description: Functionality this new feature would impact
options:
- - Taps (catalog, state, stream maps, tests, etc.)
+ - Taps (catalog, state, tests, etc.)
+ - Inline mapping (stream maps, flattening, etc.)
- Targets (data type handling, batching, SQL object generation, tests, etc.)
- Configuration (settings parsing, validation, etc.)
- CLI (options, error messages, logging, etc.)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 4b12bfa9b..8625a873a 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -15,6 +15,7 @@ updates:
update-types:
- "minor"
- "patch"
+ versioning-strategy: increase-if-necessary
- package-ecosystem: pip
directory: "/.github/workflows"
schedule:
diff --git a/.github/workflows/api-changes.yml b/.github/workflows/api-changes.yml
index c758517c4..108d5b6f0 100644
--- a/.github/workflows/api-changes.yml
+++ b/.github/workflows/api-changes.yml
@@ -30,7 +30,7 @@ jobs:
- name: Setup Python
uses: actions/setup-python@v5
with:
- python-version: 3.12
+ python-version: 3.x
- name: Install tools
env:
@@ -38,7 +38,6 @@ jobs:
run: |
python -Im pip install -U pip
pipx install griffe nox
- pipx inject nox nox-poetry
pipx list
- name: Set REF
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 7f3194489..99f946341 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -8,7 +8,9 @@ name: "CodeQL"
on:
push:
- branches: [ "main" ]
+ branches:
+ - "main"
+ - "v*"
paths:
- .github/workflows/codeql-analysis.yml
- '**.py' # Any Python file
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
index 122e9d090..23d2f79eb 100644
--- a/.github/workflows/codspeed.yml
+++ b/.github/workflows/codspeed.yml
@@ -4,6 +4,7 @@ on:
push:
branches:
- "main"
+ - "v*"
paths:
- "singer_sdk/**"
- "tests/**"
@@ -30,7 +31,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
- python-version: 3.12
+ python-version: 3.x
architecture: x64
- name: Install poetry
diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
index e6615dcc3..9c097dc1a 100644
--- a/.github/workflows/constraints.txt
+++ b/.github/workflows/constraints.txt
@@ -1,8 +1,5 @@
-griffe==0.48.0
-pip==24.2
-poetry==1.8.3
-poetry-plugin-export==1.8.0
-poetry-dynamic-versioning==1.4.0
-pre-commit==3.8.0
-nox==2024.4.15
-nox-poetry==1.0.3
+griffe~=1.5
+pip==24.3.1
+poetry==1.8.4
+pre-commit==4.0.1
+nox==2024.10.9
diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml
index 051d21d95..90454b8dd 100644
--- a/.github/workflows/cookiecutter-e2e.yml
+++ b/.github/workflows/cookiecutter-e2e.yml
@@ -8,7 +8,9 @@ on:
- "e2e-tests/cookiecutters/**"
- ".github/workflows/cookiecutter-e2e.yml"
push:
- branches: [main]
+ branches:
+ - main
+ - v*
paths:
- "cookiecutter/**"
- "e2e-tests/cookiecutters/**"
@@ -24,14 +26,8 @@ env:
jobs:
lint:
- name: Cookiecutter E2E Python ${{ matrix.python-version }} / ${{ matrix.os }}
- runs-on: ${{ matrix.os }}
- strategy:
- fail-fast: true
- matrix:
- include:
- - { python-version: "3.12", os: "ubuntu-latest" }
-
+ name: Cookiecutter E2E Python
+ runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Upgrade pip
@@ -46,38 +42,36 @@ jobs:
PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
run: |
pipx install poetry
- pipx inject poetry poetry-plugin-export
poetry --version
- poetry self show plugins
- uses: actions/setup-python@v5
with:
- python-version: ${{ matrix.python-version }}
- architecture: x64
- cache: 'pip'
- cache-dependency-path: 'poetry.lock'
+ python-version: 3.x
+
+ - uses: astral-sh/setup-uv@v4
+ with:
+ version: ">=0.4.30"
- name: Install pre-commit
run: |
- pipx install pre-commit
+ uv tool install --with=pre-commit-uv pre-commit
pre-commit --version
- name: Install Nox
env:
PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
run: |
- pipx install nox
- pipx inject nox nox-poetry
+ uv tool install nox
nox --version
- name: Run Nox
run: |
- nox --python=${{ matrix.python-version }} --session=test_cookiecutter
+ nox --session=test_cookiecutter
- uses: actions/upload-artifact@v4
if: always()
with:
- name: cookiecutter-${{ matrix.os }}-py${{ matrix.python-version }}
+ name: cookiecutter-ubuntu-latest-py3x
path: |
/tmp/tap-*
/tmp/target-*
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 712f2e3db..1cd706cb6 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -5,6 +5,7 @@ on:
jobs:
build:
+ name: Build artifacts
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -12,8 +13,44 @@ jobs:
fetch-depth: 0
- uses: hynek/build-and-inspect-python-package@v2
+ check-tag:
+ name: Check tag
+ runs-on: ubuntu-latest
+ if: startsWith(github.ref, 'refs/tags/')
+ outputs:
+ is_final: ${{ steps.check.outputs.is_final }}
+ steps:
+ - name: Check if tag is a pre-release
+ id: check
+ run: |
+ echo "is_final=$(echo '${{ github.ref }}' | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+$' && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT
+
+ provenance:
+ name: Provenance
+ runs-on: ubuntu-latest
+ needs: [build]
+ if: startsWith(github.ref, 'refs/tags/')
+ permissions:
+ id-token: write # Needed for attestations
+ attestations: write # Needed for attestations
+ outputs:
+ bundle-path: ${{ steps.attest.outputs.bundle-path }}
+ steps:
+ - uses: actions/download-artifact@v4
+ with:
+ name: Packages
+ path: dist
+ - uses: actions/attest-build-provenance@v1
+ id: attest
+ with:
+ subject-path: "./dist/singer_sdk*"
+ - uses: actions/upload-artifact@v4
+ with:
+ name: Attestations
+ path: ${{ steps.attest.outputs.bundle-path }}
+
publish:
- name: Publish to PyPI
+ name: PyPI
runs-on: ubuntu-latest
needs: [build]
environment:
@@ -28,23 +65,27 @@ jobs:
name: Packages
path: dist
- name: Publish
- uses: pypa/gh-action-pypi-publish@v1.9.0
+ uses: pypa/gh-action-pypi-publish@v1.12.2
upload-to-release:
name: Upload files to release
runs-on: ubuntu-latest
- needs: [build]
- if: startsWith(github.ref, 'refs/tags/')
+ needs: [build, check-tag, provenance]
+ if: ${{ startsWith(github.ref, 'refs/tags/') && needs.check-tag.outputs.is_final == 'true' }}
permissions:
contents: write # Needed for uploading files to the release
- id-token: write # Needed for attestations
- attestations: write # Needed for attestations
steps:
- uses: actions/download-artifact@v4
with:
name: Packages
path: dist
+
+ - uses: actions/download-artifact@v4
+ with:
+ name: Attestations
+ path: attestations
+
- name: Upload wheel and sdist to release
uses: svenstaro/upload-release-action@v2
with:
@@ -52,14 +93,11 @@ jobs:
tag: ${{ github.ref }}
overwrite: true
file_glob: true
- - uses: actions/attest-build-provenance@v1
- id: attest
- with:
- subject-path: "./dist/singer_sdk*"
+
- name: Upload attestations to release
uses: svenstaro/upload-release-action@v2
with:
- file: ${{ steps.attest.outputs.bundle-path }}
+ file: attestations/attestation.jsonl
tag: ${{ github.ref }}
overwrite: true
asset_name: attestations.intoto.jsonl
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3acc509fb..9f946d098 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,9 @@ on:
- ".github/workflows/test.yml"
- ".github/workflows/constraints.txt"
push:
- branches: [main]
+ branches:
+ - main
+ - v*
paths:
- "cookiecutter/**"
- "samples/**"
@@ -37,7 +39,7 @@ env:
jobs:
tests:
- name: "Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }} / SQLAlchemy: ${{ matrix.sqlalchemy }}"
+ name: "Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }}"
runs-on: ${{ matrix.os }}
continue-on-error: true
env:
@@ -48,32 +50,26 @@ jobs:
matrix:
session: [tests]
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
- sqlalchemy: ["2"]
+ python-version:
+ - "3.9"
+ - "3.10"
+ - "3.11"
+ - "3.12"
+ - "3.13"
include:
- - { session: tests, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "1" }
- - { session: doctest, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" }
- - { session: mypy, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" }
- - { session: deps, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" }
+ - { session: doctest, python-version: "3.13", os: "ubuntu-latest" }
+ - { session: mypy, python-version: "3.13", os: "ubuntu-latest" }
+ - { session: deps, python-version: "3.13", os: "ubuntu-latest" }
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- - name: Install Poetry
- env:
- PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
- run: |
- pipx install poetry
- pipx inject poetry poetry-plugin-export
- pipx inject poetry poetry-dynamic-versioning[plugin]
- poetry --version
- poetry self show plugins
-
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
+ allow-prereleases: true
- name: Upgrade pip
env:
@@ -87,19 +83,26 @@ jobs:
PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
run: |
pipx install 'nox[uv]'
- pipx inject nox nox-poetry
nox --version
+ - uses: actions/cache@v4
+ if: matrix.session == 'tests'
+ with:
+ path: http_cache.sqlite
+ key: http_cache-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.sqlalchemy }}
+
- name: Run Nox
env:
- SQLALCHEMY_VERSION: ${{ matrix.sqlalchemy }}
+ PIP_PRE: "1"
+ UV_PRERELEASE: allow
run: |
nox --verbose
- uses: actions/upload-artifact@v4
if: always() && (matrix.session == 'tests')
with:
- name: coverage-data-nox_${{ matrix.session }}-${{ matrix.os }}-py${{ matrix.python-version }}_sqlalchemy_${{ matrix.sqlalchemy }}
+ include-hidden-files: true
+ name: coverage-data-nox_-${{ matrix.os }}-py${{ matrix.python-version }}_sqlalchemy_${{ matrix.sqlalchemy }}
path: ".coverage.*"
tests-external:
@@ -119,16 +122,6 @@ jobs:
with:
fetch-depth: 0
- - name: Install Poetry
- env:
- PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
- run: |
- pipx install poetry
- pipx inject poetry poetry-plugin-export
- pipx inject poetry poetry-dynamic-versioning[plugin]
- poetry --version
- poetry self show plugins
-
- uses: actions/setup-python@v5
with:
python-version: ${{ env.NOXPYTHON }}
@@ -145,10 +138,12 @@ jobs:
PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
run: |
pipx install 'nox[uv]'
- pipx inject nox nox-poetry
nox --version
- name: Run Nox
+ env:
+ PIP_PRE: "1"
+ UV_PRERELEASE: allow
run: |
nox -- -m "external"
@@ -160,19 +155,9 @@ jobs:
NOXSESSION: coverage
steps:
- uses: actions/checkout@v4
-
- - name: Install Poetry
- env:
- PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
- run: |
- pipx install poetry
- pipx inject poetry poetry-plugin-export
- poetry --version
- poetry self show plugins
-
- uses: actions/setup-python@v5
with:
- python-version: '3.12'
+ python-version: '3.x'
- name: Upgrade pip
env:
@@ -191,7 +176,6 @@ jobs:
PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
run: |
pipx install 'nox[uv]'
- pipx inject nox nox-poetry
nox --version
- run: nox --install-only
@@ -205,7 +189,7 @@ jobs:
run: |
nox -r --no-install -- xml
- - uses: codecov/codecov-action@v4
+ - uses: codecov/codecov-action@v5
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
index 1a7bbc872..ab81bb067 100644
--- a/.github/workflows/version_bump.yml
+++ b/.github/workflows/version_bump.yml
@@ -6,6 +6,7 @@ on:
dry_run:
description: "Run the action without creating a PR or release draft"
required: true
+ default: false
type: boolean
bump:
description: "Version bump type"
@@ -46,12 +47,11 @@ jobs:
- uses: actions/setup-python@v5
with:
- python-version: "3.12"
- architecture: x64
+ python-version: "3.x"
- name: Bump version
id: cz-bump
- uses: commitizen-tools/commitizen-action@0.21.0
+ uses: commitizen-tools/commitizen-action@0.22.0
with:
increment: ${{ github.event.inputs.bump != 'auto' && github.event.inputs.bump || '' }}
prerelease: ${{ github.event.inputs.prerelease != 'none' && github.event.inputs.prerelease || '' }}
@@ -84,7 +84,7 @@ jobs:
- name: Create Pull Request
if: ${{ github.event.inputs.dry_run == 'false' }}
- uses: peter-evans/create-pull-request@v6
+ uses: peter-evans/create-pull-request@v7
id: create-pull-request
with:
token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}
@@ -101,6 +101,5 @@ jobs:
[Release Draft](${{ steps.draft-release.outputs.url }})
branch: release/v${{ steps.cz-bump.outputs.version }}
- base: main
labels: release
assignees: "${{ github.actor }}"
diff --git a/.gitignore b/.gitignore
index d6a92f65e..e10d0f06a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
+# HTTP cache
+http_cache.sqlite
+
# Local Poetry configuration file
poetry.toml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e3bfd2a9f..12ea90545 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ ci:
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.6.0
+ rev: v5.0.0
hooks:
- id: check-json
exclude: |
@@ -35,22 +35,16 @@ repos:
tests/snapshots/.*
)$
- id: trailing-whitespace
- exclude: |
- (?x)^(
- .bumpversion.cfg|
- singer_sdk/helpers/_simpleeval.py|
- tests/core/test_simpleeval.py
- )$
- repo: https://github.com/python-jsonschema/check-jsonschema
- rev: 0.29.1
+ rev: 0.29.4
hooks:
- id: check-dependabot
- id: check-github-workflows
- id: check-readthedocs
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.5.7
+ rev: v0.8.0
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix, --show-fixes]
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 3b35af1b7..19438e79e 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,7 +1,7 @@
version: 2
build:
- os: ubuntu-22.04
+ os: ubuntu-24.04
tools:
python: "3.12"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 27d538a6d..78d17d1e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,117 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## v0.42.1 (2024-11-11)
+
+### 🐛 Fixes
+
+- [#2756](https://github.com/meltano/sdk/issues/2756) Safely compare UUID replication keys with state bookmarks -- _**Thanks @nikzavada!**_
+
+## v0.42.0 (2024-11-11)
+
+### ✨ New
+
+- [#2742](https://github.com/meltano/sdk/issues/2742) Update dependencies in templates
+- [#2732](https://github.com/meltano/sdk/issues/2732) SQL target developers can now more easily override the mapping from JSON schema to SQL column type
+- [#2730](https://github.com/meltano/sdk/issues/2730) Added `SQLConnector.prepare_primary_key` for target to implement for custom table primary key adaptation
+- [#2488](https://github.com/meltano/sdk/issues/2488) Nested schema properties can now be defined as nullable
+- [#2518](https://github.com/meltano/sdk/issues/2518) Python 3.13 is officially supported
+- [#2637](https://github.com/meltano/sdk/issues/2637) Environment variables are now parsed for boolean, integer, array and object setting values
+- [#2699](https://github.com/meltano/sdk/issues/2699) Stream name can now be accessed in stream maps -- _**Thanks @holly-evans!**_
+- [#2712](https://github.com/meltano/sdk/issues/2712) JSON schema `title` is now supported in configuration and stream properties
+- [#2707](https://github.com/meltano/sdk/issues/2707) Bumped simpleeval to 1.0
+- [#2701](https://github.com/meltano/sdk/issues/2701) Stream name can now be accessed in `__alias__` context of stream maps -- _**Thanks @holly-evans!**_
+
+### 🐛 Fixes
+
+- [#2741](https://github.com/meltano/sdk/issues/2741) `datetime.datetime` instances in stream maps are now correctly mapped to `date-time` JSON schema strings
+- [#2727](https://github.com/meltano/sdk/issues/2727) Object and array JSON types are now handled before primitive types when converting them to SQL types
+- [#2723](https://github.com/meltano/sdk/issues/2723) JSON schema union types are no longer conformed into boolean values
+
+### ⚙️ Under the Hood
+
+- [#2743](https://github.com/meltano/sdk/issues/2743) Deprecate passing file paths to plugin and stream initialization
+
+### 📚 Documentation Improvements
+
+- [#2745](https://github.com/meltano/sdk/issues/2745) Document the current release process
+- [#2717](https://github.com/meltano/sdk/issues/2717) Update Meltano commands in examples
+
+### 📦 Packaging changes
+
+- [#2736](https://github.com/meltano/sdk/issues/2736) Skip `simpleeval` 1.0.1
+- [#2716](https://github.com/meltano/sdk/issues/2716) Stopped testing with SQLAlchemy 1.4
+- [#2714](https://github.com/meltano/sdk/issues/2714) Remove constraint on `urllib3`
+
+## v0.41.0 (2024-10-02)
+
+### ✨ New
+
+- [#2667](https://github.com/meltano/sdk/issues/2667) Support stream aliasing of `BATCH` messages via stream maps -- _**Thanks @ReubenFrankel!**_
+- [#2651](https://github.com/meltano/sdk/issues/2651) SQL taps now emit schemas with `maxLength` when applicable
+- [#2618](https://github.com/meltano/sdk/issues/2618) Developers can now more easily override the mapping from SQL column type to JSON schema
+
+### 🐛 Fixes
+
+- [#2697](https://github.com/meltano/sdk/issues/2697) All HTTP timeout exceptions are now retried in REST and GraphQL streams
+- [#2683](https://github.com/meltano/sdk/issues/2683) A clear error message is now emitted when flattening is enabled but `flattening_max_depth` is not set
+- [#2665](https://github.com/meltano/sdk/issues/2665) Mapped datetime values are now typed as `date-time` strings in the schema message -- _**Thanks @gregkoutsimp!**_
+- [#2663](https://github.com/meltano/sdk/issues/2663) Properties dropped using `None` or `__NULL__` in stream maps are now also removed from the schema `required` array
+
+### ⚙️ Under the Hood
+
+- [#2696](https://github.com/meltano/sdk/issues/2696) Use tox without installing Poetry explicitly in workflows
+- [#2654](https://github.com/meltano/sdk/issues/2654) Added a generic `FileStream` (still in active development!)
+- [#2695](https://github.com/meltano/sdk/issues/2695) Update dependencies in templates
+- [#2661](https://github.com/meltano/sdk/issues/2661) Drop support for Python 3.8 in templates
+- [#2670](https://github.com/meltano/sdk/issues/2670) Deprecated `Faker` class in stream maps
+- [#2666](https://github.com/meltano/sdk/issues/2666) Remove non-functional `record-flattening` capability -- _**Thanks @ReubenFrankel!**_
+- [#2652](https://github.com/meltano/sdk/issues/2652) Renamed `SQLConnector.type_mapping` to `SQLConnector.sql_to_jsonschema`
+- [#2647](https://github.com/meltano/sdk/issues/2647) Use future `warnings.deprecated` decorator
+
+### 📚 Documentation Improvements
+
+- [#2658](https://github.com/meltano/sdk/issues/2658) Added more versions when stuff changed or was added
+
+### 📦 Packaging changes
+
+- [#2694](https://github.com/meltano/sdk/issues/2694) Removed unused backport `importlib_resources` dependency in tap template
+- [#2664](https://github.com/meltano/sdk/issues/2664) Added a constraint on setuptools <= 70.3.0 to fix incompatibility with some dependencies
+
+## v0.40.0 (2024-09-02)
+
+### ✨ New
+
+- [#2486](https://github.com/meltano/sdk/issues/2486) Emit target metrics
+- [#2567](https://github.com/meltano/sdk/issues/2567) A new `schema_is_valid` built-in tap test validates stream schemas against the JSON Schema specification
+- [#2598](https://github.com/meltano/sdk/issues/2598) Stream map expressions now have access to the `Faker` class, rather than just a faker instance
+- [#2549](https://github.com/meltano/sdk/issues/2549) Added a default user agent for REST and GraphQL taps
+
+### 🐛 Fixes
+
+- [#2613](https://github.com/meltano/sdk/issues/2613) Mismatch between timezone-aware and naive datetimes in start date and bookmarks is now correctly handled
+
+### ⚙️ Under the Hood
+
+- [#2628](https://github.com/meltano/sdk/issues/2628) Use context manager to read gzip batch files
+- [#2619](https://github.com/meltano/sdk/issues/2619) Default to UTC when parsing dates without a known timezone
+- [#2603](https://github.com/meltano/sdk/issues/2603) Backwards-compatible identifier quoting in fully qualified names
+- [#2601](https://github.com/meltano/sdk/issues/2601) Improved SQL identifier (de)normalization
+- [#2599](https://github.com/meltano/sdk/issues/2599) Remove `pytest-durations` dependency from `testing` and use native pytest option `--durations`
+- [#2597](https://github.com/meltano/sdk/issues/2597) Mark pagination classes with `@override` decorator
+- [#2596](https://github.com/meltano/sdk/issues/2596) Made `auth_headers` and `auth_params` of `APIAuthenticatorBase` simple instance attributes instead of decorated properties
+
+### 📚 Documentation Improvements
+
+- [#2639](https://github.com/meltano/sdk/issues/2639) Documented versions where `fake` and `Faker` objects were added to the stream maps context
+- [#2629](https://github.com/meltano/sdk/issues/2629) Reference `get_starting_timestamp` in incremental replication guide
+- [#2604](https://github.com/meltano/sdk/issues/2604) Update project sample links
+- [#2595](https://github.com/meltano/sdk/issues/2595) Documented examples of stream glob expressions and property aliasing
+
+### 📦 Packaging changes
+
+- [#2640](https://github.com/meltano/sdk/issues/2640) Remove upper constraint on `faker` extra
+
## v0.39.1 (2024-08-07)
### 🐛 Fixes
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/dependabot.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/dependabot.yml
index 0660ffdd4..d96600409 100644
--- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/dependabot.yml
+++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/dependabot.yml
@@ -19,6 +19,7 @@ updates:
dependency-type: production
update-types:
- "patch"
+ versioning-strategy: increase-if-necessary
- package-ecosystem: pip
directory: "/.github/workflows"
schedule:
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml
index f9503baf6..179acf18a 100644
--- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml
+++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml
@@ -42,4 +42,4 @@ jobs:
- name: Publish
## TODO: create a trusted publisher on PyPI
## https://docs.pypi.org/trusted-publishers/
- uses: pypa/gh-action-pypi-publish@v1.9.0
+ uses: pypa/gh-action-pypi-publish@v1.12.2
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml
index 2bbeebf75..4acc3b57b 100644
--- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml
+++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml
@@ -3,30 +3,49 @@
name: Test {{cookiecutter.mapper_id}}
-on: [push]
+on:
+ push:
+ branches: [main]
+ paths:
+ - .github/workflows/test.yml
+ - {{ cookiecutter.library_name }}/**
+ - tests/**
+ - poetry.lock
+ - pyproject.toml
+ - tox.ini
+ pull_request:
+ branches: [main]
+ paths:
+ - .github/workflows/test.yml
+ - {{ cookiecutter.library_name }}/**
+ - tests/**
+ - poetry.lock
+ - pyproject.toml
+ - tox.ini
+ workflow_dispatch:
+
+env:
+ FORCE_COLOR: 1
jobs:
pytest:
runs-on: ubuntu-latest
- env:
- GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
strategy:
fail-fast: false
matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version:
+ - "3.9"
+ - "3.10"
+ - "3.11"
+ - "3.12"
+ - "3.13"
steps:
- uses: actions/checkout@v4
- name: Set up Python {{ '${{ matrix.python-version }}' }}
uses: actions/setup-python@v5
with:
python-version: {{ '${{ matrix.python-version }}' }}
- - name: Install Poetry
- run: |
- pip install poetry
- - name: Install dependencies
- run: |
- poetry env use {{ '${{ matrix.python-version }}' }}
- poetry install
- - name: Test with pytest
+ - run: pipx install tox
+ - name: Run Tox
run: |
- poetry run pytest
+ tox -e $(echo py{{ '${{ matrix.python-version }}' }} | tr -d .)
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
index 29308ec2d..e456b9896 100644
--- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
+++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ ci:
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.6.0
+ rev: v5.0.0
hooks:
- id: check-json
exclude: |
@@ -18,19 +18,19 @@ repos:
- id: trailing-whitespace
- repo: https://github.com/python-jsonschema/check-jsonschema
- rev: 0.29.1
+ rev: 0.29.4
hooks:
- id: check-dependabot
- id: check-github-workflows
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.5.6
+ rev: v0.7.3
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix, --show-fixes]
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.11.1
+ rev: v1.13.0
hooks:
- id: mypy
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
index e77cf5346..f09be6d50 100644
--- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
+++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -16,11 +16,11 @@ keywords = [
classifiers = [
"Intended Audience :: Developers",
"Operating System :: OS Independent",
- "Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
]
license = "Apache-2.0"
{%- if cookiecutter.variant != "None (Skip)" %}
@@ -30,13 +30,13 @@ packages = [
{%- endif %}
[tool.poetry.dependencies]
-python = ">=3.8"
-singer-sdk = { version="~=0.39.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+python = ">=3.9"
+singer-sdk = { version="~=0.42.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
fs-s3fs = { version = "~=1.1.1", optional = true }
[tool.poetry.group.dev.dependencies]
pytest = ">=8"
-singer-sdk = { version="~=0.39.1", extras = ["testing"] }
+singer-sdk = { version="~=0.42.1", extras = ["testing"] }
[tool.poetry.extras]
s3 = ["fs-s3fs"]
@@ -49,8 +49,7 @@ python_version = "3.12"
warn_unused_configs = true
[tool.ruff]
-src = ["{{cookiecutter.library_name}}"]
-target-version = "py38"
+target-version = "py39"
[tool.ruff.lint]
ignore = [
@@ -64,14 +63,11 @@ select = ["ALL"]
[tool.ruff.lint.flake8-annotations]
allow-star-arg-any = true
-[tool.ruff.lint.isort]
-known-first-party = ["{{cookiecutter.library_name}}"]
-
[tool.ruff.lint.pydocstyle]
convention = "google"
[build-system]
-requires = ["poetry-core==1.9.0"]
+requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.poetry.scripts]
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini
index 6be1c116a..90c122a54 100644
--- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini
+++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini
@@ -1,19 +1,12 @@
# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
[tox]
-envlist = py{38,39,310,311,312}
-isolated_build = true
+envlist = py3{9,10,11,12,13}
+requires =
+ tox>=4.19
[testenv]
-allowlist_externals = poetry
+deps =
+ pytest
commands =
- poetry install -v
- poetry run pytest
-
-[testenv:pytest]
-# Run the python tests.
-# To execute, run `tox -e pytest`
-envlist = py{38,39,310,311,312}
-commands =
- poetry install -v
- poetry run pytest
+ pytest {posargs}
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py
index c8c3d23ec..af6406201 100644
--- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py
+++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py
@@ -3,14 +3,13 @@
from __future__ import annotations
import typing as t
-from typing import TYPE_CHECKING
import singer_sdk.typing as th
from singer_sdk import _singerlib as singer
from singer_sdk.mapper import PluginMapper
from singer_sdk.mapper_base import InlineMapper
-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
from pathlib import PurePath
@@ -24,6 +23,7 @@ class {{ cookiecutter.name }}Mapper(InlineMapper):
th.Property(
"example_config",
th.StringType,
+ title="Example Configuration",
description="An example config, replace or remove based on your needs.",
),
).to_dict()
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/dependabot.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/dependabot.yml
index 0660ffdd4..d96600409 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/dependabot.yml
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/dependabot.yml
@@ -19,6 +19,7 @@ updates:
dependency-type: production
update-types:
- "patch"
+ versioning-strategy: increase-if-necessary
- package-ecosystem: pip
directory: "/.github/workflows"
schedule:
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/build.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/build.yml
index f9503baf6..179acf18a 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/build.yml
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/build.yml
@@ -42,4 +42,4 @@ jobs:
- name: Publish
## TODO: create a trusted publisher on PyPI
## https://docs.pypi.org/trusted-publishers/
- uses: pypa/gh-action-pypi-publish@v1.9.0
+ uses: pypa/gh-action-pypi-publish@v1.12.2
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml
index a6a631c2c..380e39371 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml
@@ -3,30 +3,49 @@
name: Test {{cookiecutter.tap_id}}
-on: [push]
+on:
+ push:
+ branches: [main]
+ paths:
+ - .github/workflows/test.yml
+ - {{ cookiecutter.library_name }}/**
+ - tests/**
+ - poetry.lock
+ - pyproject.toml
+ - tox.ini
+ pull_request:
+ branches: [main]
+ paths:
+ - .github/workflows/test.yml
+ - {{ cookiecutter.library_name }}/**
+ - tests/**
+ - poetry.lock
+ - pyproject.toml
+ - tox.ini
+ workflow_dispatch:
+
+env:
+ FORCE_COLOR: 1
jobs:
pytest:
runs-on: ubuntu-latest
- env:
- GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
strategy:
fail-fast: false
matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version:
+ - "3.9"
+ - "3.10"
+ - "3.11"
+ - "3.12"
+ - "3.13"
steps:
- uses: actions/checkout@v4
- name: Set up Python {{ '${{ matrix.python-version }}' }}
uses: actions/setup-python@v5
with:
python-version: {{ '${{ matrix.python-version }}' }}
- - name: Install Poetry
- run: |
- pip install poetry
- - name: Install dependencies
- run: |
- poetry env use {{ '${{ matrix.python-version }}' }}
- poetry install
- - name: Test with pytest
+ - run: pipx install tox
+ - name: Run Tox
run: |
- poetry run pytest
+ tox -e $(echo py{{ '${{ matrix.python-version }}' }} | tr -d .)
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
index ced959bcc..869ab2492 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ ci:
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.6.0
+ rev: v5.0.0
hooks:
- id: check-json
exclude: |
@@ -18,20 +18,20 @@ repos:
- id: trailing-whitespace
- repo: https://github.com/python-jsonschema/check-jsonschema
- rev: 0.29.1
+ rev: 0.29.4
hooks:
- id: check-dependabot
- id: check-github-workflows
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.5.6
+ rev: v0.7.3
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix, --show-fixes]
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.11.1
+ rev: v1.13.0
hooks:
- id: mypy
additional_dependencies:
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/README.md b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/README.md
index 0454d048c..2136340ef 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/README.md
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/README.md
@@ -122,7 +122,7 @@ Now you can test and orchestrate using Meltano:
# Test invocation:
meltano invoke {{ cookiecutter.tap_id }} --version
# OR run a test `elt` pipeline:
-meltano elt {{ cookiecutter.tap_id }} target-jsonl
+meltano run {{ cookiecutter.tap_id }} target-jsonl
```
### SDK Dev Guide
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
index 98851d925..bf1deb719 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -15,11 +15,11 @@ keywords = [
classifiers = [
"Intended Audience :: Developers",
"Operating System :: OS Independent",
- "Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
]
license = "Apache-2.0"
{%- if cookiecutter.variant != "None (Skip)" %}
@@ -29,9 +29,8 @@ packages = [
{%- endif %}
[tool.poetry.dependencies]
-python = ">=3.8"
-importlib-resources = { version = "==6.4.*", python = "<3.9" }
-singer-sdk = { version="~=0.39.1", extras = [
+python = ">=3.9"
+singer-sdk = { version="~=0.42.1", extras = [
{%- if cookiecutter.auth_method == "JWT" -%}"jwt", {% endif -%}
{%- if cookiecutter.faker_extra -%}"faker",{%- endif -%}
] }
@@ -43,9 +42,9 @@ requests = "~=2.32.3"
[tool.poetry.group.dev.dependencies]
pytest = ">=8"
{%- if cookiecutter.auth_method == "JWT" %}
-singer-sdk = { version="~=0.39.1", extras = ["jwt", "testing"] }
+singer-sdk = { version="~=0.42.1", extras = ["jwt", "testing"] }
{%- else %}
-singer-sdk = { version="~=0.39.1", extras = ["testing"] }
+singer-sdk = { version="~=0.42.1", extras = ["testing"] }
{%- endif %}
[tool.poetry.extras]
@@ -64,8 +63,7 @@ plugins = "sqlmypy"
{%- endif %}
[tool.ruff]
-src = ["{{cookiecutter.library_name}}"]
-target-version = "py38"
+target-version = "py39"
[tool.ruff.lint]
ignore = [
@@ -79,14 +77,11 @@ select = ["ALL"]
[tool.ruff.lint.flake8-annotations]
allow-star-arg-any = true
-[tool.ruff.lint.isort]
-known-first-party = ["{{cookiecutter.library_name}}"]
-
[tool.ruff.lint.pydocstyle]
convention = "google"
[build-system]
-requires = ["poetry-core==1.9.0"]
+requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.poetry.scripts]
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini
index 6be1c116a..90c122a54 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini
@@ -1,19 +1,12 @@
# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
[tox]
-envlist = py{38,39,310,311,312}
-isolated_build = true
+envlist = py3{9,10,11,12,13}
+requires =
+ tox>=4.19
[testenv]
-allowlist_externals = poetry
+deps =
+ pytest
commands =
- poetry install -v
- poetry run pytest
-
-[testenv:pytest]
-# Run the python tests.
-# To execute, run `tox -e pytest`
-envlist = py{38,39,310,311,312}
-commands =
- poetry install -v
- poetry run pytest
+ pytest {posargs}
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py
index 4e878cb23..289199b11 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Iterable
+import typing as t
import requests # noqa: TCH002
from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
@@ -12,7 +12,7 @@
from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator
{%- endif %}
-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
from singer_sdk.helpers.types import Context
@@ -45,16 +45,13 @@ def http_headers(self) -> dict:
Returns:
A dictionary of HTTP headers.
"""
- headers = {}
- if "user_agent" in self.config:
- headers["User-Agent"] = self.config.get("user_agent")
{%- if cookiecutter.auth_method not in ("OAuth2", "JWT") %}
# If not using an authenticator, you may also provide inline auth headers:
# headers["Private-Token"] = self.config.get("auth_token")
{%- endif %}
- return headers
+ return {}
- def parse_response(self, response: requests.Response) -> Iterable[dict]:
+ def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
"""Parse the response and return an iterator of result records.
Args:
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/other-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/other-client.py
index 1952579d7..6564b4926 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/other-client.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/other-client.py
@@ -2,11 +2,11 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Iterable
+import typing as t
from singer_sdk.streams import Stream
-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
from singer_sdk.helpers.types import Context
@@ -15,8 +15,8 @@ class {{ cookiecutter.source_name }}Stream(Stream):
def get_records(
self,
- context: Context | None, # noqa: ARG002
- ) -> Iterable[dict]:
+ context: Context | None,
+ ) -> t.Iterable[dict]:
"""Return a generator of record-type dictionary objects.
The optional `context` argument is used to identify a specific slice of the
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py
index fb805ad55..35a53303c 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py
@@ -2,11 +2,11 @@
from __future__ import annotations
-import sys
-{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
+import typing as t
+{% if cookiecutter.auth_method in ("OAuth2", "JWT") -%}
from functools import cached_property
-{%- endif %}
-from typing import TYPE_CHECKING, Any, Iterable
+{% endif -%}
+from importlib import resources
{% if cookiecutter.auth_method == "API Key" -%}
from singer_sdk.authenticators import APIKeyAuthenticator
@@ -40,12 +40,7 @@
{% endif -%}
-if sys.version_info >= (3, 9):
- import importlib.resources as importlib_resources
-else:
- import importlib_resources
-
-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
import requests
{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
from singer_sdk.helpers.types import Auth, Context
@@ -55,7 +50,7 @@
# TODO: Delete this is if not using json files for schema definition
-SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
+SCHEMAS_DIR = resources.files(__package__) / "schemas"
class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream):
@@ -137,14 +132,11 @@ def http_headers(self) -> dict:
Returns:
A dictionary of HTTP headers.
"""
- headers = {}
- if "user_agent" in self.config:
- headers["User-Agent"] = self.config.get("user_agent")
{%- if cookiecutter.auth_method not in ("OAuth2", "JWT") %}
# If not using an authenticator, you may also provide inline auth headers:
# headers["Private-Token"] = self.config.get("auth_token") # noqa: ERA001
{%- endif %}
- return headers
+ return {}
def get_new_paginator(self) -> BaseAPIPaginator:
"""Create a new pagination helper instance.
@@ -164,8 +156,8 @@ def get_new_paginator(self) -> BaseAPIPaginator:
def get_url_params(
self,
context: Context | None, # noqa: ARG002
- next_page_token: Any | None, # noqa: ANN401
- ) -> dict[str, Any]:
+ next_page_token: t.Any | None, # noqa: ANN401
+ ) -> dict[str, t.Any]:
"""Return a dictionary of values to be used in URL parameterization.
Args:
@@ -186,7 +178,7 @@ def get_url_params(
def prepare_request_payload(
self,
context: Context | None, # noqa: ARG002
- next_page_token: Any | None, # noqa: ARG002, ANN401
+ next_page_token: t.Any | None, # noqa: ARG002, ANN401
) -> dict | None:
"""Prepare the data payload for the REST API request.
@@ -202,7 +194,7 @@ def prepare_request_payload(
# TODO: Delete this method if no payload is required. (Most REST APIs.)
return None
- def parse_response(self, response: requests.Response) -> Iterable[dict]:
+ def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
"""Parse the response and return an iterator of result records.
Args:
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py
index a34cee4d0..886b0d8f7 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py
@@ -5,7 +5,7 @@
from __future__ import annotations
-from typing import Any, Iterable
+import typing as t
import sqlalchemy # noqa: TCH002
from singer_sdk import SQLConnector, SQLStream
@@ -77,7 +77,7 @@ class {{ cookiecutter.source_name }}Stream(SQLStream):
connector_class = {{ cookiecutter.source_name }}Connector
- def get_records(self, partition: dict | None) -> Iterable[dict[str, Any]]:
+ def get_records(self, partition: dict | None) -> t.Iterable[dict[str, t.Any]]:
"""Return a generator of record-type dictionary objects.
Developers may optionally add custom logic before calling the default
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py
index 69c955e6f..6e296a259 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/streams.py
@@ -2,21 +2,15 @@
from __future__ import annotations
-import sys
import typing as t
+from importlib import resources
from singer_sdk import typing as th # JSON Schema typing helpers
from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
-if sys.version_info >= (3, 9):
- import importlib.resources as importlib_resources
-else:
- import importlib_resources
-
-
# TODO: Delete this is if not using json files for schema definition
-SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
+SCHEMAS_DIR = resources.files(__package__) / "schemas"
{%- if cookiecutter.stream_type == "GraphQL" %}
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py
index df3f9f754..3fbd58a43 100644
--- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py
+++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py
@@ -31,12 +31,14 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
th.StringType,
required=True,
secret=True, # Flag config as protected.
+ title="Auth Token",
description="The token to authenticate against the API service",
),
th.Property(
"project_ids",
th.ArrayType(th.StringType),
required=True,
+ title="Project IDs",
description="Project IDs to replicate",
),
th.Property(
@@ -47,9 +49,20 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
th.Property(
"api_url",
th.StringType,
+ title="API URL",
default="https://api.mysample.com",
description="The url for the API service",
),
+ {%- if cookiecutter.stream_type in ("GraphQL", "REST") %}
+ th.Property(
+ "user_agent",
+ th.StringType,
+ description=(
+ "A custom User-Agent header to send with each request. Default is "
+ "'/'"
+ ),
+ ),
+ {%- endif %}
).to_dict()
{%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/dependabot.yml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/dependabot.yml
index 0660ffdd4..d96600409 100644
--- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/dependabot.yml
+++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/dependabot.yml
@@ -19,6 +19,7 @@ updates:
dependency-type: production
update-types:
- "patch"
+ versioning-strategy: increase-if-necessary
- package-ecosystem: pip
directory: "/.github/workflows"
schedule:
diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/build.yml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/build.yml
index f9503baf6..179acf18a 100644
--- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/build.yml
+++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/build.yml
@@ -42,4 +42,4 @@ jobs:
- name: Publish
## TODO: create a trusted publisher on PyPI
## https://docs.pypi.org/trusted-publishers/
- uses: pypa/gh-action-pypi-publish@v1.9.0
+ uses: pypa/gh-action-pypi-publish@v1.12.2
diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml
index fda907e07..db8a39d17 100644
--- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml
+++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml
@@ -3,30 +3,49 @@
name: Test {{cookiecutter.target_id}}
-on: [push]
+on:
+ push:
+ branches: [main]
+ paths:
+ - .github/workflows/test.yml
+ - {{ cookiecutter.library_name }}/**
+ - tests/**
+ - poetry.lock
+ - pyproject.toml
+ - tox.ini
+ pull_request:
+ branches: [main]
+ paths:
+ - .github/workflows/test.yml
+ - {{ cookiecutter.library_name }}/**
+ - tests/**
+ - poetry.lock
+ - pyproject.toml
+ - tox.ini
+ workflow_dispatch:
+
+env:
+ FORCE_COLOR: 1
jobs:
pytest:
runs-on: ubuntu-latest
- env:
- GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
strategy:
fail-fast: false
matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version:
+ - "3.9"
+ - "3.10"
+ - "3.11"
+ - "3.12"
+ - "3.13"
steps:
- uses: actions/checkout@v4
- name: Set up Python {{ '${{ matrix.python-version }}' }}
uses: actions/setup-python@v5
with:
python-version: {{ '${{ matrix.python-version }}' }}
- - name: Install Poetry
- run: |
- pip install poetry
- - name: Install dependencies
- run: |
- poetry env use {{ '${{ matrix.python-version }}' }}
- poetry install
- - name: Test with pytest
+ - run: pipx install tox
+ - name: Run Tox
run: |
- poetry run pytest
+ tox -e $(echo py{{ '${{ matrix.python-version }}' }} | tr -d .)
diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml
index 3cab6f92b..86aedf0e9 100644
--- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml
+++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ ci:
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.6.0
+ rev: v5.0.0
hooks:
- id: check-json
exclude: |
@@ -18,20 +18,20 @@ repos:
- id: trailing-whitespace
- repo: https://github.com/python-jsonschema/check-jsonschema
- rev: 0.29.1
+ rev: 0.29.4
hooks:
- id: check-dependabot
- id: check-github-workflows
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.5.6
+ rev: v0.7.3
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix, --show-fixes]
- id: ruff-format
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.11.1
+ rev: v1.13.0
hooks:
- id: mypy
additional_dependencies:
diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md b/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md
index 983be1ce5..6733f0fc0 100644
--- a/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md
+++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md
@@ -51,7 +51,7 @@ This Singer target will automatically import any environment variables within th
`.env` if the `--config=ENV` is provided, such that config values will be considered if a matching
environment variable is set either in the terminal context or in the `.env` file.
-### Source Authentication and Authorization
+### Authentication and Authorization