Merge branch 'main' into 2779-failwarn-when-there-are-no-selected-str…

…eams
meltano · Dec 9, 2024 · d1d4287 · d1d4287
2 parents 4191f2e + e2a78e9
commit d1d4287
Show file tree

Hide file tree

Showing 23 changed files with 766 additions and 627 deletions.
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -1,4 +1,4 @@
-name: codspeed
+name: Performance Testing with CodSpeed 🐇
 
 on:
   push:
@@ -24,6 +24,13 @@ on:
   # performance analysis in order to generate initial data.
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: "1"
+
 jobs:
   benchmarks:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,5 +1,5 @@
 griffe~=1.5
 pip==24.3.1
-poetry==1.8.4
+poetry==1.8.5
 pre-commit==4.0.1
 nox==2024.10.9
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -40,7 +40,7 @@ jobs:
       with:
         name: Packages
         path: dist
-    - uses: actions/attest-build-provenance@v1
+    - uses: actions/attest-build-provenance@v2
       id: attest
       with:
         subject-path: "./dist/singer_sdk*"

diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -51,7 +51,7 @@ jobs:
 
     - name: Bump version
       id: cz-bump
-      uses: commitizen-tools/commitizen-action@0.22.0
+      uses: commitizen-tools/commitizen-action@0.23.0
       with:
         increment: ${{ github.event.inputs.bump != 'auto' && github.event.inputs.bump || '' }}
         prerelease: ${{ github.event.inputs.prerelease != 'none' && github.event.inputs.prerelease || '' }}

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-24.04
   tools:
-    python: "3.12"
+    python: "3.13"
 
 sphinx:
   builder: html

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
@@ -18,13 +18,13 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.4
+  rev: 0.30.0
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.7.3
+  rev: v0.8.1
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -53,8 +53,6 @@ target-version = "py39"
 
 [tool.ruff.lint]
 ignore = [
-    "ANN101",  # missing-type-self
-    "ANN102",  # missing-type-cls
     "COM812",  # missing-trailing-comma
     "ISC001",  # single-line-implicit-string-concatenation
 ]

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
@@ -18,13 +18,13 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.4
+  rev: 0.30.0
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.7.3
+  rev: v0.8.1
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -38,6 +38,9 @@ fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 requests = "~=2.32.3"
 {%- endif %}
+{%- if cookiecutter.stream_type == "SQL" %}
+sqlalchemy = "~=2.0.36"
+{%- endif %}
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
@@ -67,8 +70,6 @@ target-version = "py39"
 
 [tool.ruff.lint]
 ignore = [
-    "ANN101",  # missing-type-self
-    "ANN102",  # missing-type-cls
     "COM812",  # missing-trailing-comma
     "ISC001",  # single-line-implicit-string-concatenation
 ]

diff --git a/...tter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py b/...tter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py
@@ -5,7 +5,7 @@
 import decimal
 import typing as t
 
-import requests  # noqa: TCH002
+import requests  # noqa: TC002
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}

diff --git a/...ecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py b/...ecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py
@@ -12,29 +12,29 @@
 {% if cookiecutter.auth_method  == "API Key" -%}
 from singer_sdk.authenticators import APIKeyAuthenticator
 from singer_sdk.helpers.jsonpath import extract_jsonpath
-from singer_sdk.pagination import BaseAPIPaginator  # noqa: TCH002
+from singer_sdk.pagination import BaseAPIPaginator  # noqa: TC002
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 
 {% elif cookiecutter.auth_method  == "Bearer Token" -%}
 from singer_sdk.authenticators import BearerTokenAuthenticator
 from singer_sdk.helpers.jsonpath import extract_jsonpath
-from singer_sdk.pagination import BaseAPIPaginator  # noqa: TCH002
+from singer_sdk.pagination import BaseAPIPaginator  # noqa: TC002
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 
 {% elif cookiecutter.auth_method == "Basic Auth" -%}
 from requests.auth import HTTPBasicAuth
 from singer_sdk.helpers.jsonpath import extract_jsonpath
-from singer_sdk.pagination import BaseAPIPaginator  # noqa: TCH002
+from singer_sdk.pagination import BaseAPIPaginator  # noqa: TC002
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 
 {% elif cookiecutter.auth_method == "Custom or N/A" -%}
 from singer_sdk.helpers.jsonpath import extract_jsonpath
-from singer_sdk.pagination import BaseAPIPaginator  # noqa: TCH002
+from singer_sdk.pagination import BaseAPIPaginator  # noqa: TC002
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 
 {% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%}
 from singer_sdk.helpers.jsonpath import extract_jsonpath
-from singer_sdk.pagination import BaseAPIPaginator  # noqa: TCH002
+from singer_sdk.pagination import BaseAPIPaginator  # noqa: TC002
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
 
 from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator

diff --git a/...iecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py b/...iecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/sql-client.py
@@ -7,7 +7,7 @@
 
 import typing as t
 
-import sqlalchemy  # noqa: TCH002
+import sqlalchemy  # noqa: TC002
 from singer_sdk import SQLConnector, SQLStream
 
 

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml
@@ -18,13 +18,13 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.4
+  rev: 0.30.0
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.7.3
+  rev: v0.8.1
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml
@@ -55,8 +55,6 @@ target-version = "py39"
 
 [tool.ruff.lint]
 ignore = [
-    "ANN101",  # missing-type-self
-    "ANN102",  # missing-type-cls
     "COM812",  # missing-trailing-comma
     "ISC001",  # single-line-implicit-string-concatenation
 ]

diff --git a/docs/guides/sql-tap.md b/docs/guides/sql-tap.md
@@ -1,5 +1,9 @@
 # Building SQL taps
 
+```{warning}
+Starting with version `0.43.0`, SQL taps require SQLAlchemy 2.0 or newer.
+```
+
 ## Mapping SQL types to JSON Schema
 
 Starting with version `0.41.0`, the Meltano Singer SDK provides a clean way to map SQL types to JSON Schema. This is useful when the SQL dialect you are using has custom types that need to be mapped accordingly to JSON Schema.

diff --git a/docs/stream_maps.md b/docs/stream_maps.md
@@ -778,3 +778,31 @@ the `key_properties` in an extract-load pipeline. For instance, it is common to
 "append-only" loading behavior in certain targets, as may be required for historical reporting. This does not change the
 underlying nature of the `primary_key` configuration in the upstream source data, only how it will be landed or deduped
 in the downstream source.
+
+
+### Q: How do I use Meltano environment variables to configure stream maps?
+
+**Answer:** Environment variables in Meltano can be used to configure stream maps, but you first need to add the corresponding settings
+to your plugins `settings` option. For example:
+
+```yaml
+plugins:
+  extractors:
+  - name: tap-csv
+    variant: meltanolabs
+    pip_url: git+https://github.com/MeltanoLabs/tap-csv.git
+    settings:
+    - name: stream_maps.customers.email
+    - name: stream_maps.customers.email_domain
+    - name: stream_maps.customers.email_hash
+    - name: stream_maps.customers.__else__
+    - name: stream_maps.stream_map_config
+```
+
+Then, you can set the following environment variables:
+
+```shell
+TAP_CSV_STREAM_MAPS_CUSTOMERS_EMAIL_DOMAIN='email.split("@")[-1]'
+TAP_CSV_STREAM_MAPS_CUSTOMERS_EMAIL_HASH='md5(config["hash_seed"] + email)'
+TAP_CSV_STREAM_MAP_CONFIG_HASH_SEED='01AWZh7A6DzGm6iJZZ2T'
+```