Merge branch 'main' into edgarrmondragon/feat/_sdc_postgres_schema

MeltanoLabs · Nov 5, 2024 · 4a51677 · 4a51677
2 parents a9188ce + 9bb40d1
commit 4a51677
Show file tree

Hide file tree

Showing 17 changed files with 1,534 additions and 1,290 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -11,8 +11,6 @@ updates:
       interval: weekly
       timezone: "America/New_York"
       time: "07:23"
-    reviewers:
-      - "visch"
     versioning-strategy: increase-if-necessary
     groups:
       development-dependencies:
@@ -39,8 +37,6 @@ updates:
       interval: weekly
       timezone: "America/New_York"
       time: "07:23"
-    reviewers:
-      - "visch"
     groups:
       actions:
         patterns:

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,2 +1,3 @@
-pip==24.2
-poetry==1.8.3
+pip==24.3.1
+poetry==1.8.4
+tox==4.23.2
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -41,4 +41,4 @@ jobs:
         overwrite: true
         file_glob: true
     - name: Deploy to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.9.0
+      uses: pypa/gh-action-pypi-publish@v1.11.0
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,10 +1,36 @@
 name: Test tap-postgres
 
+env:
+  FORCE_COLOR: 1
+
 on:
   pull_request:
+    branches: [main]
     types: [opened, synchronize, reopened]
+    paths:
+    - .github/workflows/test.yml
+    - log_based/**
+    - pagila/**
+    - ssh_tunnel/**
+    - ssl/**
+    - tap_postgres/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
   push:
     branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - log_based/**
+    - pagila/**
+    - ssh_tunnel/**
+    - ssl/**
+    - tap_postgres/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
   workflow_dispatch:
     inputs: {}
 
@@ -14,18 +40,32 @@ concurrency:
 
 jobs:
   tests:
+    name: Python ${{ matrix.python-version }} / Postgres ${{ matrix.postgres-version }}
     runs-on: ubuntu-latest
     env:
       PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
     strategy:
       fail-fast: false
       matrix:
         python-version:
-        - "3.8"
-        - "3.9"
-        - "3.10"
-        - "3.11"
+        - "3.13"
         - "3.12"
+        - "3.11"
+        - "3.10"
+        - "3.9"
+        postgres-version:
+        - "17"
+        include:
+        - python-version: "3.13"
+          postgres-version: "12"
+        - python-version: "3.13"
+          postgres-version: "13"
+        - python-version: "3.13"
+          postgres-version: "14"
+        - python-version: "3.13"
+          postgres-version: "15"
+        - python-version: "3.13"
+          postgres-version: "16"
 
     steps:
     - name: Checkout code
@@ -39,13 +79,19 @@ jobs:
         chmod 600 ssl/pkey.key
 
     - name: Build Postgres container
+      env:
+        POSTGRES_VERSION: ${{ matrix.postgres-version }}
       run: |
-        docker build . --tag meltano/log_based
+        docker build . --build-arg POSTGRES_VERSION=${POSTGRES_VERSION} --tag meltano/log_based
     - name: Compose Postgres container
+      env:
+        POSTGRES_VERSION: ${{ matrix.postgres-version }}
       run: |
         docker compose -f docker-compose.yml up -d --wait --wait-timeout=30
 
-    - uses: isbang/[email protected]
+    - uses: isbang/[email protected]
+      env:
+        POSTGRES_VERSION: ${{ matrix.postgres-version }}
 
  # Leaving comments as this makes testing the openssh docker image easier and should save someone some time later
  #   - name: Test SSH File Permissiosn
@@ -55,25 +101,21 @@ jobs:
  #       docker exec openssh-server ps aux
 
 
+    - run: docker ps
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
 
-    - name: Upgrade pip
-      run: |
-        pip install pip
-        pip --version
-    - name: Install Poetry
-      run: |
-        pip install poetry
-        poetry --version
-    - name: Install dependencies
+    - name: Install Tox
       run: |
-        poetry install
+        python -m pip install --upgrade pip
+        pipx install tox
     - name: Run pytest
       run: |
-        poetry run pytest
+        tox -e $(echo py${{ matrix.python-version }} | tr -d .)
     - name: Run lint
       run: |
-        poetry run tox -e lint
+        tox -e lint
diff --git a/.gitignore b/.gitignore
@@ -136,3 +136,30 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,11 +1,11 @@
 ci:
   autofix_prs: false
-  autoupdate_schedule: weekly
+  autoupdate_schedule: monthly
   autoupdate_commit_msg: 'chore: pre-commit autoupdate'
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
   - id: check-json
   - id: check-toml
@@ -18,7 +18,7 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.5
+  rev: v0.7.2
   hooks:
   - id: ruff
     args: [--fix]

diff --git a/Dockerfile b/Dockerfile
@@ -1,12 +1,17 @@
-FROM postgres:16
+ARG POSTGRES_VERSION=17
+FROM postgres:${POSTGRES_VERSION}
 
-RUN apt-get update
-RUN apt-mark hold locales
-RUN apt-get install curl ca-certificates -y
-RUN install -d /usr/share/postgresql-common/pgdg
-RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
-RUN sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-RUN apt-get update
-RUN apt-get install postgresql-server-dev-16 -y
-RUN sh -c 'export PATH=/usr/lib/postgresql/16/bin:$PATH'
-RUN apt-get install postgresql-16-wal2json -y
+ARG POSTGRES_VERSION=17
+
+# Install prerequisites and configure PostgreSQL for wal2json
+RUN apt-get update && apt-mark hold locales && \
+    apt-get install -y curl ca-certificates && \
+    install -d /usr/share/postgresql-common/pgdg && \
+    curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
+    sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+
+RUN echo "Setting up PostgreSQL ${POSTGRES_VERSION} with wal2json" && \
+    apt-get update && \
+    apt-get install -y postgresql-server-dev-${POSTGRES_VERSION} && \
+    export PATH=/usr/lib/postgresql/${POSTGRES_VERSION}/bin:$PATH && \
+    apt-get install -y postgresql-${POSTGRES_VERSION}-wal2json
diff --git a/README.md b/README.md
@@ -13,6 +13,10 @@ Built with the [Meltano Singer SDK](https://sdk.meltano.com).
 * `stream-maps`
 * `schema-flattening`
 
+## Supported Python and PostgreSQL Versions
+
+This tap is tested with all actively supported [Python](https://devguide.python.org/versions/#supported-versions) and [PostgreSQL](https://www.postgresql.org/support/versioning/) versions. At the time of writing, this includes Python 3.9 through 3.13 and PostgreSQL 12 through 17.
+
 ## Settings
 
 | Setting                           | Required | Default                      | Description                                                                                                                                                                                                                                                                                              |
@@ -26,6 +30,7 @@ Built with the [Meltano Singer SDK](https://sdk.meltano.com).
 | sqlalchemy_url                    | False    | None                         | Example postgresql://[username]:[password]@localhost:5432/[db_name]                                                                                                                                                                                                                                      |
 | filter_schemas                    | False    | None                         | If an array of schema names is provided, the tap will only process the specified Postgres schemas and ignore others. If left blank, the tap automatically determines ALL available Postgres schemas.                                                                                                     |
 | dates_as_string                   | False    | 0                            | Defaults to false, if true, date, and timestamp fields will be Strings. If you see ValueError: Year is out of range, try setting this to True.                                                                                                                                                           |
+| json_as_object                    | False    | 0                            | Defaults to false, if true, json and jsonb fields will be Objects.                                                                                                     |
 | ssh_tunnel                        | False    | None                         | SSH Tunnel Configuration, this is a json object                                                                                                                                                                                                                                                          |
 | ssh_tunnel.enable                 | False    | 0                            | Enable an ssh tunnel (also known as bastion server), see the other ssh_tunnel.* properties for more details                                                                                                                                                                                              |
 | ssh_tunnel.host                   | False    | None                         | Host of the bastion server, this is the host we'll connect to via ssh                                                                                                                                                                                                                                    |
@@ -132,6 +137,8 @@ Create tests within the `tap_postgres/tests` subfolder and
 poetry run pytest
 ```
 
+NOTE: Running the tests requires a locally running postgres. See tests/settings.py for the expected configuration.
+
 You can also test the `tap-postgres` CLI interface directly using `poetry run`:
 
 ```bash

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - inner
 
   postgres_no_ssl:
-    image: postgres:latest
+    image: postgres:${POSTGRES_VERSION:-latest}
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
@@ -30,7 +30,7 @@ services:
       - "5432:5432"
 
   postgresdb:
-    image: postgres:13.0
+    image: postgres:${POSTGRES_VERSION:-latest}
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
@@ -40,7 +40,7 @@ services:
         ipv4_address: 10.5.0.5
 
   postgres_ssl:
-    image: postgres:16
+    image: postgres:${POSTGRES_VERSION:-latest}
     command: postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key -c ssl_ca_file=/var/lib/postgresql/ca.crt -c hba_file=/var/lib/postgresql/pg_hba.conf
     environment:
       POSTGRES_USER: postgres