call yoyo from code, wip sqlalchemy-core

Author: liangwp

README.md

@@ -28,10 +28,14 @@ This is suitable for:
     - [ ] knex.js?
     - [ ] https://sqorn.org/benchmarks.html ?
 - [ ] Example python code to use this cluster
-    - [x] using psycopg3 directly
-    - [x] using yoyo migrations
+    - `./python-psycopg3`
+        - [x] using psycopg3 directly
+        - [x] using yoyo migrations through terminal
+            - docker exec into the container to run the `yoyo` cli
     - ~~[ ] sqlalchemy core and alembic? (alembic is too coupled with sqlalchemy for my taste)~~
-    - [ ] sqlalchemy core and yoyo-migrations?
+    - `./python-sql-alchemy-core`
+        - [x] yoyo migrations called from python code
+        - [ ] sqlalchemy-core for data manipulation
 
 ### Prerequisites
 
@@ -140,8 +144,19 @@ Using yoyo migrations:
     ```
 1. Also possible to [perform migrations using code](https://ollycope.com/software/yoyo/latest/#calling-yoyo-from-python-code)
 
-#### More clients WIP
+#### Python sqlalchemy client
 
+1. Reset all yoyo tables in the database (if yoyo cli was run in the psycopg3
+   client test).
+    - docker exec into one of the cockroachdb instances (see above)
+    - `\c test_app`: Connect to the `test_app` database
+    - `\dt`: List tables
+    - `DROP TABLE IF EXISTS _yoyo_log, _yoyo_migration, _yoyo_version, yoyo_lock`
+1. `docker compose -f ./python-sqlalchemy-core/docker-compose.yml up --build`
+1. (WIP) see some logs...
+1. `docker compose -f ./python-sqlalchemy-core/docker-compose.yml down`
+
+#### More Clients WIP
 
 ### Stop the Cluster
 

dbcluster/scripts/create-db-and-user.sh renamed to dbcluster/scripts/create-test-db-and-user.sh

@@ -1,5 +1,5 @@
 [DEFAULT]
 sources = migrations
 database = postgresql+psycopg://%(DB_USER)s:%(DB_PSWD)s@%(DB_HOST)s/%(DB_NAME)s
-batch_mode = off
+batch_mode = off # "on" disable interactive features
 verbosity = 0

python-psycopg3/yoyo.ini

@@ -0,0 +1,30 @@
+FROM debian:12.1-slim
+
+ENV PYTHONUNBUFFERED=1
+ENV NODE_ENV=production
+
+RUN apt-get update -y \
+    && apt-get install -y \
+        dumb-init vim curl \
+        python3 python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /workspace/src \
+    && curl -sSL https://install.python-poetry.org | python3 - --version 1.6.1
+# set poetry path
+ENV PATH="/root/.local/bin:$PATH"
+
+WORKDIR /workspace
+COPY pyproject.toml poetry.lock /workspace/
+
+RUN poetry install
+
+# Set up migrations
+# COPY ./yoyo.ini /workspace/yoyo.ini
+
+COPY ./src /workspace/src/
+
+ENTRYPOINT [ "dumb-init", "--" ]
+
+CMD [ "poetry", "run", "python3", "src/main.py" ]
+

python-sqlalchemy-core/Dockerfile

@@ -0,0 +1,20 @@
+networks:
+  default:
+    name: cockroachdb-net
+    external: true
+
+services:
+  python-psycopg3-client:
+    image: python-psycopg3-client:latest
+    build: .
+    container_name: python-psycopg3-client
+    environment:
+      - DB_NAME=test_app
+      - DB_USER=test_app_user
+      - DB_PSWD=test_app_pswd
+      - DB_HOST=cockroach-db.local:26257
+      # for testing
+      # - DB_HOST=postgres-db-1.local:5432
+    networks:
+      - default
+

python-sqlalchemy-core/docker-compose.yml

@@ -0,0 +1,17 @@
+[tool.poetry]
+name = "python-sqlalchemy-core"
+version = "0.1.0"
+description = ""
+authors = ["weipang <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+yoyo-migrations = "^8.2.0"
+psycopg = {extras = ["binary", "pool"], version = "^3.1.12"}
+sqlalchemy = {extras = ["asyncio"], version = "^2.0.21"}
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

python-sqlalchemy-core/poetry.lock

@@ -0,0 +1,33 @@
+from yoyo import read_migrations
+from yoyo import get_backend
+
+import os
+import asyncio
+
+async def async_main():
+    while True:
+        print('hello world')
+        await asyncio.sleep(2)
+
+def main():
+    asyncio.run(async_main())
+
+if __name__ == '__main__':
+
+    DB_USER = os.environ['DB_USER']
+    DB_PSWD = os.environ['DB_PSWD']
+    DB_HOST = os.environ['DB_HOST']
+    DB_NAME = os.environ['DB_NAME']
+
+    backend = get_backend(f'postgresql+psycopg://{DB_USER}:{DB_PSWD}@{DB_HOST}/{DB_NAME}')
+    migrations = read_migrations('/workspace/src/migrations')
+
+    with backend.lock():
+
+        # Apply any outstanding migrations
+        backend.apply_migrations(backend.to_apply(migrations))
+
+        # Rollback all migrations
+        # backend.rollback_migrations(backend.to_rollback(migrations))
+
+    main()

python-sqlalchemy-core/pyproject.toml

@@ -0,0 +1,12 @@
+from yoyo import step
+
+# Using python is more flexible and allows being more explicit intent when
+# defining migration scripts.
+# https://ollycope.com/software/yoyo/latest/#migrations-as-python-scripts
+
+apply = 'CREATE TABLE test_sqlalchemy (id INT, value VARCHAR(20), value2 INT, PRIMARY KEY (id))'
+rollback = 'DROP TABLE test_sqlalchemy'
+
+steps = [
+    step(apply, rollback)
+]

python-sqlalchemy-core/src/main.py

@@ -0,0 +1,10 @@
+from yoyo import step
+
+__depends__ = {"0000-initial_schema"}
+
+apply = 'ALTER TABLE test_sqlalchemy ADD COLUMN new_column BOOLEAN NOT NULL DEFAULT TRUE'
+rollback = 'ALTER TABLE test_sqlalchemy DROP COLUMN new_column'
+
+steps = [
+    step(apply, rollback)
+]