From 2e0fcd8d7a5af1c990e2200177140459076efb8c Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sat, 7 Dec 2024 10:36:34 +0100 Subject: [PATCH] records: Add example about program use --- .github/dependabot.yml | 5 ++ .github/workflows/application-records.yml | 74 +++++++++++++++++++++++ application/records/README.md | 57 +++++++++++++++++ application/records/example.sh | 34 +++++++++++ application/records/requirements.txt | 3 + application/records/test.sh | 3 + 6 files changed, 176 insertions(+) create mode 100644 .github/workflows/application-records.yml create mode 100644 application/records/README.md create mode 100644 application/records/example.sh create mode 100644 application/records/requirements.txt create mode 100644 application/records/test.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a1b742ac..444a5c5c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -100,6 +100,11 @@ updates: schedule: interval: "daily" + - directory: "/application/records" + package-ecosystem: "pip" + schedule: + interval: "daily" + # Frameworks. - directory: "/framework/dbt/basic" diff --git a/.github/workflows/application-records.yml b/.github/workflows/application-records.yml new file mode 100644 index 00000000..47a6e1de --- /dev/null +++ b/.github/workflows/application-records.yml @@ -0,0 +1,74 @@ +name: records + +on: + pull_request: + branches: ~ + paths: + - '.github/workflows/application-records.yml' + - 'application/records/**' + - '/requirements.txt' + push: + branches: [ main ] + paths: + - '.github/workflows/application-records.yml' + - 'application/records/**' + - '/requirements.txt' + + # Allow job to be triggered manually. + workflow_dispatch: + + # Run job each night after CrateDB nightly has been published. + schedule: + - cron: '0 3 * * *' + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + test: + name: " + Python: ${{ matrix.python-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ 'ubuntu-latest' ] + python-version: [ '3.9', '3.13' ] + cratedb-version: [ 'nightly' ] + + services: + cratedb: + image: crate/crate:${{ matrix.cratedb-version }} + ports: + - 4200:4200 + - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g + + steps: + + - name: Acquire sources + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + cache: 'pip' + cache-dependency-path: | + requirements.txt + application/records/requirements.txt + application/records/requirements-dev.txt + + - name: Install utilities + run: | + pip install -r requirements.txt + + - name: Validate application/records + run: | + ngr test --accept-no-venv application/records diff --git a/application/records/README.md b/application/records/README.md new file mode 100644 index 00000000..25f77f9d --- /dev/null +++ b/application/records/README.md @@ -0,0 +1,57 @@ +# Verify the `records` program with CrateDB + +Records: SQL for Humans™ + +## About + +This folder includes software integration tests for verifying +that the [Records] Python program works well together with [CrateDB]. + +Records is a very simple, but powerful, library for making raw SQL +queries to most relational databases. It uses [SQLAlchemy]. + +Records is intended for report-style exports of database queries, and +has not yet been optimized for extremely large data dumps. + +## What's Inside + +- `example.sh`: A few examples that read CrateDB's `sys.summits` table + using the `records` program. A single example that inserts data into + a column using CrateDB's `OBJECT` column. + +## Install + +Set up sandbox and install packages. +```bash +pip install uv +uv venv .venv +source .venv/bin/activate +uv pip install -r requirements.txt +``` + +## Synopsis +Install packages. +```shell +pip install --upgrade records sqlalchemy-cratedb +``` +Define database connection URL, suitable for CrateDB on localhost. +For CrateDB Cloud, use `crate://:@`. +```shell +export DATABASE_URL="crate://" +``` +Invoke query. +```shell +records "SELECT * FROM sys.summits WHERE region ILIKE :region" region="ortler%" +``` + +## Tests + +Run integration tests. +```bash +sh test.sh +``` + + +[CrateDB]: https://cratedb.com/database +[Records]: https://pypi.org/project/records/ +[SQLAlchemy]: https://www.sqlalchemy.org/ diff --git a/application/records/example.sh b/application/records/example.sh new file mode 100644 index 00000000..206db63e --- /dev/null +++ b/application/records/example.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env sh + +# Using `records` with CrateDB: Basic usage. +# +# pip install --upgrade records sqlalchemy-cratedb +# +# A few basic operations using the `records` program with CrateDB. +# +# - https://pypi.org/project/records/ + +# Define database connection URL, suitable for CrateDB on localhost. +# For CrateDB Cloud, use `crate://:@`. +export DATABASE_URL="crate://" + +# Basic query, tabular output. +records "SELECT * FROM sys.summits ORDER BY height DESC LIMIT 3" + +# Query with parameters. +records "SELECT * FROM sys.summits WHERE region ILIKE :region" region="ortler%" + +# Export data. +# Supported formats: csv tsv json yaml html xls xlsx dbf latex ods +records "SELECT * FROM sys.summits ORDER BY height DESC LIMIT 3" csv +records "SELECT * FROM sys.summits ORDER BY height DESC LIMIT 3" json +records "SELECT * FROM sys.summits LIMIT 42" html > "${TMPDIR}/sys_summits.html" +records "SELECT * FROM sys.summits LIMIT 42" ods > "${TMPDIR}/sys_summits.ods" +records "SELECT * FROM sys.summits LIMIT 42" xlsx > "${TMPDIR}/sys_summits.xlsx" + +# Insert data. +records "DROP TABLE IF EXISTS testdrive.example" +records "CREATE TABLE testdrive.example (data OBJECT(DYNAMIC))" +records "INSERT INTO testdrive.example (data) VALUES (:data)" data='{"temperature": 42.42, "humidity": 84.84}' +records "REFRESH TABLE testdrive.example" +records "SELECT * FROM testdrive.example" diff --git a/application/records/requirements.txt b/application/records/requirements.txt new file mode 100644 index 00000000..7bfcd8a6 --- /dev/null +++ b/application/records/requirements.txt @@ -0,0 +1,3 @@ +records<0.7 +sqlalchemy-cratedb<0.41 +tablib[ods] diff --git a/application/records/test.sh b/application/records/test.sh new file mode 100644 index 00000000..1437ae2f --- /dev/null +++ b/application/records/test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +sh example.sh