diff --git a/.github/actions/provision-meos/action.yml b/.github/actions/provision-meos/action.yml index 7c66f57..e8e91db 100644 --- a/.github/actions/provision-meos/action.yml +++ b/.github/actions/provision-meos/action.yml @@ -4,6 +4,11 @@ description: >- generate output/meos-idl.json via run.py and, optionally, build and install libmeos with all families. Any binding consumes it from one pinned commit. + The derivation itself (configure, build, install, parse) is tools/provision-meos.sh, + the one recipe this action and a local refresh both call, so CI and by-hand cannot + drift. This action adds only the runner bootstrapping around it: checkout, Python, and + the apt packages. + inputs: mobilitydb-ref: description: MobilityDB git ref (SHA or branch) to derive the artifacts from. @@ -20,10 +25,10 @@ inputs: outputs: catalog-path: description: Absolute path to the generated meos-idl.json. - value: ${{ steps.catalog.outputs.catalog-path || steps.catalog_installed.outputs.catalog-path }} + value: ${{ steps.catalog.outputs.catalog-path || steps.provision.outputs.catalog-path }} libmeos-prefix: description: Install prefix of libmeos when build-libmeos=true (else empty). - value: ${{ steps.libmeos.outputs.libmeos-prefix }} + value: ${{ steps.provision.outputs.libmeos-prefix }} runs: using: composite @@ -56,10 +61,11 @@ runs: libjson-c-dev libgsl-dev libproj-dev libgeos-dev \ postgresql-server-dev-16 - # run.py + parser/ + meta/ + requirements.txt all live at the MEOS-API - # repo root. When this action is consumed cross-repo GitHub checks out the - # WHOLE MEOS-API repository under $GITHUB_ACTION_PATH, so the repo root is - # $GITHUB_ACTION_PATH/../../.. in both the local dogfood and cross-repo use. + # run.py + parser/ + meta/ + requirements.txt + tools/provision-meos.sh all + # live at the MEOS-API repo root. When this action is consumed cross-repo + # GitHub checks out the WHOLE MEOS-API repository under $GITHUB_ACTION_PATH, + # so the repo root is $GITHUB_ACTION_PATH/../../.. in both the local dogfood + # and cross-repo use. - name: Install Python dependencies shell: bash run: | @@ -67,28 +73,18 @@ runs: python -m pip install --upgrade pip pip install -r "$REPO_ROOT/requirements.txt" - # Catalog-only path (no libmeos build): parse the pinned MobilityDB MEOS - # source headers with libclang and emit output/meos-idl.json, pointing the - # @sqlfn/@ingroup extraction (MDB_SRC_ROOT) at the SAME pinned checkout so - # the catalog is reproducibly equivalent to that commit. The source-tree - # headers wrap PostgreSQL types in stubs conditionally, so struct field - # layouts are approximate here; the build path below derives them exactly. + # Catalog-only path (no libmeos build): provision-meos.sh parses the source + # MobilityDB MEOS headers with libclang and emits output/meos-idl.json, + # pointing the @sqlfn/@ingroup extraction (MDB_SRC_ROOT) at the SAME checkout. + # The source-tree headers wrap PostgreSQL types in stubs conditionally, so + # struct field layouts are approximate here; the build path below is exact. - name: Generate the catalog id: catalog if: inputs.build-libmeos != 'true' shell: bash run: | REPO_ROOT="$(cd "$GITHUB_ACTION_PATH/../../.." && pwd)" - cd "$REPO_ROOT" - MDB_SRC_ROOT="$GITHUB_WORKSPACE/_mobilitydb_src" \ - python3 run.py "$GITHUB_WORKSPACE/_mobilitydb_src/meos/include" - CATALOG="$REPO_ROOT/output/meos-idl.json" - if [ ! -s "$CATALOG" ]; then - echo "::error::catalog not generated or empty at $CATALOG" - exit 1 - fi - echo "catalog-path=$CATALOG" >> "$GITHUB_OUTPUT" - echo "Catalog written: $CATALOG ($(wc -c < "$CATALOG") bytes)" + "$REPO_ROOT/tools/provision-meos.sh" --mdb-src "$GITHUB_WORKSPACE/_mobilitydb_src" # Optional: build and install libmeos with all families. Mirrors the green # all-families recipe used by the JVM bindings: purge the runner's @@ -122,52 +118,23 @@ runs: autoconf automake libtool pkg-config \ postgresql-17 postgresql-server-dev-17 - - name: Build and install libmeos - id: libmeos - if: inputs.build-libmeos == 'true' - shell: bash - run: | - cd "$GITHUB_WORKSPACE/_mobilitydb_src" - mkdir -p build - cd build - cmake -DCMAKE_BUILD_TYPE=Release -DMEOS=ON ${{ inputs.families }} \ - -DH3_LIBRARY=/usr/lib/x86_64-linux-gnu/libh3.so \ - -DH3_INCLUDE_DIR=/usr/include/h3 \ - -DPOSTGRESQL_PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config \ - .. 2>&1 | tee "$RUNNER_TEMP/cmake-configure.log" - make -j "$(nproc)" - # Stage the MEOS headers alone into a clean prefix for the catalog parse - # BEFORE the shared install: /usr/local/include on the runner carries - # unrelated system headers (OpenSSL, PostgreSQL, …) that run.py's header - # glob would otherwise pull into the catalog; this prefix holds only MEOS - # headers. Runs before `sudo make install` so the build directory (where - # cmake writes install_manifest.txt) is still writable by the runner. - cmake --install . --prefix "$RUNNER_TEMP/meos-headers" - echo "installed MEOS headers under $RUNNER_TEMP/meos-headers/include" - sudo make install - echo "libmeos-prefix=/usr/local" >> "$GITHUB_OUTPUT" - echo "libmeos installed under /usr/local" - - # Build path: derive the catalog from the INSTALLED headers. `make install` - # writes the generated, self-contained headers (meos_export.h spliced with - # postgres_ext_defs.in.h in place of the source's `#include `), - # so libclang resolves every struct field to its real C type and byte - # offset — the FFI-accurate layout the native bindings' `#[repr(C)]` structs - # and cgo/cffi field access require. MDB_SRC_ROOT stays the source checkout - # for the @sqlfn/@ingroup maps. - - name: Generate the catalog from installed headers - id: catalog_installed + # Build path: provision-meos.sh builds libmeos, stages a clean MEOS-only + # header prefix ($RUNNER_TEMP/meos-headers) that the parse reads, and installs + # the runtime library under /usr/local for the consumer builds. It derives the + # catalog from the INSTALLED headers: `make install` writes the generated, + # self-contained headers (meos_export.h spliced with postgres_ext_defs.in.h in + # place of the source's `#include `), so libclang resolves every + # struct field to its real C type and byte offset — the FFI-accurate layout the + # native bindings need. MDB_SRC_ROOT stays the source checkout for @sqlfn/@ingroup. + - name: Build libmeos and generate the catalog from installed headers + id: provision if: inputs.build-libmeos == 'true' shell: bash run: | REPO_ROOT="$(cd "$GITHUB_ACTION_PATH/../../.." && pwd)" - cd "$REPO_ROOT" - MDB_SRC_ROOT="$GITHUB_WORKSPACE/_mobilitydb_src" \ - python3 run.py "$RUNNER_TEMP/meos-headers/include" - CATALOG="$REPO_ROOT/output/meos-idl.json" - if [ ! -s "$CATALOG" ]; then - echo "::error::catalog not generated or empty at $CATALOG" - exit 1 - fi - echo "catalog-path=$CATALOG" >> "$GITHUB_OUTPUT" - echo "Catalog written from installed headers: $CATALOG ($(wc -c < "$CATALOG") bytes)" + "$REPO_ROOT/tools/provision-meos.sh" \ + --mdb-src "$GITHUB_WORKSPACE/_mobilitydb_src" \ + --build-libmeos \ + --families "${{ inputs.families }}" \ + --parse-prefix "$RUNNER_TEMP/meos-headers" \ + --runtime-prefix /usr/local diff --git a/GENERATION.md b/GENERATION.md index c24541a..4c95f18 100644 --- a/GENERATION.md +++ b/GENERATION.md @@ -43,27 +43,28 @@ different fidelity, and the choice matters: - the **source** tree headers (`/meos/include`) parse without a build, but wrap the PostgreSQL types in stubs, so struct layouts are approximate. -Either way `MDB_SRC_ROOT` must point at the MobilityDB *source* checkout, because the Doxygen -`@ingroup` groups and the `@sqlfn` SQL-name map are read from `meos/src`, `mobilitydb/src` and -`mobilitydb/sql`. +Either way the Doxygen `@ingroup` groups and the `@sqlfn` SQL-name map are read from the +MobilityDB *source* checkout (`meos/src`, `mobilitydb/src`, `mobilitydb/sql`). + +`tools/provision-meos.sh` is that recipe — configure, build, install, parse — in one script. +The `provision-meos` GitHub action calls it too, so the by-hand catalog and the CI catalog are +produced the same way and cannot drift. On Ubuntu, with the build/parse dependencies installed +(`pip install -r requirements.txt`, plus the packages the action's apt steps list): ```bash MDB=~/src/MobilityDB # a checkout at the commit you are deriving from -MEOSAPI=~/src/MEOS-API -pip install -r "$MEOSAPI/requirements.txt" -# Full fidelity: build and install libmeos first, then parse the installed headers. -cmake -S "$MDB" -B "$MDB/build" -DCMAKE_BUILD_TYPE=Release -DMEOS=ON -DALL=ON -cmake --build "$MDB/build" -j"$(nproc)" -cmake --install "$MDB/build" --prefix "$MDB/.prefix" -cd "$MEOSAPI" && MDB_SRC_ROOT="$MDB" python3 run.py "$MDB/.prefix/include" +# Full fidelity: build and install libmeos, then parse the installed headers. +tools/provision-meos.sh --mdb-src "$MDB" --build-libmeos --parse-prefix "$MDB/.prefix" # Headers only, no build (approximate struct layouts): -cd "$MEOSAPI" && MDB_SRC_ROOT="$MDB" python3 run.py "$MDB/meos/include" +tools/provision-meos.sh --mdb-src "$MDB" ``` -Both write `output/meos-idl.json`, which is what every binding consumes. `-DALL=ON` enables -each optional family, so the catalog covers the whole surface; a narrower family set yields a +Both write `output/meos-idl.json`, which is what every binding consumes; the script sets +`MDB_SRC_ROOT` and picks the header source for you (`--help` lists every option). `--build-libmeos` +also installs libmeos into `--parse-prefix` (default `/.prefix`). Families default to +`-DALL=ON`, so the catalog covers the whole surface; a narrower `--families` set yields a correspondingly narrower catalog. Each binding then regenerates from that file through its own entry point — for the JVM diff --git a/tools/provision-meos.sh b/tools/provision-meos.sh new file mode 100755 index 0000000..5130fa2 --- /dev/null +++ b/tools/provision-meos.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# provision-meos.sh — derive the MEOS catalog (output/meos-idl.json) from a MobilityDB +# checkout, and optionally build and install libmeos. This is the single, runnable +# definition of that derivation: the provision-meos GitHub action calls it, and a human +# refreshing the chain locally calls the same script, so the CI path and the by-hand path +# cannot drift. +# +# It does not check out MobilityDB, install system packages, or set up Python — those are +# environment bootstrapping (the action does them as its own steps; a developer has the +# build/parse dependencies and `pip install -r requirements.txt` already). This script is +# the reproducible recipe: configure, build, install, parse. +# +# The header source decides the catalog, deliberately: without --build-libmeos the +# source-tree headers are parsed (no build, approximate struct layouts); with it the +# installed headers are parsed (full FFI-accurate layouts). MDB_SRC_ROOT is the source +# checkout either way, for the @ingroup/@sqlfn maps. +# +# Usage: +# tools/provision-meos.sh --mdb-src [options] +# +# Required: +# --mdb-src MobilityDB source checkout to derive from. +# +# Options: +# --build-libmeos Build and install libmeos, and parse the *installed* headers. +# --families cmake family flags for the libmeos build (default: -DALL=ON). +# --parse-prefix Clean prefix the MEOS headers (and libmeos) are installed into +# for the parse; its include/ holds only MEOS headers, so the +# catalog glob does not pull in system headers. Build path only. +# Default: /.prefix. +# --runtime-prefix Prefix libmeos is additionally installed into for runtime use. +# Installed with sudo when not writable (e.g. /usr/local in CI). +# Default: same as --parse-prefix (one install, no sudo). +# --meos-api MEOS-API repo root (holds run.py). Default: this script's repo. +# --catalog-out Copy the catalog here in addition to /output/meos-idl.json. +# -h, --help +# +# Outputs (printed as KEY=value, and appended to $GITHUB_OUTPUT when set): +# catalog-path= +# libmeos-prefix= +set -euo pipefail + +usage() { sed -n '2,/^set -euo/{/^set -euo/d;s/^# \{0,1\}//;p;}' "$0"; } + +MDB_SRC="" +BUILD_LIBMEOS=0 +FAMILIES="-DALL=ON" +PARSE_PREFIX="" +RUNTIME_PREFIX="" +MEOSAPI="$(cd "$(dirname "$0")/.." && pwd)" +CATALOG_OUT="" + +while [ $# -gt 0 ]; do + case "$1" in + --mdb-src) MDB_SRC="$2"; shift 2 ;; + --build-libmeos) BUILD_LIBMEOS=1; shift ;; + --families) FAMILIES="$2"; shift 2 ;; + --parse-prefix) PARSE_PREFIX="$2"; shift 2 ;; + --runtime-prefix) RUNTIME_PREFIX="$2"; shift 2 ;; + --meos-api) MEOSAPI="$(cd "$2" && pwd)"; shift 2 ;; + --catalog-out) CATALOG_OUT="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "provision-meos.sh: unknown argument '$1'" >&2; usage >&2; exit 2 ;; + esac +done + +[ -n "$MDB_SRC" ] || { echo "provision-meos.sh: --mdb-src is required" >&2; exit 2; } +[ -d "$MDB_SRC/meos/include" ] || { + echo "provision-meos.sh: '$MDB_SRC' is not a MobilityDB checkout (no meos/include)" >&2; exit 2; } +MDB_SRC="$(cd "$MDB_SRC" && pwd)" +[ -f "$MEOSAPI/run.py" ] || { + echo "provision-meos.sh: no run.py under --meos-api '$MEOSAPI'" >&2; exit 2; } + +# Install libmeos into a prefix; use sudo only when the prefix is not writable, so a system +# prefix (/usr/local) works in CI and a private prefix needs no privileges. +install_into() { + local prefix="$1" parent + parent="$(dirname "$prefix")" + if [ -w "$prefix" ] || { [ ! -e "$prefix" ] && [ -w "$parent" ]; }; then + cmake --install "$MDB_SRC/build" --prefix "$prefix" + else + sudo cmake --install "$MDB_SRC/build" --prefix "$prefix" + fi +} + +LIBMEOS_PREFIX="" +if [ "$BUILD_LIBMEOS" = 1 ]; then + : "${PARSE_PREFIX:=$MDB_SRC/.prefix}" + : "${RUNTIME_PREFIX:=$PARSE_PREFIX}" + + # Optional CI-provisioned toolchain locations: pass each only when it exists, so CI (where + # they do) stays byte-for-byte and a developer's cmake auto-detects its own instead. + cfg=(-DCMAKE_BUILD_TYPE=Release -DMEOS=ON $FAMILIES) + [ -e "${H3_LIBRARY:=/usr/lib/x86_64-linux-gnu/libh3.so}" ] && cfg+=(-DH3_LIBRARY="$H3_LIBRARY") + [ -e "${H3_INCLUDE_DIR:=/usr/include/h3}" ] && cfg+=(-DH3_INCLUDE_DIR="$H3_INCLUDE_DIR") + [ -x "${PG_CONFIG:=/usr/lib/postgresql/17/bin/pg_config}" ] && cfg+=(-DPOSTGRESQL_PG_CONFIG="$PG_CONFIG") + + cmake -S "$MDB_SRC" -B "$MDB_SRC/build" "${cfg[@]}" + cmake --build "$MDB_SRC/build" -j "$(nproc)" + + # The parse prefix holds a clean MEOS-only include/; install there first (writable), then + # into the runtime prefix if it is a different location (e.g. the shared /usr/local). + install_into "$PARSE_PREFIX" + [ "$RUNTIME_PREFIX" != "$PARSE_PREFIX" ] && install_into "$RUNTIME_PREFIX" + + PARSE_HEADERS="$PARSE_PREFIX/include" + LIBMEOS_PREFIX="$RUNTIME_PREFIX" +else + # No build: parse the source-tree headers directly. + PARSE_HEADERS="$MDB_SRC/meos/include" +fi + +cd "$MEOSAPI" +MDB_SRC_ROOT="$MDB_SRC" python3 run.py "$PARSE_HEADERS" +CATALOG="$MEOSAPI/output/meos-idl.json" +[ -s "$CATALOG" ] || { echo "::error::catalog not generated or empty at $CATALOG" >&2; exit 1; } + +if [ -n "$CATALOG_OUT" ]; then + mkdir -p "$(dirname "$CATALOG_OUT")" + cp "$CATALOG" "$CATALOG_OUT" + CATALOG="$(cd "$(dirname "$CATALOG_OUT")" && pwd)/$(basename "$CATALOG_OUT")" +fi + +echo "catalog-path=$CATALOG" +echo "libmeos-prefix=$LIBMEOS_PREFIX" +if [ -n "${GITHUB_OUTPUT:-}" ]; then + { + echo "catalog-path=$CATALOG" + echo "libmeos-prefix=$LIBMEOS_PREFIX" + } >> "$GITHUB_OUTPUT" +fi +echo "Catalog written: $CATALOG ($(wc -c < "$CATALOG") bytes)" >&2