From 5f80c5f23a164844c0008ca5b7c80925f61d22a5 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 9 Jul 2025 22:08:15 -0700 Subject: [PATCH 1/7] builtins: generate in 1 command --- .github/workflows/ci.yml | 12 ++++ generator/README.md | 28 +++++++++ generator/docker-compose.yml | 22 +++++++ generator/generate.js | 111 +++++++++++++++++++++++++++++++++++ lib/builtins.js | 22 ++++--- package.json | 2 + 6 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 generator/README.md create mode 100644 generator/docker-compose.yml create mode 100644 generator/generate.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9654afb..8d4fb4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,3 +53,15 @@ jobs: node-version: '20' - run: npm install - run: npm run test-ts + + generate: + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + - run: npm install + - run: npm run generate + - run: git diff --exit-code lib/ diff --git a/generator/README.md b/generator/README.md new file mode 100644 index 0000000..1978b95 --- /dev/null +++ b/generator/README.md @@ -0,0 +1,28 @@ +# PostgreSQL Type OID Generator + +Generates PostgreSQL builtin type OIDs for `node-pg-types`. + +## Usage + +```sh +npm install +npm run generate +``` + +This will: + +- Start PostgreSQL 11 in a container +- Run the generator in another container +- Query for builtin type OIDs +- Generate `../lib/builtins.js` +- Tear down the containers + +## Adding PostgreSQL versions + +To query additional PostgreSQL versions, edit the files in this directory: + +1. Update the version list in `generate.js` +2. Add the new PostgreSQL service to `docker-compose.yml` +3. Add the service dependency to the generator service instructing it to wait for the PostgreSQL service to be ready + +Types from multiple PostgreSQL versions are automatically merged. diff --git a/generator/docker-compose.yml b/generator/docker-compose.yml new file mode 100644 index 0000000..a03c4ae --- /dev/null +++ b/generator/docker-compose.yml @@ -0,0 +1,22 @@ +services: + postgres-11: + image: postgres:11-alpine + environment: + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 5s + retries: 5 + + generator: + image: node:20-alpine + working_dir: /app + command: node generator/generate.js + depends_on: + postgres-11: + condition: service_healthy + volumes: + - ../:/app + environment: + OUTPUT_PATH: /app/lib \ No newline at end of file diff --git a/generator/generate.js b/generator/generate.js new file mode 100644 index 0000000..b2cbdfe --- /dev/null +++ b/generator/generate.js @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +const { Client } = require('pg') +const fs = require('fs') +const path = require('path') + +const query = ` +SELECT json_object_agg(UPPER(PT.typname), PT.oid::int4 ORDER BY pt.oid) +FROM pg_type PT +WHERE typnamespace = (SELECT pgn.oid FROM pg_namespace pgn WHERE nspname = 'pg_catalog') -- Take only builting Postgres types with stable OID (extension types are not guaranted to be stable) +AND typtype = 'b' -- Only basic types +AND typelem = 0 -- Ignore aliases +AND typisdefined -- Ignore undefined types +` + +const postgresVersions = ['11'] + +async function queryPostgresVersion (version) { + const host = `postgres-${version}` + console.log(`Querying PostgreSQL ${version} at ${host}...`) + + const client = new Client({ + host: host, + port: 5432, + user: process.env.PGUSER || 'postgres', + password: process.env.PGPASSWORD || 'postgres', + database: process.env.PGDATABASE || 'postgres' + }) + + try { + await client.connect() + const result = await client.query(query) + const types = result.rows[0].json_object_agg + console.log(`Found ${Object.keys(types).length} types in PostgreSQL ${version}`) + return types + } finally { + await client.end() + } +} + +// Legacy types that were removed in newer PostgreSQL versions +// These maintain backward compatibility for older applications +const legacyTypes = { + SMGR: 210, + ABSTIME: 702, // Deprecated: 7, Removed: 12 + RELTIME: 703, // Deprecated: 7, Removed: 12 + TINTERVAL: 704 // Deprecated: 7, Removed: 12 +} + +async function generate () { + console.log('Starting PostgreSQL type generation...') + + // Query all available PostgreSQL versions + let allTypes = {} + for (const version of postgresVersions) { + const types = await queryPostgresVersion(version) + allTypes = { ...allTypes, ...types } + } + + // Add legacy types for backward compatibility + allTypes = { ...allTypes, ...legacyTypes } + + console.log(`Total unique types found: ${Object.keys(allTypes).length}`) + + // Determine output directory + const outputDir = process.env.OUTPUT_PATH || path.join(__dirname, '..', 'lib') + + // Generate main builtins file + const header = `/** +PostgreSQL builtin type OIDs + +DO NOT EDIT THIS FILE BY HAND! +This file is generated automatically by the generator in generator/generate.js +To modify the types, edit the generator script and run: npm run generate + +Generated by querying PostgreSQL ${postgresVersions.join(', ')} to ensure comprehensive +type coverage for parsing. Includes legacy types for backward compatibility. + +Query used: +${query.trim()} + */` + + const entries = Object.entries(allTypes) + .sort(([, a], [, b]) => a - b) + .map(([name, oid]) => ` ${name}: ${oid}`) + .join(',\n') + + const content = `${header} + +module.exports = { +${entries} +} +` + + // Write main builtins file + const outputPath = path.join(outputDir, 'builtins.js') + await fs.promises.writeFile(outputPath, content) + + console.log('Generated lib/builtins.js successfully') +} + +async function main () { + try { + await generate() + } catch (err) { + console.error('Error generating types:', err) + process.exit(1) + } +} + +main() diff --git a/lib/builtins.js b/lib/builtins.js index 6a8b89d..07f52c5 100644 --- a/lib/builtins.js +++ b/lib/builtins.js @@ -1,12 +1,20 @@ /** - * Following query was used to generate this file: +PostgreSQL builtin type OIDs - SELECT json_object_agg(UPPER(PT.typname), PT.oid::int4 ORDER BY pt.oid) - FROM pg_type PT - WHERE typnamespace = (SELECT pgn.oid FROM pg_namespace pgn WHERE nspname = 'pg_catalog') -- Take only builting Postgres types with stable OID (extension types are not guaranted to be stable) - AND typtype = 'b' -- Only basic types - AND typelem = 0 -- Ignore aliases - AND typisdefined -- Ignore undefined types +DO NOT EDIT THIS FILE BY HAND! +This file is generated automatically by the generator in generator/generate.js +To modify the types, edit the generator script and run: npm run generate + +Generated by querying PostgreSQL 11 to ensure comprehensive +type coverage for parsing. Includes legacy types for backward compatibility. + +Query used: +SELECT json_object_agg(UPPER(PT.typname), PT.oid::int4 ORDER BY pt.oid) +FROM pg_type PT +WHERE typnamespace = (SELECT pgn.oid FROM pg_namespace pgn WHERE nspname = 'pg_catalog') -- Take only builting Postgres types with stable OID (extension types are not guaranted to be stable) +AND typtype = 'b' -- Only basic types +AND typelem = 0 -- Ignore aliases +AND typisdefined -- Ignore undefined types */ module.exports = { diff --git a/package.json b/package.json index 570f6e0..4ee767e 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "coverage": "nyc --reporter=html npm test && open-cli coverage/index.html", "coverage-ci": "nyc --reporter=lcov npm test && codecov", + "generate": "docker compose --project-directory generator run --rm generator", "lint": "standard", "test": "npm run test-js && npm run test-ts && npm run lint", "test-js": "tape test/*.js | tap-spec", @@ -31,6 +32,7 @@ "codecov": "^3.8.1", "nyc": "^15.1.0", "open-cli": "^6.0.1", + "pg": "^8.11.3", "standard": "^16.0.3", "tap-spec": "^5.0.0", "tape": "^5.2.2", From 9095627c178febf8f8c3b867632369819971dfea Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 9 Jul 2025 22:15:06 -0700 Subject: [PATCH 2/7] parallel versions --- generator/generate.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/generator/generate.js b/generator/generate.js index b2cbdfe..79be1aa 100644 --- a/generator/generate.js +++ b/generator/generate.js @@ -50,10 +50,14 @@ const legacyTypes = { async function generate () { console.log('Starting PostgreSQL type generation...') - // Query all available PostgreSQL versions + // Query all available PostgreSQL versions in parallel + const typeResults = await Promise.all( + postgresVersions.map(version => queryPostgresVersion(version)) + ) + + // Merge all types from different versions let allTypes = {} - for (const version of postgresVersions) { - const types = await queryPostgresVersion(version) + for (const types of typeResults) { allTypes = { ...allTypes, ...types } } From a014eca1596eb82bb6761afa37fa341023351907 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 9 Jul 2025 22:16:57 -0700 Subject: [PATCH 3/7] rm v11 in docs --- generator/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/README.md b/generator/README.md index 1978b95..65697e9 100644 --- a/generator/README.md +++ b/generator/README.md @@ -11,13 +11,13 @@ npm run generate This will: -- Start PostgreSQL 11 in a container +- Start all specified Postgres versions in containers - Run the generator in another container - Query for builtin type OIDs - Generate `../lib/builtins.js` - Tear down the containers -## Adding PostgreSQL versions +## Adding PostgreSQL Versions To query additional PostgreSQL versions, edit the files in this directory: From 57313045cb1bf0cb108d127c7dd0a8761e3e5c68 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 9 Jul 2025 22:27:21 -0700 Subject: [PATCH 4/7] rm legacy --- generator/generate.js | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/generator/generate.js b/generator/generate.js index 79be1aa..9199dad 100644 --- a/generator/generate.js +++ b/generator/generate.js @@ -38,14 +38,6 @@ async function queryPostgresVersion (version) { } } -// Legacy types that were removed in newer PostgreSQL versions -// These maintain backward compatibility for older applications -const legacyTypes = { - SMGR: 210, - ABSTIME: 702, // Deprecated: 7, Removed: 12 - RELTIME: 703, // Deprecated: 7, Removed: 12 - TINTERVAL: 704 // Deprecated: 7, Removed: 12 -} async function generate () { console.log('Starting PostgreSQL type generation...') @@ -61,9 +53,6 @@ async function generate () { allTypes = { ...allTypes, ...types } } - // Add legacy types for backward compatibility - allTypes = { ...allTypes, ...legacyTypes } - console.log(`Total unique types found: ${Object.keys(allTypes).length}`) // Determine output directory @@ -78,7 +67,7 @@ This file is generated automatically by the generator in generator/generate.js To modify the types, edit the generator script and run: npm run generate Generated by querying PostgreSQL ${postgresVersions.join(', ')} to ensure comprehensive -type coverage for parsing. Includes legacy types for backward compatibility. +type coverage for parsing. Query used: ${query.trim()} From 609672cf3a28c15b028852c84c135cc703a76c45 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 9 Jul 2025 22:28:55 -0700 Subject: [PATCH 5/7] standard --- generator/generate.js | 1 - 1 file changed, 1 deletion(-) diff --git a/generator/generate.js b/generator/generate.js index 9199dad..49e6345 100644 --- a/generator/generate.js +++ b/generator/generate.js @@ -38,7 +38,6 @@ async function queryPostgresVersion (version) { } } - async function generate () { console.log('Starting PostgreSQL type generation...') From cf2377796aadefa61f6012aa1af0a842932d86ec Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 9 Jul 2025 22:29:59 -0700 Subject: [PATCH 6/7] generate --- lib/builtins.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/builtins.js b/lib/builtins.js index 07f52c5..74474d3 100644 --- a/lib/builtins.js +++ b/lib/builtins.js @@ -6,7 +6,7 @@ This file is generated automatically by the generator in generator/generate.js To modify the types, edit the generator script and run: npm run generate Generated by querying PostgreSQL 11 to ensure comprehensive -type coverage for parsing. Includes legacy types for backward compatibility. +type coverage for parsing. Query used: SELECT json_object_agg(UPPER(PT.typname), PT.oid::int4 ORDER BY pt.oid) From d261f15adad636501c92b7c89d1d36eab58d257f Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 9 Jul 2025 22:49:33 -0700 Subject: [PATCH 7/7] generate typescript --- generator/generate.js | 33 +++++++++++++++++- index.d.ts | 68 +++--------------------------------- lib/builtins.d.ts | 81 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 65 deletions(-) create mode 100644 lib/builtins.d.ts diff --git a/generator/generate.js b/generator/generate.js index 49e6345..2cdfbe7 100644 --- a/generator/generate.js +++ b/generator/generate.js @@ -88,7 +88,38 @@ ${entries} const outputPath = path.join(outputDir, 'builtins.js') await fs.promises.writeFile(outputPath, content) - console.log('Generated lib/builtins.js successfully') + // Generate TypeScript definitions + const tsHeader = `/** +PostgreSQL builtin type OIDs (TypeScript definitions) + +DO NOT EDIT THIS FILE BY HAND! +This file is generated automatically by the generator in generator/generate.js +To modify the types, edit the generator script and run: npm run generate + +Generated by querying PostgreSQL ${postgresVersions.join(', ')} to ensure comprehensive +type coverage for parsing. + +Query used: +${query.trim()} + */` + + const tsEntries = Object.entries(allTypes) + .sort(([, a], [, b]) => a - b) + .map(([name, oid]) => ` ${name} = ${oid}`) + .join(',\n') + + const tsContent = `${tsHeader} + +export enum builtins { +${tsEntries} +} +` + + // Write TypeScript definitions file + const tsOutputPath = path.join(outputDir, 'builtins.d.ts') + await fs.promises.writeFile(tsOutputPath, tsContent) + + console.log('Generated lib/builtins.js and lib/builtins.d.ts successfully') } async function main () { diff --git a/index.d.ts b/index.d.ts index 3bb2738..b0bcfe3 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,68 +1,8 @@ -export enum builtins { - BOOL = 16, - BYTEA = 17, - CHAR = 18, - INT8 = 20, - INT2 = 21, - INT4 = 23, - REGPROC = 24, - TEXT = 25, - OID = 26, - TID = 27, - XID = 28, - CID = 29, - JSON = 114, - XML = 142, - PG_NODE_TREE = 194, - SMGR = 210, - PATH = 602, - POLYGON = 604, - CIDR = 650, - FLOAT4 = 700, - FLOAT8 = 701, - ABSTIME = 702, - RELTIME = 703, - TINTERVAL = 704, - CIRCLE = 718, - MACADDR8 = 774, - MONEY = 790, - MACADDR = 829, - INET = 869, - ACLITEM = 1033, - BPCHAR = 1042, - VARCHAR = 1043, - DATE = 1082, - TIME = 1083, - TIMESTAMP = 1114, - TIMESTAMPTZ = 1184, - INTERVAL = 1186, - TIMETZ = 1266, - BIT = 1560, - VARBIT = 1562, - NUMERIC = 1700, - REFCURSOR = 1790, - REGPROCEDURE = 2202, - REGOPER = 2203, - REGOPERATOR = 2204, - REGCLASS = 2205, - REGTYPE = 2206, - UUID = 2950, - TXID_SNAPSHOT = 2970, - PG_LSN = 3220, - PG_NDISTINCT = 3361, - PG_DEPENDENCIES = 3402, - TSVECTOR = 3614, - TSQUERY = 3615, - GTSVECTOR = 3642, - REGCONFIG = 3734, - REGDICTIONARY = 3769, - JSONB = 3802, - REGNAMESPACE = 4089, - REGROLE = 4096 -} +import * as _builtins from './lib/builtins'; -export type TypeId = builtins; -export type TypesBuiltins = typeof builtins; +export const builtins: typeof _builtins.builtins; +export type TypeId = _builtins.builtins; +export type TypesBuiltins = typeof _builtins.builtins; export type TypeParser = (value: I) => T; diff --git a/lib/builtins.d.ts b/lib/builtins.d.ts new file mode 100644 index 0000000..898eb8f --- /dev/null +++ b/lib/builtins.d.ts @@ -0,0 +1,81 @@ +/** +PostgreSQL builtin type OIDs (TypeScript definitions) + +DO NOT EDIT THIS FILE BY HAND! +This file is generated automatically by the generator in generator/generate.js +To modify the types, edit the generator script and run: npm run generate + +Generated by querying PostgreSQL 11 to ensure comprehensive +type coverage for parsing. + +Query used: +SELECT json_object_agg(UPPER(PT.typname), PT.oid::int4 ORDER BY pt.oid) +FROM pg_type PT +WHERE typnamespace = (SELECT pgn.oid FROM pg_namespace pgn WHERE nspname = 'pg_catalog') -- Take only builting Postgres types with stable OID (extension types are not guaranted to be stable) +AND typtype = 'b' -- Only basic types +AND typelem = 0 -- Ignore aliases +AND typisdefined -- Ignore undefined types + */ + +export enum builtins { + BOOL = 16, + BYTEA = 17, + CHAR = 18, + INT8 = 20, + INT2 = 21, + INT4 = 23, + REGPROC = 24, + TEXT = 25, + OID = 26, + TID = 27, + XID = 28, + CID = 29, + JSON = 114, + XML = 142, + PG_NODE_TREE = 194, + SMGR = 210, + PATH = 602, + POLYGON = 604, + CIDR = 650, + FLOAT4 = 700, + FLOAT8 = 701, + ABSTIME = 702, + RELTIME = 703, + TINTERVAL = 704, + CIRCLE = 718, + MACADDR8 = 774, + MONEY = 790, + MACADDR = 829, + INET = 869, + ACLITEM = 1033, + BPCHAR = 1042, + VARCHAR = 1043, + DATE = 1082, + TIME = 1083, + TIMESTAMP = 1114, + TIMESTAMPTZ = 1184, + INTERVAL = 1186, + TIMETZ = 1266, + BIT = 1560, + VARBIT = 1562, + NUMERIC = 1700, + REFCURSOR = 1790, + REGPROCEDURE = 2202, + REGOPER = 2203, + REGOPERATOR = 2204, + REGCLASS = 2205, + REGTYPE = 2206, + UUID = 2950, + TXID_SNAPSHOT = 2970, + PG_LSN = 3220, + PG_NDISTINCT = 3361, + PG_DEPENDENCIES = 3402, + TSVECTOR = 3614, + TSQUERY = 3615, + GTSVECTOR = 3642, + REGCONFIG = 3734, + REGDICTIONARY = 3769, + JSONB = 3802, + REGNAMESPACE = 4089, + REGROLE = 4096 +}