diff --git a/docker-compose.yml b/docker-compose.yml index 3b1f531..433a33e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: - api-shared postgres: - container_name: postgres + container_name: postgres-opendata image: postgres:15 ports: - ${DB_PORT}:5432 @@ -34,7 +34,7 @@ services: - api-shared pgadmin: - container_name: pgadmin + container_name: pgadmin-opendata image: dpage/pgadmin4:6.14 restart: unless-stopped environment: @@ -51,7 +51,7 @@ services: - api-shared pghero: - container_name: pghero + container_name: pghero-opendata image: ankane/pghero environment: DATABASE_URL: "postgres://${DB_USER}:${DB_PASSWORD}@postgres:${DB_PORT}/${DB_NAME}" diff --git a/docker-service b/docker-service index 9db2bea..f5c45c6 100755 --- a/docker-service +++ b/docker-service @@ -68,6 +68,11 @@ importData() { docker-compose -f docker-compose"$env".yml exec opendata-api bash -c "npm run importData $datasetName" } +indexTable() { + datasetName="$1" + docker-compose -f docker-compose"$env".yml exec opendata-api bash -c "npm run indexTable $datasetName" +} + downloadAssets() { datasetName="$1" docker-compose -f docker-compose"$env".yml exec opendata-api bash -c "npm run downloadAssets $datasetName" @@ -106,6 +111,7 @@ Commands: install Run app installation scripts initTables Run Postgres tables installation scripts importData Run dataset import into a Postgres table + indexTable Creates the index on given table downloadAssets Download the given data set from configured source serve Serve NodeJS server for local dev tests Run unit and quality tests @@ -143,6 +149,9 @@ initTables) importData) importData $2 ;; +indexTable) + indexTable $2 + ;; downloadAssets) downloadAssets $2 ;; diff --git a/package.json b/package.json index 3ae1c5d..d256a3f 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "engineStrict": true, "scripts": { "initTables": "ts-node-dev src/utils/initTables.ts", + "indexTable": "ts-node-dev src/utils/indexTable.ts", "importData": "ts-node-dev src/utils/importData.ts --", "downloadAssets": "ts-node-dev src/utils/downloadAssets.ts -- ", "install:ci": "npm ci --loglevel=error --cache .npm --prefer-offline --no-audit", @@ -86,4 +87,4 @@ "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1" } -} +} \ No newline at end of file diff --git a/src/tables/ban.ts b/src/tables/ban.ts index bb975b6..c1ea2f0 100644 --- a/src/tables/ban.ts +++ b/src/tables/ban.ts @@ -29,19 +29,12 @@ export const initBaseAdresseNationale = () => { "cad_parcelles" CHARACTER VARYING );`; - const createIndexes = ` - CREATE INDEX IF NOT EXISTS idx_ban_id ON ban(id); - CREATE INDEX IF NOT EXISTS idx_ban_id_fantoir ON ban(id_fantoir); - CREATE INDEX IF NOT EXISTS idx_ban_code_insee ON ban(code_insee); - `; - return Pg.execute(createTable).then(async (res) => { if (res) { console.log('ban table initialized ✓'); } else { console.log('ban table already exist (nothing done)'); } - await Pg.execute(createIndexes); return res; }); diff --git a/src/tables/departementsFrance.ts b/src/tables/departementsFrance.ts index 04ca98b..8bdcbfc 100644 --- a/src/tables/departementsFrance.ts +++ b/src/tables/departementsFrance.ts @@ -10,18 +10,12 @@ export const initDepartementsFranceTable = () => { "region" CHARACTER VARYING );`; - const createIndexes = ` - CREATE INDEX IF NOT EXISTS idx_departementsfr_code_departement ON departementsfr(code_departement); - CREATE INDEX IF NOT EXISTS idx_departementsfr_code_region ON departementsfr(code_region); - `; - return Pg.execute(createTable).then(async (res) => { if (res) { console.log('departementsfr table initialized ✓'); } else { console.log('departementsfr table already exist (nothing done)'); } - await Pg.execute(createIndexes); return res; }); diff --git a/src/tables/sirene.ts b/src/tables/sirene.ts index c7f57a3..b25b279 100644 --- a/src/tables/sirene.ts +++ b/src/tables/sirene.ts @@ -113,18 +113,12 @@ export const initSireneTable = () => { "geolocetablissement" CHARACTER VARYING );`; - const createIndexes = ` - CREATE INDEX IF NOT EXISTS idx_sirene_siren ON sirene(siren); - CREATE INDEX IF NOT EXISTS idx_sirene_siret ON sirene(siret); - `; - return Pg.execute(createTable).then(async (res) => { if (res) { console.log('sirene table initialized ✓'); } else { console.log('sirene table already exist (nothing done)'); } - await Pg.execute(createIndexes); return res; }); diff --git a/src/utils/importData.ts b/src/utils/importData.ts index 7ad100a..6eb8c69 100644 --- a/src/utils/importData.ts +++ b/src/utils/importData.ts @@ -67,7 +67,7 @@ const importDatabase = async (dataSetType: string) => { // Drop tmp table if exists await client.query(`DROP TABLE IF EXISTS ${tmpTable}`); - await client.query(`CREATE TABLE ${tmpTable} (LIKE ${dataSetType} INCLUDING ALL)`); + await client.query(`CREATE TABLE ${tmpTable} (LIKE ${dataSetType})`); // Copy data in this temp table const ingestStream = client.query(copyFrom(` diff --git a/src/utils/indexTable.ts b/src/utils/indexTable.ts new file mode 100644 index 0000000..f8f7d3f --- /dev/null +++ b/src/utils/indexTable.ts @@ -0,0 +1,62 @@ +import { Pg } from '../helpers'; + + +// Get the arguments +const dataSetName = process.argv[2]; // dataset name + +const indexCommands = { + sirene: ` + CREATE INDEX IF NOT EXISTS idx_sirene_siren ON sirene(siren); + CREATE INDEX IF NOT EXISTS idx_sirene_siret ON sirene(siret); + `, + ban: ` + CREATE INDEX IF NOT EXISTS idx_ban_id ON ban(id); + CREATE INDEX IF NOT EXISTS idx_ban_id_fantoir ON ban(id_fantoir); + CREATE INDEX IF NOT EXISTS idx_ban_code_insee ON ban(code_insee); + `, + departementsfr: ` + CREATE INDEX IF NOT EXISTS idx_departementsfr_code_departement ON departementsfr(code_departement); + CREATE INDEX IF NOT EXISTS idx_departementsfr_code_region ON departementsfr(code_region); + ` +}; + +export const indexTable = async (dataSetType: string) => { + + // In case of no matching dataset found + if (!dataSetType) { + console.log(` + Please provide as argument the dataset name you want to create index on: ${Object.keys(indexCommands).join(' / ')}) + `); + } + + if (!Object.keys(indexCommands).includes(dataSetType)) { + throw Error(`No dataset with key "${dataSetType}" exists. Please check available datasets in the docs.`); + } + + const pool = Pg.getPool(); + const client = await pool.connect(); + + try { + await client.query(indexCommands[dataSetType]); + } catch (error) { + console.log(error); + process.exit(1); // Error + } finally { + client.release(); + } + + process.exit(0); +}; + +indexTable(dataSetName); + +// Exit message +process.on('exit', function (code) { + if (code === 0) { + console.log('\nSuccess!'); + } else { + console.log(`\nProcess exited with code ${code}.`); + } + + return; +}); \ No newline at end of file diff --git a/src/utils/initTables.ts b/src/utils/initTables.ts index ea718f5..e2b41d4 100644 --- a/src/utils/initTables.ts +++ b/src/utils/initTables.ts @@ -1,14 +1,9 @@ import { initBaseAdresseNationale, initDepartementsFranceTable, initSireneTable } from '../tables'; -const initTables = async () => { - - try { - await initSireneTable(); - await initDepartementsFranceTable(); - await initBaseAdresseNationale(); - } catch (error) { - // - } +export const initTables = async () => { + await initSireneTable(); + await initDepartementsFranceTable(); + await initBaseAdresseNationale(); process.exit(0); };