feat: add seperated index command, remove duplication of index for te…

…mp table on import

docker-compose.yml

@@ -19,7 +19,7 @@ services:
       - api-shared
 
   postgres:
-    container_name: postgres
+    container_name: postgres-opendata
     image: postgres:15
     ports:
       - ${DB_PORT}:5432
@@ -34,7 +34,7 @@ services:
       - api-shared
 
   pgadmin:
-    container_name: pgadmin
+    container_name: pgadmin-opendata
     image: dpage/pgadmin4:6.14
     restart: unless-stopped
     environment:
@@ -51,7 +51,7 @@ services:
       - api-shared
 
   pghero:
-    container_name: pghero
+    container_name: pghero-opendata
     image: ankane/pghero
     environment:
       DATABASE_URL: "postgres://${DB_USER}:${DB_PASSWORD}@postgres:${DB_PORT}/${DB_NAME}"

docker-service

@@ -68,6 +68,11 @@ importData() {
     docker-compose -f docker-compose"$env".yml exec opendata-api bash -c "npm run importData $datasetName"
 }
 
+indexTable() {
+    datasetName="$1"
+    docker-compose -f docker-compose"$env".yml exec opendata-api bash -c "npm run indexTable $datasetName"
+}
+
 downloadAssets() {
     datasetName="$1"
     docker-compose -f docker-compose"$env".yml exec opendata-api bash -c "npm run downloadAssets $datasetName"
@@ -106,6 +111,7 @@ Commands:
   install                     Run app installation scripts
   initTables                  Run Postgres tables installation scripts
   importData                  Run dataset import into a Postgres table
+  indexTable                  Creates the index on given table
   downloadAssets              Download the given data set from configured source
   serve                       Serve NodeJS server for local dev
   tests                       Run unit and quality tests
@@ -143,6 +149,9 @@ initTables)
 importData)
         importData $2
         ;;
+indexTable)
+        indexTable $2
+        ;;
 downloadAssets)
         downloadAssets $2
         ;;

package.json

@@ -9,6 +9,7 @@
     "engineStrict": true,
     "scripts": {
         "initTables": "ts-node-dev src/utils/initTables.ts",
+        "indexTable": "ts-node-dev src/utils/indexTable.ts",
         "importData": "ts-node-dev src/utils/importData.ts --",
         "downloadAssets": "ts-node-dev src/utils/downloadAssets.ts -- ",
         "install:ci": "npm ci --loglevel=error --cache .npm --prefer-offline --no-audit",
@@ -86,4 +87,4 @@
         "winston": "^3.8.2",
         "winston-daily-rotate-file": "^4.7.1"
     }
-}
+}

src/tables/ban.ts

@@ -29,19 +29,12 @@ export const initBaseAdresseNationale = () => {
         "cad_parcelles" CHARACTER VARYING
     );`;
 
-    const createIndexes = `
-    CREATE INDEX IF NOT EXISTS idx_ban_id ON ban(id);
-    CREATE INDEX IF NOT EXISTS idx_ban_id_fantoir ON ban(id_fantoir);
-    CREATE INDEX IF NOT EXISTS idx_ban_code_insee ON ban(code_insee);
-    `;
-
     return Pg.execute(createTable).then(async (res) => {
         if (res) {
             console.log('ban table initialized ✓');
         } else {
             console.log('ban table already exist (nothing done)');
         }
-        await Pg.execute(createIndexes);
 
         return res;
     });

src/tables/departementsFrance.ts

@@ -10,18 +10,12 @@ export const initDepartementsFranceTable = () => {
         "region" CHARACTER VARYING
     );`;
 
-    const createIndexes = `
-    CREATE INDEX IF NOT EXISTS idx_departementsfr_code_departement ON departementsfr(code_departement);
-    CREATE INDEX IF NOT EXISTS idx_departementsfr_code_region ON departementsfr(code_region);
-    `;
-
     return Pg.execute(createTable).then(async (res) => {
         if (res) {
             console.log('departementsfr table initialized ✓');
         } else {
             console.log('departementsfr table already exist (nothing done)');
         }
-        await Pg.execute(createIndexes);
 
         return res;
     });

src/tables/sirene.ts

@@ -113,18 +113,12 @@ export const initSireneTable = () => {
         "geolocetablissement" CHARACTER VARYING
     );`;
 
-    const createIndexes = `
-    CREATE INDEX IF NOT EXISTS idx_sirene_siren ON sirene(siren);
-    CREATE INDEX IF NOT EXISTS idx_sirene_siret ON sirene(siret);
-    `;
-
     return Pg.execute(createTable).then(async (res) => {
         if (res) {
             console.log('sirene table initialized ✓');
         } else {
             console.log('sirene table already exist (nothing done)');
         }
-        await Pg.execute(createIndexes);
 
         return res;
     });

src/utils/importData.ts

@@ -67,7 +67,7 @@ const importDatabase = async (dataSetType: string) => {
 
         // Drop tmp table if exists
         await client.query(`DROP TABLE IF EXISTS ${tmpTable}`);
-        await client.query(`CREATE TABLE ${tmpTable} (LIKE ${dataSetType} INCLUDING ALL)`);
+        await client.query(`CREATE TABLE ${tmpTable} (LIKE ${dataSetType})`);
 
         // Copy data in this temp table
         const ingestStream = client.query(copyFrom(`

src/utils/indexTable.ts

@@ -0,0 +1,62 @@
+import { Pg } from '../helpers';
+
+
+// Get the arguments
+const dataSetName = process.argv[2]; // dataset name
+
+const indexCommands = {
+    sirene: `
+    CREATE INDEX IF NOT EXISTS idx_sirene_siren ON sirene(siren);
+    CREATE INDEX IF NOT EXISTS idx_sirene_siret ON sirene(siret);
+    `,
+    ban: `
+    CREATE INDEX IF NOT EXISTS idx_ban_id ON ban(id);
+    CREATE INDEX IF NOT EXISTS idx_ban_id_fantoir ON ban(id_fantoir);
+    CREATE INDEX IF NOT EXISTS idx_ban_code_insee ON ban(code_insee);
+    `,
+    departementsfr: `
+    CREATE INDEX IF NOT EXISTS idx_departementsfr_code_departement ON departementsfr(code_departement);
+    CREATE INDEX IF NOT EXISTS idx_departementsfr_code_region ON departementsfr(code_region);
+    `
+};
+
+export const indexTable = async (dataSetType: string) => {
+
+    // In case of no matching dataset found
+    if (!dataSetType) {
+        console.log(`
+        Please provide as argument the dataset name you want to create index on: ${Object.keys(indexCommands).join(' / ')})
+    `);
+    }
+
+    if (!Object.keys(indexCommands).includes(dataSetType)) {
+        throw Error(`No dataset with key "${dataSetType}" exists. Please check available datasets in the docs.`);
+    }
+
+    const pool = Pg.getPool();
+    const client = await pool.connect();
+
+    try {
+        await client.query(indexCommands[dataSetType]);
+    } catch (error) {
+        console.log(error);
+        process.exit(1); // Error
+    } finally {
+        client.release();
+    }
+
+    process.exit(0);
+};
+
+indexTable(dataSetName);
+
+// Exit message
+process.on('exit', function (code) {
+    if (code === 0) {
+        console.log('\nSuccess!');
+    } else {
+        console.log(`\nProcess exited with code ${code}.`);
+    }
+
+    return;
+});

src/utils/initTables.ts

@@ -1,14 +1,9 @@
 import { initBaseAdresseNationale, initDepartementsFranceTable, initSireneTable } from '../tables';
 
-const initTables = async () => {
-
-    try {
-        await initSireneTable();
-        await initDepartementsFranceTable();
-        await initBaseAdresseNationale();
-    } catch (error) {
-        //
-    }
+export const initTables = async () => {
+    await initSireneTable();
+    await initDepartementsFranceTable();
+    await initBaseAdresseNationale();
 
     process.exit(0);
 };