Database Cleanup Optimization & Seeder Update

apps/server/seeders/db.js

@@ -0,0 +1,27 @@
+"use strict";
+const { PrismaClient } = require("@prisma/client");
+
+// Dynamically import the ES Module
+const envPromise = import("../src/env.mjs");
+
+let prismaGlobal;
+
+// Asynchronously create and configure the PrismaClient instance
+async function createPrismaClient() {
+    const env = await envPromise; // Await the resolved module
+    const prisma = new PrismaClient({
+        log: env.NODE_ENV === 'development' ? ['error'] : ['error'],
+    });
+
+    if (env.NODE_ENV !== 'production') {
+        prismaGlobal = prisma;
+    }
+
+    return prisma;
+}
+
+// Export a promise that resolves to the PrismaClient instance
+exports.prisma = createPrismaClient();
+
+// If you need to access the global prisma instance elsewhere
+exports.getGlobalPrisma = () => prismaGlobal;

apps/server/seeders/seedAlertMethods.js

@@ -0,0 +1,62 @@
+const db = require('./db');
+
+async function seedAlertMethods(totalUsers, batchSize=500) {
+    const prisma = await db.prisma; // Await the prisma instance
+    let batch = [];
+
+    const processBatch = async () => {
+        if (batch.length > 0) {
+            await prisma.alertMethod.createMany({data: batch});
+            batch = []; // Reset the batch
+        }
+    };
+    const twoMonthsAgo = new Date();
+    twoMonthsAgo.setMonth(twoMonthsAgo.getMonth() - 2);
+
+    for (let userId = 1; userId <= totalUsers; userId++) {
+        const shouldDelete = Math.random() < 0.05; // 25% chance of being marked for deletion
+        batch.push({
+            id:`${userId}alertMethodEmail0`,
+            destination: `user${userId}[email protected]`,
+            method: "email",
+            isEnabled: true,
+            isVerified: true,
+            userId: userId.toString(),
+            deletedAt: shouldDelete ? twoMonthsAgo : null,
+        })
+        batch.push({
+            id:`${userId}alertMethodSMS`,
+            destination: `+977${userId}`,
+            method: "sms",
+            isEnabled: true,
+            isVerified: true,
+            userId: userId.toString(),
+            deletedAt: shouldDelete ? twoMonthsAgo : null,
+        })
+        batch.push({
+            id:`${userId}alertMethodEmail1`,
+            destination: `user${userId}[email protected]`,
+            method: "email",
+            isEnabled: true,
+            isVerified: true,
+            userId: userId.toString(),
+            deletedAt: shouldDelete ? twoMonthsAgo : null,
+        })
+        if(batch.length >= batchSize){
+            await processBatch()
+        }
+    }
+    await processBatch()
+    console.log(`Successfully Seeded AlertMethods`);
+}
+module.exports.seedAlertMethods = seedAlertMethods;
+
+// seedAlertMethods(2000)
+//     .then(() => {
+//         console.log('AlertMethods seeded successfully.');
+//         process.exit(0);
+//     })
+//     .catch(error => {
+//         console.error('Error seeding alertMethods', error);
+//         process.exit(1);
+//     });

apps/server/seeders/seedDatabase.js

@@ -0,0 +1,43 @@
+// To run this file, navigate to the directory of this FireAlert app
+// first set node environment as development by typing this in the terminal: $env:NODE_ENV="development" 
+// and, run this in the terminal: node apps/server/seeders/seedDatabase.js
+// If Error -> Reached heap limit Allocation failed - Javascript heap out of memory
+// Use this command instead: node --max-old-space-size=4096 apps/server/seeders/seedDatabase.js
+
+// With about million geoEvents and 2000 users, it may take upto 1 hour to seed the entire database
+
+const { seedGeoEvents } = require('./seedGeoEvents');
+const { seedSites } = require('./seedSites');
+const { seedUsers } = require('./seedUsers');
+const { seedAlertMethods } = require('./seedAlertMethods');
+const { seedSiteAlertsAndNotifications } = require('./seedSiteAlertsAndNotifications');
+
+// The total number of sites created will be 5 times the number of users, with each user having 5 sites
+// Each site has 1000 siteAlerts from dates ranging from now to 2 months ago
+// Each site has about 300 notifications
+// Each user has about 3 alertMethods
+// The deletedAt for sites, alertMethods, and users will be set in random, for 5% rows to have a deletedAt of 1 month ago in every table
+
+async function seedDatabase(numberOfUsers, numberOfGeoEvents) {
+    try {
+        await seedGeoEvents(numberOfGeoEvents);
+        await seedUsers(numberOfUsers);
+        await seedSites(numberOfUsers);
+        await seedAlertMethods(numberOfUsers);
+        await seedSiteAlertsAndNotifications(numberOfUsers);
+        console.log('Database seeded successfully.');
+    } catch (error) {
+        console.error('Error seeding database:', error);
+    }
+}
+
+seedDatabase(500, 150000)
+    .then(() => {
+        console.log('Database seeded successfully.');
+        process.exit(0);
+    })
+    .catch(error => {
+        console.error('Error seeding database', error);
+        process.exit(1);
+    });
+

apps/server/seeders/seedGeoEvents.js

@@ -1,33 +1,79 @@
 const fs = require('fs');
-const {parse} = require('csv-parse');
-
-import {prisma} from '../src/server/db';
-
-async function seedData() {
-  fs.createReadStream(__dirname + '/data/GeoEvents.csv')
-    .pipe(parse({delimiter: ','}))
-    .on('data', async row => {
-      // Process each row of data and insert it into the database using Prisma
-      await prisma.geoEvent.create({
-        data: {
-          type: 'fire',
-          latitude: parseFloat(row[0]),
-          longitude: parseFloat(row[1]),
-          eventDate: new Date(),
-          confidence: 'high',
-          providerKey: 'FIRMS',
-          identityGroup: 'MODIS',
-          // Map other columns as needed
-        },
-      });
-    })
-    .on('end', () => {
-      console.log('CSV file successfully processed');
-      prisma.$disconnect();
-    });
+const { parse } = require('csv-parse');
+const db = require('./db');
+
+
+function getRandomDate(startDate, endDate) {
+  return new Date(startDate.getTime() + Math.random() * (endDate.getTime() - startDate.getTime()));
 }
 
-seedData().catch(error => {
-  console.error(error);
+async function seedGeoEvents(totalGeoEvents, batchSize = 10000) {
+  const prisma = await db.prisma; // Await the prisma instance
+  const filePath = __dirname + '/data/GeoEvents.csv';
+  const twoMonthsAgo = new Date();
+  twoMonthsAgo.setMonth(twoMonthsAgo.getMonth() - 2);
+
+  let batch = [];
+  let createdRecords = 0;
+
+  const processBatch = async () => {
+    if (batch.length > 0) {
+      const recordsToInsert = batch.splice(0, batchSize);
+      await prisma.geoEvent.createMany({ data: recordsToInsert });
+      createdRecords += recordsToInsert.length;
+    }
+  };
+
+  while (createdRecords < totalGeoEvents) {
+    await new Promise((resolve, reject) => {
+      const stream = fs.createReadStream(filePath)
+        .pipe(parse({ delimiter: ',' }))
+        .on('data', (row) => {
+          if (createdRecords < totalGeoEvents) {
+            batch.push({
+              type: 'fire',
+              latitude: parseFloat(row[0]),
+              longitude: parseFloat(row[1]),
+              eventDate: getRandomDate(twoMonthsAgo, new Date()),
+              confidence: 'high',
+              isProcessed: true,
+              geoEventProviderClientId: 'LANDSAT_NRT',
+              geoEventProviderId: '4',
+              slice: '4',
+            });
+          }
+
+          if (batch.length >= batchSize) {
+            stream.pause(); // Pause the stream
+            processBatch().then(() => {
+              stream.resume(); // Resume the stream
+              if (createdRecords >= totalGeoEvents) {
+                resolve();
+              }
+            });
+          }
+        })
+        .on('end', () => {
+          processBatch().then(resolve);
+        })
+        .on('error', (error) => {
+          reject(error);
+        });
+    });
+  }
+  console.log(`Total ${createdRecords} GeoEvent records created.`);
   prisma.$disconnect();
-});
+}
+module.exports.seedGeoEvents = seedGeoEvents;
+
+// Example usage with 5000 records and 500 batchSize
+// seedGeoEvents(5000, 500)
+// .then(() => {
+//   console.log('GeoEvents seeded successfully.');
+//   process.exit(0);
+// })
+// .catch(error => {
+//   console.error('Error seeding geoEvents', error);
+//   prisma.$disconnect();
+//   process.exit(1);
+// });

apps/server/seeders/seedSiteAlertsAndNotifications.js

@@ -0,0 +1,63 @@
+const db = require('./db');
+
+function getRandomDate(startDate, endDate) {
+    return new Date(startDate.getTime() + Math.random() * (endDate.getTime() - startDate.getTime()));
+}
+
+async function seedSiteAlertsAndNotifications(totalUsers, siteAlertBatchSize=10000) {
+    const prisma = await db.prisma; // Await the prisma instance
+    const totalSites = totalUsers*5
+    let siteAlertBatch = [];
+    let notificationBatch = [];
+
+    const processBatch = async () => {
+        if (siteAlertBatch.length > 0) {
+            await prisma.siteAlert.createMany({ data: siteAlertBatch });
+            await prisma.notification.createMany({ data: notificationBatch });
+            siteAlertBatch = []; // Reset the siteAlert batch
+            notificationBatch = []; // Reset the notificatons batch
+        }
+    };
+    const twoMonthsAgo = new Date();
+    twoMonthsAgo.setMonth(twoMonthsAgo.getMonth() - 2);
+
+    for (let siteId = 1; siteId <= totalSites; siteId++) {
+        for (let i = 0; i < 300; i++) {
+            siteAlertBatch.push({
+                id: `${siteId}siteAlert${i}`,
+                confidence: 'high',
+                detectedBy: 'SEEDER',
+                distance: 0,
+                eventDate: getRandomDate(twoMonthsAgo, new Date()),
+                latitude: 10,
+                longitude: 15,
+                type: 'fire',
+                siteId: siteId.toString(),
+            })
+            if (i < 100) {
+                notificationBatch.push({
+                    id: `${siteId}notification${i}`,
+                    alertMethod: "email",
+                    destination: `siteId${siteId}${i}@plant-for-the-planet.org`,
+                    siteAlertId: `${siteId}siteAlert${i}`
+                })
+            }
+            if(siteAlertBatch.length >= siteAlertBatchSize){
+                await processBatch()
+            }
+        }
+    }
+    await processBatch()
+    console.log(`Successfully Seeded SiteAlerts and Notifications`);
+}
+module.exports.seedSiteAlertsAndNotifications = seedSiteAlertsAndNotifications;
+
+// seedSiteAlertsAndNotifications(2000)
+//     .then(() => {
+//         console.log('SiteAlerts and Notifications seeded successfully.');
+//         process.exit(0);
+//     })
+//     .catch(error => {
+//         console.error('Error seeding siteAlerts and notifications', error);
+//         process.exit(1);
+//     });