diff --git a/apps/server/prisma/migrations/20231219000000_stats/migration.sql b/apps/server/prisma/migrations/20231219000000_stats/migration.sql new file mode 100644 index 000000000..c520d18aa --- /dev/null +++ b/apps/server/prisma/migrations/20231219000000_stats/migration.sql @@ -0,0 +1,10 @@ +-- CreateTable +CREATE TABLE "Stats" ( + "id" TEXT NOT NULL, + "metric" TEXT NOT NULL, + "count" INTEGER NOT NULL, + "lastUpdated" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "Stats_pkey" PRIMARY KEY ("id"), + CONSTRAINT "Stats_metric_key" UNIQUE ("metric") +); diff --git a/apps/server/prisma/schema.prisma b/apps/server/prisma/schema.prisma index fd92121b9..e4864ef42 100644 --- a/apps/server/prisma/schema.prisma +++ b/apps/server/prisma/schema.prisma @@ -151,6 +151,14 @@ model Notification { isDelivered Boolean @default(false) } +model Stats { + id String @id @default(cuid()) + metric String @unique + count Int + lastUpdated DateTime +} + + enum Role { ROLE_CLIENT ROLE_ADMIN diff --git a/apps/server/src/pages/api/cron/db-cleanup.ts b/apps/server/src/pages/api/cron/db-cleanup.ts index 960b2f425..c7e0c51c2 100644 --- a/apps/server/src/pages/api/cron/db-cleanup.ts +++ b/apps/server/src/pages/api/cron/db-cleanup.ts @@ -11,6 +11,26 @@ export const config = { maxDuration: 60, }; +// Function to get unique values after combining two arrays +function getUniqueValuesInTwoArrays(array1:string[], array2:string[]) { + const combinedArray = [...array1, ...array2]; + return Array.from(new Set(combinedArray)); +} + + +// Function to update or create stats data +async function updateOrCreateStats(metric:string, count:number) { + await prisma.stats.upsert({ + where: { metric: metric }, + update: { count: { increment: count } }, + create: { + metric: metric, + count: count, + lastUpdated: new Date(), + }, + }); +} + // This cron will also help with GDPR compliance and data retention. export default async function dbCleanup(req: NextApiRequest, res: NextApiResponse) { @@ -24,104 +44,127 @@ export default async function dbCleanup(req: NextApiRequest, res: NextApiRespons } } - const promises = []; - - // Since our database follows cascade-on-delete constraint, when a User is deleted, - // it will cascade delete AlertMethod, Site, and Project. - // Also, deleting a Site will cascade delete SiteAlert, - // and deleting a SiteAlert will cascade delete Notification, so the db-cleanup follows this order - // to optimize the cleanup process. - - // item 1: - // delete all geo events that are older than 30 days and have been processed - promises.push(prisma.geoEvent.deleteMany({ - where: { - isProcessed: true, - eventDate: { - lt: new Date(new Date().getTime() - 30 * 24 * 60 * 60 * 1000) - } - } - })); - - // item 2: - // Delete all users who've requested to be deleted and have deletedAt date older than 7 days - // Send sendAccountDeletionConfirmationEmail to all users who qualify for deletion, - // and then delete them immediately - const usersToBeDeleted = - await prisma.user.findMany({ - where: { - deletedAt: { - lt: new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000) - } - } - }); - - // Send email to all users who qualify for deletion - for (const user of usersToBeDeleted) { - await sendAccountDeletionConfirmationEmail(user); - - // Note that we aren't deleting Auth0 Accounts using FireAlert. We should likely delete it if there is no remoteID set. - // Let's leave this as TODO for now. - // We have a n8n worker that deletes Auth0 accounts if sub is provided. - - logger(`USER DELETED: Sent account deletion confirmation email to ${user.id}`, 'info',); - } - - - // Delete all users who qualify for deletion - promises.push(prisma.user.deleteMany({ - where: { - deletedAt: { - lt: new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000) - } - } - })); - - // item 3: - // Delete all Sites that have been soft-deleted and have deletedAt date older than 30 days and doesn't have a remoteId - promises.push(prisma.site.deleteMany({ - where: { - deletedAt: { - lte: new Date(new Date().getTime() - 30 * 24 * 60 * 60 * 1000) - }, - remoteId: null - } - })); - - // item 4: - // Delete all AlertMethods that have been soft-deleteted for longer than 7 days - promises.push(prisma.alertMethod.deleteMany({ - where: { - deletedAt: { - lte: new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000) - } - } - })) - - // item 5: - // Delete all SiteAlerts that have deletedAt date older than 30 days - promises.push(prisma.siteAlert.deleteMany({ - where: { - eventDate: { - lte: new Date(new Date().getTime() - 30 * 24 * 60 * 60 * 1000) - } - } - })); - - // We do not delete notifications, as we will need notifications data in the future for further analysis + let total_delCount_user = 0; + let total_delCount_site = 0; + let total_delCount_alertMethod = 0; + let total_delCount_siteAlert = 0; + let total_delCount_notification = 0; + let total_delCount_geoEvents = 0; + let total_delCount_verificationRequest = 0; try { + await prisma.$transaction(async (prisma) => { - const [deletedGeoEvents, deletedUsers, deletedSites, deletedAlertMethods, deletedSiteAlerts] = - await Promise.all(promises); + let userCleanupDeletion_Ids: string[] = []; + let siteCleanupDeletion_Ids: string[] = []; + let alertMethodCleanupDeletion_Ids: string[] = []; + let alertMethodCascadeDeletion_Ids: string[] = []; + let siteCascadeDeletion_Ids: string[] = []; + + // Getting users to be deleted with their associated alertMethods and sites + const usersToBeDeleted = await prisma.user.findMany({ + where: { + deletedAt: { + lt: new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000) + } + }, + include: { + alertMethods: true, + sites: true + } + }); + + // Process each user for deletion + usersToBeDeleted.forEach(async user => { + user.alertMethods.forEach(alertMethod => { + alertMethodCascadeDeletion_Ids.push(alertMethod.id); + }); + const siteAlertPromises = user.sites.map(async site => { + siteCascadeDeletion_Ids.push(site.id); + + // Counting cascade-deleted siteAlerts and notifications for each site + const siteAlerts = await prisma.siteAlert.findMany({ + where: { siteId: site.id }, + include: { notifications: true } + }); + + siteAlerts.forEach(siteAlert => { + total_delCount_siteAlert++; + total_delCount_notification += siteAlert.notifications.length; + }); + }); + + await Promise.all(siteAlertPromises); + + // Adding user ID for deletion count + userCleanupDeletion_Ids.push(user.id); + sendAccountDeletionConfirmationEmail(user); + logger(`USER DELETED: Sent account deletion confirmation email to ${user.id}`, 'info',); + }); + + // Fetching expired site and alertMethod IDs + siteCleanupDeletion_Ids = (await prisma.site.findMany({ + where: { deletedAt: { lte: new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000) } }, + select: { id: true } + })).map(site => site.id); + + alertMethodCleanupDeletion_Ids = (await prisma.alertMethod.findMany({ + where: { deletedAt: { lte: new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000) } }, + select: { id: true } + })).map(am => am.id); + + total_delCount_site = getUniqueValuesInTwoArrays(siteCleanupDeletion_Ids, siteCascadeDeletion_Ids).length; + total_delCount_alertMethod = getUniqueValuesInTwoArrays(alertMethodCleanupDeletion_Ids, alertMethodCascadeDeletion_Ids).length; + + + // Calculating deletion counts + total_delCount_user = userCleanupDeletion_Ids.length; + + // Performing user, site and alertMethod deletions + await prisma.user.deleteMany({ where: { id: { in: userCleanupDeletion_Ids } } }); + await prisma.site.deleteMany({ where: { id: { in: siteCleanupDeletion_Ids } } }); + await prisma.alertMethod.deleteMany({ where: { id: { in: alertMethodCleanupDeletion_Ids } } }); + // Deleting old geoEvents and expired verificationRequests + const deletedGeoEvents = await prisma.geoEvent.deleteMany({ + where: { + eventDate: { + lt: new Date(new Date().getTime() - 30 * 24 * 60 * 60 * 1000) + } + } + }); + total_delCount_geoEvents = deletedGeoEvents.count + + const deletedVeificationRequests = await prisma.verificationRequest.deleteMany({ + where: { + expires: { + lt: new Date() + } + } + }); + total_delCount_verificationRequest = deletedVeificationRequests.count + + // Update stats table + await updateOrCreateStats('users_deleted', total_delCount_user); + await updateOrCreateStats('sites_deleted', total_delCount_site); + await updateOrCreateStats('alertMethods_deleted', total_delCount_alertMethod); + await updateOrCreateStats('siteAlerts_deleted', total_delCount_siteAlert); + await updateOrCreateStats('notifications_deleted', total_delCount_notification); + await updateOrCreateStats('geoEvents_deleted', total_delCount_geoEvents); + await updateOrCreateStats('verificationRequests_deleted', total_delCount_verificationRequest); + }); + // End of Prisma Transaction + + // Logging deletion counts logger(` - Deleted ${deletedGeoEvents.count} geo events that are older than 30 days and have been processed - Deleted ${deletedUsers.count} users who've requested to be deleted and have deletedAt date older than 7 days - Deleted ${deletedSites.count} soft-deleted Sites - Deleted ${deletedAlertMethods.count} soft-deleted AlertMethods - Deleted ${deletedSiteAlerts.count} soft-deleted SiteAlerts - `, 'info'); + Deleted ${total_delCount_geoEvents} geo events + Deleted ${total_delCount_user} users + Deleted ${total_delCount_site} sites + Deleted ${total_delCount_alertMethod} alert methods + Deleted ${total_delCount_verificationRequest} expired verification requests + Cascade Deleted ${total_delCount_siteAlert} site alerts + Cascade Deleted ${total_delCount_notification} notifications + `, 'info'); res.status(200).json({ message: "Success! Db is as clean as a whistle!", @@ -134,4 +177,4 @@ export default async function dbCleanup(req: NextApiRequest, res: NextApiRespons status: 500 }); } -} +} \ No newline at end of file diff --git a/apps/server/src/server/api/routers/site.ts b/apps/server/src/server/api/routers/site.ts index d28f5596d..f90cb5992 100644 --- a/apps/server/src/server/api/routers/site.ts +++ b/apps/server/src/server/api/routers/site.ts @@ -341,49 +341,49 @@ export const siteRouter = createTRPCRouter({ } }), - pauseAlertForSite: protectedProcedure - .input(pauseAlertInputSchema) - .mutation(async ({ctx, input}) => { - try { - // Destructure input parameters, including siteId - const {siteId, duration, unit} = input; + pauseAlertForSite: protectedProcedure + .input(pauseAlertInputSchema) + .mutation(async ({ctx, input}) => { + try { + // Destructure input parameters, including siteId + const {siteId, duration, unit} = input; - // Calculate the time for the stopAlertUntil field based on unit - const additionFactor = { - minutes: 1000 * 60, - hours: 1000 * 60 * 60, - days: 1000 * 60 * 60 * 24, - }; + // Calculate the time for the stopAlertUntil field based on unit + const additionFactor = { + minutes: 1000 * 60, + hours: 1000 * 60 * 60, + days: 1000 * 60 * 60 * 24, + }; - // Calculate future date based on current time, duration, and unit - const futureDate = new Date(Date.now() + duration * additionFactor[unit]); + // Calculate future date based on current time, duration, and unit + const futureDate = new Date(Date.now() + duration * additionFactor[unit]); - // Update specific site's stopAlertUntil field in the database - await ctx.prisma.site.update({ - where: { - id: siteId - }, - data: { - stopAlertUntil: futureDate - }, - }); - // Constructing a readable duration message - const durationUnit = unit === 'minutes' && duration === 1 ? 'minute' : - unit === 'hours' && duration === 1 ? 'hour' : - unit === 'days' && duration === 1 ? 'day' : unit; + // Update specific site's stopAlertUntil field in the database + await ctx.prisma.site.update({ + where: { + id: siteId + }, + data: { + stopAlertUntil: futureDate + }, + }); + // Constructing a readable duration message + const durationUnit = unit === 'minutes' && duration === 1 ? 'minute' : + unit === 'hours' && duration === 1 ? 'hour' : + unit === 'days' && duration === 1 ? 'day' : unit; - // Respond with a success message including pause duration details - return { - status: 'success', - message: `Alert has been successfully paused for the site for ${duration} ${durationUnit}.` - }; - } catch (error) { - throw new TRPCError({ - code: 'INTERNAL_SERVER_ERROR', - message: 'An error occurred while pausing the alert for the site', - }); - } - }), + // Respond with a success message including pause duration details + return { + status: 'success', + message: `Alert has been successfully paused for the site for ${duration} ${durationUnit}.` + }; + } catch (error) { + throw new TRPCError({ + code: 'INTERNAL_SERVER_ERROR', + message: 'An error occurred while pausing the alert for the site', + }); + } + }), deleteSite: protectedProcedure .input(params) @@ -409,7 +409,9 @@ export const siteRouter = createTRPCRouter({ deletedAt: new Date(), } }); - + // SiteAlert is automatically cascade deleted during db-cleanup, when the site gets permanently deleted, + // However, notifications are still created when siteAlert is not marked as soft-deleted + // Therefore, SiteAlerts need to be soft-deleted, as we want to stop creating notifications for soft-deleted sites await ctx.prisma.siteAlert.updateMany({ where: { siteId: input.siteId,