From 15936d63b89e0c715fdfe4e46b4a633a9982e1dc Mon Sep 17 00:00:00 2001 From: baalmart Date: Sat, 4 Nov 2023 11:50:11 +0300 Subject: [PATCH 01/13] scheduling daily SIM status checks --- src/incentives/bin/cronJob.js | 65 ++++++++++++++++++++++++++++++++ src/incentives/package-lock.json | 35 +++++++++++++++++ src/incentives/package.json | 1 + 3 files changed, 101 insertions(+) create mode 100644 src/incentives/bin/cronJob.js diff --git a/src/incentives/bin/cronJob.js b/src/incentives/bin/cronJob.js new file mode 100644 index 0000000000..89d7156b1d --- /dev/null +++ b/src/incentives/bin/cronJob.js @@ -0,0 +1,65 @@ +const cron = require("node-cron"); +const UserModel = require("@models/User"); +const constants = require("@config/constants"); +const inactiveThreshold = constants.INACTIVE_THRESHOLD || 2592000000; // 30 days +const log4js = require("log4js"); +const logger = log4js.getLogger( + `${constants.ENVIRONMENT} -- bin/cronJob script` +); + +// Everyday at midnight +cron.schedule("0 0 * * *", async () => { + try { + const batchSize = 100; // Process 100 users at a time + let skip = 0; + + while (true) { + const users = await UserModel("airqo") + .find({ + $or: [ + { + lastLogin: { + $lt: new Date(Date.now() - inactiveThreshold), + }, + }, + { + lastLogin: null, + }, + ], + isActive: { $ne: false }, // Exclude users where isActive is false + }) + .limit(batchSize) + .skip(skip) + .select("_id") + .lean(); + + if (users.length === 0) { + break; + } + + const userIds = users.map((user) => user._id); + + // Process the users with a delay between each batch + await processUsersWithDelay(userIds); + + skip += batchSize; + } + } catch (error) { + logger.error( + `An error occurred in the cron job --- ${JSON.stringify(error)}` + ); + } +}); + +async function processUsersWithDelay(userIds) { + const delay = 5000; // 5 seconds delay between requests + for (const userId of userIds) { + await updateUserAndDelay(userId, delay); + } +} + +async function updateUserAndDelay(userId, delay) { + await UserModel("airqo").updateMany({ _id: userId }, { isActive: false }); + + return new Promise((resolve) => setTimeout(resolve, delay)); +} diff --git a/src/incentives/package-lock.json b/src/incentives/package-lock.json index 53276b6267..1a05daf3f7 100644 --- a/src/incentives/package-lock.json +++ b/src/incentives/package-lock.json @@ -35,6 +35,7 @@ "mongoose-unique-validator": "^2.0.3", "morgan": "~1.9.0", "mtn-momo": "^2.0.0", + "node-cron": "^3.0.2", "node-fetch": "^2.6.1", "node-schedule": "^2.1.1", "nodemailer": "^6.9.4", @@ -5868,6 +5869,25 @@ "node": ">=4.0.0" } }, + "node_modules/node-cron": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.2.tgz", + "integrity": "sha512-iP8l0yGlNpE0e6q1o185yOApANRe47UPbLf4YxfbiNHt/RU5eBcGB/e0oudruheSf+LQeDMezqC5BVAb5wwRcQ==", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/node-cron/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -13209,6 +13229,21 @@ "resolved": "https://registry.npmjs.org/nocache/-/nocache-2.1.0.tgz", "integrity": "sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==" }, + "node-cron": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.2.tgz", + "integrity": "sha512-iP8l0yGlNpE0e6q1o185yOApANRe47UPbLf4YxfbiNHt/RU5eBcGB/e0oudruheSf+LQeDMezqC5BVAb5wwRcQ==", + "requires": { + "uuid": "8.3.2" + }, + "dependencies": { + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", diff --git a/src/incentives/package.json b/src/incentives/package.json index d1b87fcd22..4a09aad1e8 100644 --- a/src/incentives/package.json +++ b/src/incentives/package.json @@ -54,6 +54,7 @@ "mongoose-unique-validator": "^2.0.3", "morgan": "~1.9.0", "mtn-momo": "^2.0.0", + "node-cron": "^3.0.2", "node-fetch": "^2.6.1", "node-schedule": "^2.1.1", "nodemailer": "^6.9.4", From 844e47ef2c6b275e1e2a55350f9de020f433b74d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:46:06 +0300 Subject: [PATCH 02/13] Update device registry staging image tag to stage-86bad23b-1699224239 --- k8s/device-registry/values-stage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/device-registry/values-stage.yaml b/k8s/device-registry/values-stage.yaml index 151e6bdf55..993255a92b 100644 --- a/k8s/device-registry/values-stage.yaml +++ b/k8s/device-registry/values-stage.yaml @@ -6,7 +6,7 @@ app: replicaCount: 2 image: repository: eu.gcr.io/airqo-250220/airqo-stage-device-registry-api - tag: stage-c32fcdc5-1699219582 + tag: stage-86bad23b-1699224239 nameOverride: '' fullnameOverride: '' podAnnotations: {} From d77b717b324bab0feba6dc843076d0ce6934df96 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:46:34 +0300 Subject: [PATCH 03/13] Update AirQo exceedance production image tag to prod-58e0f4e4-1699224315 --- k8s/exceedance/values-prod-airqo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml index 9f49a6c1a4..adf4f99409 100644 --- a/k8s/exceedance/values-prod-airqo.yaml +++ b/k8s/exceedance/values-prod-airqo.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/airqo-exceedance-job - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 nameOverride: '' fullnameOverride: '' From c0422354401ec4a7d7b56a5fe18d20f31d646626 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:46:46 +0300 Subject: [PATCH 04/13] Update KCCA exceedance production image tag to prod-58e0f4e4-1699224315 --- k8s/exceedance/values-prod-kcca.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml index 3799021abe..f00814c371 100644 --- a/k8s/exceedance/values-prod-kcca.yaml +++ b/k8s/exceedance/values-prod-kcca.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/kcca-exceedance-job - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 nameOverride: '' fullnameOverride: '' From 0c7d2700fb1205562ca2588e44a83ade863b44cf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:47:06 +0300 Subject: [PATCH 05/13] Update incentives production image tag to prod-58e0f4e4-1699224315 --- k8s/incentives/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/incentives/values-prod.yaml b/k8s/incentives/values-prod.yaml index f38ce3629d..f26e3787f4 100644 --- a/k8s/incentives/values-prod.yaml +++ b/k8s/incentives/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-incentives-api - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 nameOverride: '' fullnameOverride: '' podAnnotations: {} From df38a6006db5ce9024071906b7570f5649bfad7c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:47:15 +0300 Subject: [PATCH 06/13] Update auth service production image tag to prod-58e0f4e4-1699224315 --- k8s/auth-service/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/auth-service/values-prod.yaml b/k8s/auth-service/values-prod.yaml index 69ae2be72f..389bc7540f 100644 --- a/k8s/auth-service/values-prod.yaml +++ b/k8s/auth-service/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-auth-api - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 0881662e988674b0d81dbf643dc32134d6b25b6f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:47:40 +0300 Subject: [PATCH 07/13] Update device registry production image tag to prod-58e0f4e4-1699224315 --- k8s/device-registry/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml index f18b0d5360..672e03af2c 100644 --- a/k8s/device-registry/values-prod.yaml +++ b/k8s/device-registry/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-device-registry-api - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 nameOverride: '' fullnameOverride: '' podAnnotations: {} From ab2703f99a75fa8f6a2bf8e89cc08dbf4e6c6cf9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:47:57 +0300 Subject: [PATCH 08/13] Update device monitor production images' tag to prod-58e0f4e4-1699224315 --- k8s/device-monitor/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/device-monitor/values-prod.yaml b/k8s/device-monitor/values-prod.yaml index 5fc943f853..213d0ba538 100644 --- a/k8s/device-monitor/values-prod.yaml +++ b/k8s/device-monitor/values-prod.yaml @@ -9,7 +9,7 @@ images: deviceMonitor: eu.gcr.io/airqo-250220/airqo-device-monitor-api celeryBeat: eu.gcr.io/airqo-250220/airqo-device-monitor-celery-beat celeryWorker: eu.gcr.io/airqo-250220/airqo-device-monitor-celery-worker - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 nameOverride: '' fullnameOverride: '' podAnnotations: {} From bf15ee9f8994c55ac0265e28b536d271cdb5e804 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:48:31 +0300 Subject: [PATCH 09/13] Update airflow prod image tag to prod-58e0f4e4-1699224315 --- k8s/airflow/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/airflow/values-prod.yaml b/k8s/airflow/values-prod.yaml index 17c8706f9a..3b00330c9e 100644 --- a/k8s/airflow/values-prod.yaml +++ b/k8s/airflow/values-prod.yaml @@ -9,7 +9,7 @@ images: repositories: initContainer: eu.gcr.io/airqo-250220/airqo-apache-airflow-xcom containers: eu.gcr.io/airqo-250220/airqo-apache-airflow - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 5ee92d255031511c6ade6d12f4233fa77c3176a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:49:27 +0300 Subject: [PATCH 10/13] Update predict production image tag to prod-58e0f4e4-1699224315 --- k8s/predict/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml index 3183599939..3e3173ae8d 100644 --- a/k8s/predict/values-prod.yaml +++ b/k8s/predict/values-prod.yaml @@ -7,7 +7,7 @@ images: predictJob: eu.gcr.io/airqo-250220/airqo-predict-job trainJob: eu.gcr.io/airqo-250220/airqo-train-job predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 api: name: airqo-prediction-api label: prediction-api From 71bbd635ec6bd5b873cbcac3c3abb5c8eb0f2f87 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:49:35 +0300 Subject: [PATCH 11/13] Update analytics production image tag to prod-58e0f4e4-1699224315 --- k8s/analytics/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml index 581bb12197..a452900b58 100644 --- a/k8s/analytics/values-prod.yaml +++ b/k8s/analytics/values-prod.yaml @@ -8,7 +8,7 @@ images: celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 api: name: airqo-analytics-api label: analytics-api From da59183af00bb922dc9cd4eaf50338325f353bab Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 01:49:46 +0300 Subject: [PATCH 12/13] Update gp-model production image tag to prod-58e0f4e4-1699224315 --- k8s/gp-model/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/gp-model/values-prod.yaml b/k8s/gp-model/values-prod.yaml index 4ad5f044a4..3e0d406bcf 100644 --- a/k8s/gp-model/values-prod.yaml +++ b/k8s/gp-model/values-prod.yaml @@ -6,7 +6,7 @@ app: namespace: production configMap: env-gp-model-production images: - tag: prod-6ea06af8-1699219770 + tag: prod-58e0f4e4-1699224315 repositories: cronJob: eu.gcr.io/airqo-250220/airqo-gp-model-job brokerConsumer: eu.gcr.io/airqo-250220/airqo-gp-model-broker-consumer From db58dec574bf6b02b502ca0dcb1c236921e89677 Mon Sep 17 00:00:00 2001 From: baalmart Date: Mon, 6 Nov 2023 02:55:07 +0300 Subject: [PATCH 13/13] cronjob for checking the status of the sim card --- src/incentives/bin/cronJob.js | 65 ++++++++++++++++------------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/src/incentives/bin/cronJob.js b/src/incentives/bin/cronJob.js index 89d7156b1d..5e54881746 100644 --- a/src/incentives/bin/cronJob.js +++ b/src/incentives/bin/cronJob.js @@ -1,46 +1,30 @@ const cron = require("node-cron"); -const UserModel = require("@models/User"); +const SimModel = require("@models/Sim"); const constants = require("@config/constants"); -const inactiveThreshold = constants.INACTIVE_THRESHOLD || 2592000000; // 30 days const log4js = require("log4js"); const logger = log4js.getLogger( `${constants.ENVIRONMENT} -- bin/cronJob script` ); +const checkStatus = require("@utils/create-sim").checkStatus; +const secondsDelayBetweenRequests = 20000; +const internetDataBalanceThreshold = 5; // Everyday at midnight cron.schedule("0 0 * * *", async () => { try { - const batchSize = 100; // Process 100 users at a time + const batchSize = 100; // Process 100 SIM cards at a time let skip = 0; + const simCards = await SimModel("airqo").find({}).select("_id").lean(); + while (true) { - const users = await UserModel("airqo") - .find({ - $or: [ - { - lastLogin: { - $lt: new Date(Date.now() - inactiveThreshold), - }, - }, - { - lastLogin: null, - }, - ], - isActive: { $ne: false }, // Exclude users where isActive is false - }) - .limit(batchSize) - .skip(skip) - .select("_id") - .lean(); + const simBatch = simCards.slice(skip, skip + batchSize); - if (users.length === 0) { + if (simBatch.length === 0) { break; } - const userIds = users.map((user) => user._id); - - // Process the users with a delay between each batch - await processUsersWithDelay(userIds); + await processSimCardsWithDelay(simBatch); skip += batchSize; } @@ -51,15 +35,24 @@ cron.schedule("0 0 * * *", async () => { } }); -async function processUsersWithDelay(userIds) { - const delay = 5000; // 5 seconds delay between requests - for (const userId of userIds) { - await updateUserAndDelay(userId, delay); +async function processSimCardsWithDelay(simBatch) { + for (const sim of simBatch) { + const responseFromCheckStatus = await checkStatus({ + query: { tenant: "airqo" }, + params: { sim_id: sim._id }, + }); + + // Check if data.balance is less than the declared threshold and log it + if ( + responseFromCheckStatus.success && + responseFromCheckStatus.data.balance < internetDataBalanceThreshold + ) { + logger.info( + `SIM card ${sim._id} has a balance less than ${internetDataBalanceThreshold}` + ); + } + await new Promise((resolve) => + setTimeout(resolve, secondsDelayBetweenRequests) + ); } } - -async function updateUserAndDelay(userId, delay) { - await UserModel("airqo").updateMany({ _id: userId }, { isActive: false }); - - return new Promise((resolve) => setTimeout(resolve, delay)); -}