From ca2bd26671e46fb61533e9243c5d10f649b871ea Mon Sep 17 00:00:00 2001 From: baalmart Date: Sun, 24 Nov 2024 17:35:38 +0300 Subject: [PATCH 1/2] updating and creating new bulk scripts --- .../utils/scripts/api-response-transformer.js | 149 ++++++++++++++++++ .../utils/scripts/bulk-script.js | 20 ++- 2 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 src/device-registry/utils/scripts/api-response-transformer.js diff --git a/src/device-registry/utils/scripts/api-response-transformer.js b/src/device-registry/utils/scripts/api-response-transformer.js new file mode 100644 index 0000000000..9682ce8426 --- /dev/null +++ b/src/device-registry/utils/scripts/api-response-transformer.js @@ -0,0 +1,149 @@ +/** + * Transforms API response data into the required format for the second API + * @param {Object} inputData - The response from the first API + * @returns {Object} Formatted data for the second API + * + */ +const axios = require("axios"); +const isEmpty = require("is-empty"); + +function transformApiResponse(inputData) { + // Extract site-category data for direct mapping + const siteCategory = inputData.site["site-category"]; + + // Initialize the base structure with direct mappings + const result = { + area_name: siteCategory.area_name, + category: siteCategory.category, + highway: siteCategory.highway, + landuse: siteCategory.landuse, + latitude: siteCategory.latitude, + longitude: siteCategory.longitude, + natural: siteCategory.natural, + search_radius: siteCategory.search_radius, + waterway: siteCategory.waterway, + tags: [], + }; + + // Process OSM_info array to extract unique values + const osmInfoArray = inputData.site.OSM_info; + const uniqueValues = new Map(); + + // Helper function to clean and extract value from OSM info line + const extractValue = (line) => { + const [key, value] = line.trim().split(": "); + return { key: key.trim(), value: value.trim() }; + }; + + // Process each entry in OSM_info + for (let i = 0; i < osmInfoArray.length; i++) { + const line = osmInfoArray[i]; + + // Skip the "Found OSM data:" lines + if (line.includes("Found OSM data:")) continue; + + const { key, value } = extractValue(line); + + // Skip empty or "unknown" values + if (!value || value === "unknown") continue; + + // For Location, format it as a string + if (key === "Location") { + const locationStr = value.replace(/[()]/g, ""); + uniqueValues.set(key, locationStr); + } + // For other keys, store if not already present or if current value is more specific + else if ( + !uniqueValues.has(key) || + (uniqueValues.get(key) === "unknown" && value !== "unknown") + ) { + uniqueValues.set(key, value); + } + } + + // Convert unique values to tags array + uniqueValues.forEach((value, key) => { + // Format the tag as "key: value" + const tag = `${key}: ${value}`; + result.tags.push(tag); + }); + + return result; +} + +// Example usage: +const createRequestBody = (apiResponse) => { + try { + return transformApiResponse(apiResponse); + } catch (error) { + console.error("Error transforming API response:", error); + throw error; + } +}; + +// Make a GET request +const url = "http://localhost:3000/api/v2/devices/sites/summary"; +const config = { + headers: { + Authorization: "", + }, +}; +axios + .get(url, config) + .then((response) => { + console.log("GET response site name" + ": "); + console.dir(response.data); + for (let i = 0; i < response.data.sites.length; i += 10) { + const batch = response.data.sites.slice(i, i + 10); + // Process batch of 10 items + batch.forEach(async (site) => { + // console.log("the site _id", site._id); + const url = `http://localhost:3000/api/v2/spatial/categorize_site?latitude=${site.latitude}&longitude=${site.longitude}`; + + //***MAKE THE GET REQUEST */ + axios + .get(url, config) + .then((response) => { + // console.log("PUT response:", response.data); + const apiResponse = response.data; + const requestBody = createRequestBody(apiResponse); + // console.log("requestBody", requestBody); + const updateURL = `http://localhost:3000/api/v2/devices/sites?id=${site._id}`; + const data = { + site_category: requestBody, + }; + + // Make a PUT request + axios + .put(updateURL, data, config) + .then((response) => { + console.log("PUT response:", response.data); + }) + .catch((error) => { + if (error.response) { + console.log(error.response.status); + console.log(error.response.data); + } else { + console.log(error.message); + } + }); + }) + .catch((error) => { + if (error.response) { + console.log(error.response.status); + console.log(error.response.data); + } else { + console.log(error.message); + } + }); + }); + } + }) + .catch((error) => { + console.error("GET error:", error); + }); + +module.exports = { + createRequestBody, + transformApiResponse, +}; diff --git a/src/device-registry/utils/scripts/bulk-script.js b/src/device-registry/utils/scripts/bulk-script.js index 9a094c5971..2b403397f6 100644 --- a/src/device-registry/utils/scripts/bulk-script.js +++ b/src/device-registry/utils/scripts/bulk-script.js @@ -1,12 +1,24 @@ const axios = require("axios"); const isEmpty = require("is-empty"); +function extractBeforeV2IncludingSlash(url) { + const v2Index = url.indexOf("v2/"); + + if (v2Index !== -1) { + // Found "v2/" in the string + return url.substring(0, v2Index + 3); // Include "v2/" + } else { + // "v2/" not found, return the original string + return url; + } +} + /** * Update each device to have the default network of VISIBLE_DEVICES_ONLY */ // Make a GET request -const url = "http://localhost:3000/api/v1/devices"; +const url = "http://localhost:3000/api/v2/devices/summary"; const config = { headers: { Authorization: "", @@ -22,9 +34,11 @@ axios // Process batch of 10 items batch.forEach(async (device) => { // console.log("the device _id", device._id); - const url = `http://localhost:3000/api/v1/devices?id=${device._id}`; + const url = `http://localhost:3000/api/v2/devices/soft?id=${device._id}`; + const new_api_code = extractBeforeV2IncludingSlash(device.api_code); + const data = { - network: "airqo", + api_code: new_api_code, }; if (isEmpty(device.network)) { From f60c18451c65e4b765ad93f7a8cd8f6f6622174a Mon Sep 17 00:00:00 2001 From: baalmart Date: Sun, 24 Nov 2024 17:37:00 +0300 Subject: [PATCH 2/2] include site category data in Readings --- src/device-registry/models/Reading.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/device-registry/models/Reading.js b/src/device-registry/models/Reading.js index 6c4c441843..cd53f39c67 100644 --- a/src/device-registry/models/Reading.js +++ b/src/device-registry/models/Reading.js @@ -20,6 +20,29 @@ const HealthTipsSchema = new Schema( { _id: false } ); +const categorySchema = new Schema( + { + area_name: { type: String }, + category: { type: String }, + highway: { type: String }, + landuse: { type: String }, + latitude: { type: Number }, + longitude: { type: Number }, + natural: { type: String }, + search_radius: { type: Number }, + waterway: { type: String }, + tags: [ + { + type: String, + trim: true, + }, + ], + }, + { + _id: false, + } +); + const SiteDetailsSchema = new Schema( { _id: Schema.Types.ObjectId, @@ -42,6 +65,7 @@ const SiteDetailsSchema = new Schema( bearing_in_radians: Number, description: String, data_provider: String, + site_category: { type: categorySchema }, }, { _id: false } );