Implement simple way to build gltf with modification

app/api/geolocalisations/geolocalisations.controller.js

@@ -126,6 +126,7 @@ exports.save = route(async (req, res) => {
   const previous_geoloc_id = data.previous_geoloc_id;
   const remark = data.remark;
   const errors_list = data.errors_list;
+  const image_modifiers = data.image_modifiers;
 
   let georeferencer_id = user_id; // in case of improvements, should be the original georeferencer id (l127)
 
@@ -189,7 +190,8 @@ exports.save = route(async (req, res) => {
       date_georef: models.sequelize.literal("now()"),
       geolocalisation_id: geoloc_id,
       state: "waiting_validation",
-      framing_mode: framing_mode
+      framing_mode: framing_mode,
+      image_modifiers: image_modifiers
     },
     {
       where: { id: image_id }

app/api/geolocalisations/schemas/save-geolocalisation.req.schema.json

@@ -150,6 +150,25 @@
     },
     "previous_geoloc_id": {
       "$ref": "ApiId#"
+    },
+    "image_modifiers": {
+      "type": "object",
+      "properties": {
+        "modifier": {
+          "type": "integer"
+        },
+        "imageSize": {
+          "type": "object",
+          "properties": {
+            "width": {
+              "type": "integer"
+            },
+            "height": {
+              "type": "integer"
+            }
+          }
+        }
+      }
     }
   },
   "additionalProperties": false,

app/api/gltf/gltf.controller.js

@@ -121,12 +121,16 @@ async function getSquareImageFromDB(image_id, regionByPx) {
   return image;
 }
 
-async function createGltfFromImageCoordinates(imageCoordinates, image_id, collection_id, regionByPx) {
+async function createGltfFromImageCoordinates(imageCoordinates, image_id, collection_id, regionByPx, path2image) {
   //regionByPx = iiif_data.regionByPx, 
   //excepted for composite_images when computing pose during geolocalisation process (= pose-estimation.controller.js "/pose/compute").
 
   const imageSquaredFromDB = await getSquareImageFromDB(image_id, regionByPx);
-  const picPath = imageSquaredFromDB.media.image_url;
+  let picPath = imageSquaredFromDB.media.image_url;
+  if (path2image) {
+    picPath = path2image;
+  }
+
   const region_url = regionByPx ? `_${regionByPx[0]}_${regionByPx[1]}_${regionByPx[2]}_${regionByPx[3]}` : "";
 
   const urCorner = [imageCoordinates.ur[0], imageCoordinates.ur[1], imageCoordinates.ur[2]];
@@ -215,4 +219,5 @@ async function copyGltf(image_id, collection_id, region_url) {
 }
 
 // exports.createGltf = createGltf;
-exports.createGltfFromImageCoordinates = createGltfFromImageCoordinates
+exports.createGltfFromImageCoordinates = createGltfFromImageCoordinates;
+exports.getSquareImageFromDB = getSquareImageFromDB;

app/api/images/images.controller.js

@@ -208,6 +208,7 @@ exports.getAttributes = utils.route(async (req, res) => {
     'iiif_data',
     'country_iso_a2',
     'framing_mode',
+    'image_modifiers',
     [models.sequelize.literal("ST_X(images.location)"), "longitude"],
     [models.sequelize.literal("ST_Y(images.location)"), "latitude"],
     [models.sequelize.literal("ST_Z(images.location)"), "altitude"],

app/api/pose-estimation/pose-estimation.controller.js

@@ -10,6 +10,11 @@ const { poseEstimationError } = require('../../utils/errors');
 const { route, getLogger } = require("../../utils/express");
 const mediaUtils = require('../../utils/media');
 
+const fs = require("fs");
+
+const cv = require("@techstark/opencv-js");
+const Jimp = require('jimp');
+
 async function getDbImage(image_id){
   const query = models.images.findOne({
     raw: true,
@@ -155,64 +160,106 @@ exports.computePoseCreateGltf = route(async (req, res) => {
    const tilt = parseFloat(req.body.tilt);
    const roll = parseFloat(req.body.roll);
    const id = parseInt(req.body.image_id);
+   const imageModifier = req.body.image_modifiers;
+
+   // The image used to compute in background has a width of 1024px where the image use by the slider in fron has a width of 500px.
+   // The conversion is done to apply the same modification here as the front.
+   const convertedModifier = imageModifier.modifier * (1024/500);
+
    let regionByPx = req.body.regionByPx; //get region from front-end for composite_images
 
-   // Get image collection id and region
-   const sql = `
-       SELECT collection_id, iiif_data->'regionByPx' AS regionbypx
-       FROM images
-       WHERE id = ${id}
-   `;
-   const queryCollectionIdPromise = await models.sequelize.query(sql, {
-     type: models.sequelize.QueryTypes.SELECT
-   });
-   const collection_id = queryCollectionIdPromise[0].collection_id;
-   if (!regionByPx) {
-     regionByPx = queryCollectionIdPromise[0].regionbypx; //for image without cumulative_views, take the region from the iiif_data
-   }
-
-   // Compute Pose
-   // ------------
-    const gcpArrayString = JSON.stringify(GCPs)
-    let lock = 0
-    if (locationLocked){
-      lock=1
-    }else{
-      lock=0
+    // Get image collection id and region
+    const sql = `
+        SELECT collection_id, iiif_data->'regionByPx' AS regionbypx
+        FROM images
+        WHERE id = ${id}
+    `;
+    const queryCollectionIdPromise = await models.sequelize.query(sql, {
+      type: models.sequelize.QueryTypes.SELECT
+    });
+    const collection_id = queryCollectionIdPromise[0].collection_id;
+    if (!regionByPx) {
+      regionByPx = queryCollectionIdPromise[0].regionbypx; //for image without cumulative_views, take the region from the iiif_data
     }
 
-    let results;
-    try {
-      results = await computeCameraPose(longitude, latitude, altitude, azimuth, tilt, roll, gcpArrayString, width, height, lock)
-    } catch(error) {
-      getLogger().error(error);
-      throw poseEstimationError(req);
+    const path2collections = "/data/collections/";
+    const path2image = `${path2collections}${
+      collection_id}/images/output/${id}.png`;
+    let srcImage = `${config.apiUrl}${path2collections}${collection_id}/images/1024/${id}.jpg`
+
+    if (!fs.existsSync(srcImage)) {
+      const imageSquaredFromDB = await gltf.getSquareImageFromDB(id, regionByPx);
+      srcImage = imageSquaredFromDB.media.image_url;
     }
 
-    if (!results) {
-      throw poseEstimationError(req);
-    } else {
+    let jimpSrc = await Jimp.read(srcImage);
+    var src = cv.matFromImageData(jimpSrc.bitmap);
+    let dst = new cv.Mat();
+    let texture = new cv.Mat();
+    let dsize = new cv.Size(src.cols, src.rows);
+    let srcTri = cv.matFromArray(4, 1, cv.CV_32FC2, [0, 0, dsize.width, 0, 0, dsize.height, dsize.width, dsize.height]);
+    let dstTri;
+
+    if (convertedModifier > 0)
+      dstTri = cv.matFromArray(4, 1, cv.CV_32FC2, [convertedModifier, 0, dsize.width-convertedModifier, 0, 0, dsize.height, dsize.width, dsize.height]);
+    else
+      dstTri = cv.matFromArray(4, 1, cv.CV_32FC2, [0, 0, dsize.width, 0, Math.abs(convertedModifier), dsize.height, dsize.width-Math.abs(convertedModifier), dsize.height]);
+    let M = cv.getPerspectiveTransform(srcTri, dstTri);
+
+    cv.warpPerspective(src, dst, M, dsize);
+    cv.flip(dst,texture,0);
+    let resized_image = new cv.Mat();
+    cv.resize(dst, resized_image,new cv.Size(imageModifier.imageSize.width, imageModifier.imageSize.height), 0, 0, cv.INTER_AREA);
+    new Jimp({
+      width: resized_image.cols,
+      height: resized_image.rows,
+      data: Buffer.from(resized_image.data)
+      })
+      .write(path2image);
 
-     const {imageCoordinatesForGltf, ...filteredResults } = results;
+    // Compute Pose
+    // ------------
+      const gcpArrayString = JSON.stringify(GCPs)
+      let lock = 0
+      if (locationLocked){
+        lock=1
+      }else{
+        lock=0
+      }
 
-     // Compute surface covered with gcps
-     const ratio = computeGCPRatio(GCPs, width, height);
-     filteredResults.gcpPercentSurface = ratio;
+      let results;
+      try {
+        results = await computeCameraPose(longitude, latitude, altitude, azimuth, tilt, roll, gcpArrayString, width, height, lock)
+      } catch(error) {
+        getLogger().error(error);
+        throw poseEstimationError(req);
+      }
 
-     //Add region and url_gltf in results
-     filteredResults.image_id = id;
-     filteredResults.regionByPx = regionByPx;
+      if (!results) {
+        throw poseEstimationError(req);
+      } else {
 
-     //Build gltf_url
-     filteredResults.gltf_url = mediaUtils.generateGltfUrl(id, collection_id, regionByPx);
-     try {
-       await gltf.createGltfFromImageCoordinates(imageCoordinatesForGltf, id, collection_id, regionByPx)
-       res.status(201).send(filteredResults);
-     } catch (error) {
-      getLogger().error(error);
-      throw poseEstimationError(req, req.__('pose.3dModelCreationError'));
+      const {imageCoordinatesForGltf, ...filteredResults } = results;
+
+      // Compute surface covered with gcps
+      const ratio = computeGCPRatio(GCPs, width, height);
+      filteredResults.gcpPercentSurface = ratio;
+
+      //Add region and url_gltf in results
+      filteredResults.image_id = id;
+      filteredResults.regionByPx = regionByPx;
+
+      //Build gltf_url
+      filteredResults.gltf_url = mediaUtils.generateGltfUrl(id, collection_id, regionByPx);
+      try {
+        await gltf.createGltfFromImageCoordinates(imageCoordinatesForGltf, id, collection_id, regionByPx, path2image)
+        res.status(201).send(filteredResults);
+      } catch (error) {
+        getLogger().error(error);
+        throw poseEstimationError(req, req.__('pose.3dModelCreationError'));
+      }
     }
-  }
+   
 });
 
 // Get parameters sent by the client

app/api/pose-estimation/schemas/compute-pose.req.schema.json

@@ -47,6 +47,25 @@
         "type": "object"
       }
     },
+    "image_modifiers": {
+      "type": "object",
+      "properties": {
+        "modifier": {
+          "type": "integer"
+        },
+        "imageSize": {
+          "type": "object",
+          "properties": {
+            "width": {
+              "type": "integer"
+            },
+            "height": {
+              "type": "integer"
+            }
+          }
+        }
+      }
+    },
     "regionByPx": {
       "type": "array",
       "items": {

app/models/images.js

@@ -290,6 +290,9 @@ module.exports = (sequelize, DataTypes) => {
         // type of framing_mode (single_image, composite_image)
         type: DataTypes.TEXT,
         allowNull: false
+      },
+      image_modifiers: {
+        type: DataTypes
       }
     },
     {

db/migrations/20240507091300-update-image-with-modifier.js

@@ -0,0 +1,15 @@
+'use strict';
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('images', 'image_modifiers', {
+      type: Sequelize.DataTypes.JSON
+    });
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    await queryInterface.removeColumn('images', 'image_modifiers', {
+      type: Sequelize.DataTypes.JSON
+    });
+  }
+};