Allow to label images.

src/api/route.rs

@@ -6,10 +6,10 @@ use crate::api::schema::{
     GetEntityColorMapResponse, GetGraphResponse, GetPromptResponse, GetPublicationsResponse,
     GetPublicationsSummaryResponse, GetRecordsResponse, GetRelationCountResponse,
     GetStatisticsResponse, GetWholeTableResponse, NodeIdsQuery, Pagination, PaginationQuery,
-    PostResponse, PredictedNodeQuery, PromptList, SubgraphIdQuery,
+    PostResponse, PredictedNodeQuery, PromptList, SubgraphIdQuery, UploadImage,
 };
 use crate::model::core::{
-    Configuration, Entity, Entity2D, EntityCuration, EntityMetadata, EntityMetadataCuration,
+    Configuration, Entity, Entity2D, EntityCuration, EntityMetadata, EntityMetadataCuration, Image,
     KeySentenceCuration, KnowledgeCuration, RecordResponse, Relation, RelationCount,
     RelationMetadata, Statistics, Subgraph, WebpageMetadata,
 };
@@ -22,10 +22,14 @@ use crate::model::llm::{ChatBot, Context, LlmResponse, PROMPTS};
 use crate::model::publication::Publication;
 use crate::model::util::match_color;
 use crate::query_builder::cypher_builder::{query_nhops, query_shared_nodes};
-use crate::query_builder::sql_builder::{get_all_field_pairs, make_order_clause_by_pairs, ComposeQuery};
+use crate::query_builder::sql_builder::{
+    get_all_field_pairs, make_order_clause_by_pairs, ComposeQuery,
+};
 use log::{debug, info, warn};
+
 use poem::web::Data;
 use poem_openapi::{param::Path, param::Query, payload::Json, OpenApi};
+use std::path::PathBuf;
 use std::sync::Arc;
 use validator::Validate;
 
@@ -2042,6 +2046,82 @@ impl BiomedgpsApi {
         }
     }
 
+    /// Call `/api/v1/key-sentence-curations/:id/images` with payload to add an image to a key sentence curation.
+    #[oai(
+        path = "/key-sentence-curations/:id/images",
+        method = "post",
+        tag = "ApiTags::KnowledgeGraph",
+        operation_id = "postKeySentenceCurationImage"
+    )]
+    async fn post_key_sentence_curation_image(
+        &self,
+        pool: Data<&Arc<sqlx::PgPool>>,
+        id: Path<i64>,
+        upload_image: UploadImage,
+        _token: CustomSecurityScheme,
+    ) -> PostResponse<KeySentenceCuration> {
+        let pool_arc = pool.clone();
+        let id = id.0;
+        let username = _token.0.username;
+
+        if id < 0 {
+            let err = format!("Invalid id: {}", id);
+            warn!("{}", err);
+            return PostResponse::bad_request(err);
+        }
+
+        let destdir = match std::env::var("UPLOAD_DIR") {
+            Ok(upload_dir) => PathBuf::from(upload_dir),
+            Err(e) => {
+                let err = format!("Failed to get upload directory: {}", e);
+                warn!("{}", err);
+                return PostResponse::bad_request(err);
+            }
+        };
+
+        let image = upload_image.image;
+        let filename = match image.file_name() {
+            Some(filename) => filename.to_string(),
+            None => "".to_string(),
+        };
+
+        let mime_type = match image.content_type() {
+            Some(mime) => mime.to_string(),
+            None => {
+                let err = format!("Failed to get image content type");
+                warn!("{}", err);
+                return PostResponse::bad_request(err);
+            }
+        };
+
+        let image_bytes = match image.into_vec().await {
+            Ok(image_bytes) => image_bytes,
+            Err(e) => {
+                let err = format!("Failed to get image bytes: {}", e);
+                warn!("{}", err);
+                return PostResponse::bad_request(err);
+            }
+        };
+
+        let image = match Image::upload(&destdir, &filename, &image_bytes, &mime_type, &upload_image.raw_image_url, &upload_image.raw_image_src) {
+            Ok(image) => image,
+            Err(e) => {
+                let err = format!("Failed to upload image: {}", e);
+                warn!("{}", err);
+                return PostResponse::bad_request(err);
+            }
+        };
+
+        match KeySentenceCuration::add_image_to_payload(&pool_arc, id, &username, &image).await {
+            Ok(ksc) => return PostResponse::created(ksc),
+            Err(e) => {
+                let err = format!("Failed to add image to key sentence curation: {}", e);
+                warn!("{}", err);
+                return PostResponse::bad_request(err);
+            }
+        }
+    }
+
     /// Call `/api/v1/key-sentence-curations` with payload to delete a key sentence curation.
     #[oai(
         path = "/key-sentence-curations",

src/api/schema.rs

@@ -7,7 +7,7 @@ use crate::model::graph::Graph;
 use crate::model::graph::{COMPOSED_ENTITIES_REGEX, COMPOSED_ENTITY_REGEX, RELATION_TYPE_REGEX};
 use log::warn;
 use poem_openapi::Object;
-use poem_openapi::{payload::Json, ApiResponse, Tags};
+use poem_openapi::{payload::Json, ApiResponse, Tags, Multipart, types::multipart::Upload};
 use serde::{Deserialize, Serialize};
 use validator::Validate;
 use validator::ValidationErrors;
@@ -676,3 +676,11 @@ impl PaginationQuery {
         Ok(pagination)
     }
 }
+
+#[derive(Multipart)]
+pub struct UploadImage {
+    pub raw_image_url: String,
+    pub raw_image_src: String,
+    pub name: String,
+    pub image: Upload,
+}

src/model/core.rs

@@ -4,14 +4,16 @@ use super::graph::COMPOSED_ENTITY_DELIMITER;
 use super::kge::get_entity_emb_table_name;
 use super::util::{get_delimiter, parse_csv_error, ValidationError};
 use std::collections::HashMap;
+use std::fs::File;
+use std::io::Write;
 // use crate::model::util::match_color;
 use crate::query_builder::sql_builder::ComposeQuery;
 use anyhow::Ok as AnyOk;
 use chrono::serde::ts_seconds;
 use chrono::{DateTime, Utc};
 use lazy_static::lazy_static;
 use log::{debug, info};
-use poem_openapi::types::Type;
+use sha2::{Digest, Sha256};
 use poem_openapi::Object;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
@@ -1535,6 +1537,64 @@ impl CheckData for KeySentenceCuration {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Object, sqlx::FromRow, Validate)]
+pub struct Image {
+    pub raw_image_url: String,
+    pub raw_image_src: String,
+    pub image_path: String,
+    pub filename: String,
+    pub mime_type: String,
+    pub checksum: String,
+    pub created_at: DateTime<Utc>,
+}
+
+impl Image {
+    pub fn upload(destdir: &PathBuf, filename: &str, image_bytes: &Vec<u8>, mime_type: &str, raw_image_url: &str, raw_image_src: &str) -> Result<Image, anyhow::Error> {
+        let hasher = Sha256::new();
+        let checksum = hasher.chain_update(image_bytes).finalize();
+        let suffix = filename.split('.').last().unwrap_or("jpg");
+        let checksum_string = format!("{:x}.{}", checksum, suffix);
+
+        // Split the checksum string to make several subfolders
+        let mut subfolders = Vec::new();
+
+        for (i, c) in checksum_string.chars().enumerate() {
+            if i < 3 {
+                subfolders.push(c.to_string());
+            } else {
+                break;
+            }
+        }
+
+        let subdir = subfolders.join("/");
+        let filepath = destdir.join(&subdir).join(&checksum_string);
+
+        if !filepath.exists() {
+            // Make sure the parent directories exist
+            let parent_dir = filepath.parent().unwrap();
+            if !parent_dir.exists() {
+                std::fs::create_dir_all(parent_dir)?;
+            }
+
+            let mut f = File::create(&filepath)?;
+            f.write_all(image_bytes)?;
+        }
+
+        let image = Image {
+            image_path: filepath.to_string_lossy().to_string().replace(destdir.to_str().unwrap(), ""),
+            filename: filename.to_string(),
+            checksum: checksum_string,
+            mime_type: mime_type.to_string(),
+            created_at: Utc::now(),
+            raw_image_url: raw_image_url.to_string(),
+            raw_image_src: raw_image_src.to_string(),
+        };
+
+        Ok(image)
+    }
+}
+
+
 impl KeySentenceCuration {
     pub fn update_curator(&mut self, curator: &str) {
         self.curator = curator.to_string();
@@ -1652,6 +1712,43 @@ impl KeySentenceCuration {
         }
     }
 
+    pub async fn add_image_to_payload(pool: &sqlx::PgPool, id: i64, curator: &str, image: &Image) -> Result<KeySentenceCuration, anyhow::Error> {
+        let sql_str = "
+            UPDATE biomedgps_key_sentence_curation 
+            SET payload = jsonb_set(
+                payload,
+                '{images}',
+                COALESCE(
+                    CASE 
+                        WHEN payload->'images' @> jsonb_build_array(jsonb_build_object('checksum', $3))
+                        THEN payload->'images'
+                        ELSE (COALESCE(payload->'images', '[]'::jsonb)) || jsonb_build_array(jsonb_build_object('filename', $1, 'image_path', $2, 'checksum', $3, 'mime_type', $4, 'created_at', $5, 'raw_image_url', $6, 'raw_image_src', $7))
+                    END,
+                    '[]'::jsonb
+                ),
+                true
+            ) 
+            WHERE id = $8 AND curator = $9
+            RETURNING *;
+        ";
+
+        let key_sentence_curation = sqlx::query_as::<_, KeySentenceCuration>(sql_str)
+            .bind(&image.filename)
+            .bind(&image.image_path)
+            .bind(&image.checksum)
+            .bind(&image.mime_type)
+            .bind(&image.created_at)
+            .bind(&image.raw_image_url)
+            .bind(&image.raw_image_src)
+            .bind(id)
+            .bind(curator)
+            .fetch_one(pool)
+            .await?;
+
+        info!("Add image to key sentence curation: {:?} {:?}", key_sentence_curation, image);
+        AnyOk(key_sentence_curation)
+    }
+
     pub async fn insert(&self, pool: &sqlx::PgPool) -> Result<KeySentenceCuration, anyhow::Error> {
         let sql_str = "SELECT * FROM biomedgps_key_sentence_curation WHERE fingerprint = $1 AND curator = $2 AND key_sentence = $3";
         let record = sqlx::query_as::<_, KeySentenceCuration>(sql_str)

studio/src/services/swagger/KnowledgeGraph.ts

@@ -523,6 +523,50 @@ export async function deleteKeySentenceCuration(
   });
 }
 
+/** Call `/api/v1/key-sentence-curations/:id/images` with payload to add an image to a key sentence curation. POST /api/v1/key-sentence-curations/${param0}/images */
+export async function postKeySentenceCurationImage(
+  // 叠加生成的Param类型 (非body参数swagger默认没有生成对象)
+  params: swagger.postKeySentenceCurationImageParams,
+  body: {
+    raw_image_url: string;
+    raw_image_src: string;
+    name: string;
+  },
+  image?: File,
+  options?: { [key: string]: any },
+) {
+  const { id: param0, ...queryParams } = params;
+  const formData = new FormData();
+
+  if (image) {
+    formData.append('image', image);
+  }
+
+  Object.keys(body).forEach((ele) => {
+    const item = (body as any)[ele];
+
+    if (item !== undefined && item !== null) {
+      if (typeof item === 'object' && !(item instanceof File)) {
+        if (item instanceof Array) {
+          item.forEach((f) => formData.append(ele, f || ''));
+        } else {
+          formData.append(ele, JSON.stringify(item));
+        }
+      } else {
+        formData.append(ele, item);
+      }
+    }
+  });
+
+  return request<swagger.KeySentenceCuration>(`/api/v1/key-sentence-curations/${param0}/images`, {
+    method: 'POST',
+    params: { ...queryParams },
+    data: formData,
+    requestType: 'form',
+    ...(options || {}),
+  });
+}
+
 /** Call `/api/v1/llm` with query params to get answer from LLM. POST /api/v1/llm */
 export async function askLlm(
   // 叠加生成的Param类型 (非body参数swagger默认没有生成对象)

studio/src/services/swagger/typings.d.ts

@@ -647,6 +647,10 @@ declare namespace swagger {
     go_classifiers: GoClassifier[];
   };
 
+  type postKeySentenceCurationImageParams = {
+    id: number;
+  };
+
   type Price = {
     description: string;
     cost: Cost;