From 6741380effd7be7f24e5865ad4af6b0f7af90c53 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 31 Jan 2025 11:06:59 +1300 Subject: [PATCH 1/4] Add empty processor --- .../src/db_diesel/key_value_store.rs | 1 + .../add_load_plugin_processor_pg_enum_type.rs | 22 +++++++++++++ .../repository/src/migrations/v2_06_00/mod.rs | 2 ++ .../src/processors/general_processor.rs | 4 ++- .../src/processors/load_plugin/load_plugin.rs | 31 +++++++++++++++++++ .../service/src/processors/load_plugin/mod.rs | 2 ++ server/service/src/processors/mod.rs | 1 + 7 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 server/repository/src/migrations/v2_06_00/add_load_plugin_processor_pg_enum_type.rs create mode 100644 server/service/src/processors/load_plugin/load_plugin.rs create mode 100644 server/service/src/processors/load_plugin/mod.rs diff --git a/server/repository/src/db_diesel/key_value_store.rs b/server/repository/src/db_diesel/key_value_store.rs index f6431c4e96..444ac01c4b 100644 --- a/server/repository/src/db_diesel/key_value_store.rs +++ b/server/repository/src/db_diesel/key_value_store.rs @@ -28,6 +28,7 @@ pub enum KeyType { ShipmentTransferProcessorCursor, RequisitionTransferProcessorCursor, ContactFormProcessorCursor, + LoadPluginProcessorCursor, SettingsSyncUrl, SettingsSyncUsername, diff --git a/server/repository/src/migrations/v2_06_00/add_load_plugin_processor_pg_enum_type.rs b/server/repository/src/migrations/v2_06_00/add_load_plugin_processor_pg_enum_type.rs new file mode 100644 index 0000000000..bed73b4625 --- /dev/null +++ b/server/repository/src/migrations/v2_06_00/add_load_plugin_processor_pg_enum_type.rs @@ -0,0 +1,22 @@ +use crate::migrations::*; + +pub(crate) struct Migrate; + +impl MigrationFragment for Migrate { + fn identifier(&self) -> &'static str { + "add_load_plugin_processor_pg_enum_type" + } + + fn migrate(&self, connection: &StorageConnection) -> anyhow::Result<()> { + if cfg!(feature = "postgres") { + sql!( + connection, + r#" + ALTER TYPE key_type ADD VALUE IF NOT EXISTS 'LOAD_PLUGIN_PROCESSOR_CURSOR'; + "# + )?; + } + + Ok(()) + } +} diff --git a/server/repository/src/migrations/v2_06_00/mod.rs b/server/repository/src/migrations/v2_06_00/mod.rs index 19a5f0c518..1f1aab7c1e 100644 --- a/server/repository/src/migrations/v2_06_00/mod.rs +++ b/server/repository/src/migrations/v2_06_00/mod.rs @@ -2,6 +2,7 @@ use super::{version::Version, Migration, MigrationFragment}; mod add_create_invoice_from_requisition_permission; mod add_index_to_sync_buffer; +mod add_load_plugin_processor_pg_enum_type; mod add_name_next_of_kin_id; mod add_program_deleted_datetime; mod backend_plugins; @@ -26,6 +27,7 @@ impl Migration for V2_06_00 { Box::new(backend_plugins::Migrate), Box::new(add_create_invoice_from_requisition_permission::Migrate), Box::new(add_name_next_of_kin_id::Migrate), + Box::new(add_load_plugin_processor_pg_enum_type::Migrate), ] } } diff --git a/server/service/src/processors/general_processor.rs b/server/service/src/processors/general_processor.rs index 5d7e0154ee..265972d4da 100644 --- a/server/service/src/processors/general_processor.rs +++ b/server/service/src/processors/general_processor.rs @@ -11,7 +11,7 @@ use crate::{ service_provider::{ServiceContext, ServiceProvider}, }; -use super::contact_form::QueueContactEmailProcessor; +use super::{contact_form::QueueContactEmailProcessor, load_plugin::LoadPlugin}; #[derive(Error, Debug)] pub(crate) enum ProcessorError { @@ -28,12 +28,14 @@ const CHANGELOG_BATCH_SIZE: u32 = 20; #[derive(Clone)] pub enum ProcessorType { ContactFormEmail, + LoadPlugin, } impl ProcessorType { pub(super) fn get_processor(&self) -> Box { match self { ProcessorType::ContactFormEmail => Box::new(QueueContactEmailProcessor), + ProcessorType::LoadPlugin => Box::new(LoadPlugin), } } } diff --git a/server/service/src/processors/load_plugin/load_plugin.rs b/server/service/src/processors/load_plugin/load_plugin.rs new file mode 100644 index 0000000000..4740ac0a5e --- /dev/null +++ b/server/service/src/processors/load_plugin/load_plugin.rs @@ -0,0 +1,31 @@ +use repository::{ChangelogRow, ChangelogTableName, KeyType, StorageConnection}; + +use crate::processors::general_processor::{Processor, ProcessorError}; + +const DESCRIPTION: &str = "Load plugins"; + +pub(crate) struct LoadPlugin; + +impl Processor for LoadPlugin { + fn get_description(&self) -> String { + DESCRIPTION.to_string() + } + + /// Only runs once because contact form is create only + /// Changelog will only be processed once + fn try_process_record( + &self, + connection: &StorageConnection, + changelog: &ChangelogRow, + ) -> Result, ProcessorError> { + Ok(Some("success".to_string())) + } + + fn change_log_table_names(&self) -> Vec { + vec![ChangelogTableName::BackendPlugin] + } + + fn cursor_type(&self) -> KeyType { + KeyType::ContactFormProcessorCursor + } +} diff --git a/server/service/src/processors/load_plugin/mod.rs b/server/service/src/processors/load_plugin/mod.rs new file mode 100644 index 0000000000..d98ee9af73 --- /dev/null +++ b/server/service/src/processors/load_plugin/mod.rs @@ -0,0 +1,2 @@ +mod load_plugin; +pub(crate) use self::load_plugin::*; diff --git a/server/service/src/processors/mod.rs b/server/service/src/processors/mod.rs index 446510936d..344827a2dd 100644 --- a/server/service/src/processors/mod.rs +++ b/server/service/src/processors/mod.rs @@ -19,6 +19,7 @@ use general_processor::{process_records, ProcessorError}; mod contact_form; mod general_processor; +mod load_plugin; pub use general_processor::ProcessorType; #[cfg(test)] mod test_helpers; From 6f2d13e25687489bc24fb441dd6316cf24e05d6f Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 31 Jan 2025 12:04:47 +1300 Subject: [PATCH 2/4] Add test for variants --- .../src/db_diesel/key_value_store.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/server/repository/src/db_diesel/key_value_store.rs b/server/repository/src/db_diesel/key_value_store.rs index 444ac01c4b..91773d2a19 100644 --- a/server/repository/src/db_diesel/key_value_store.rs +++ b/server/repository/src/db_diesel/key_value_store.rs @@ -18,6 +18,7 @@ table! { // Database: https://github.com/openmsupply/open-msupply/blob/d6645711184c63593949c3e8b6dc96b5a5ded39f/server/repository/migrations/postgres/2022-02-11T15-00_create_key_value_store/up.sql#L2-L16 #[derive(DbEnum, Debug, Clone, PartialEq, Eq, Default)] +#[cfg_attr(test, derive(strum::EnumIter))] #[DbValueStyle = "SCREAMING_SNAKE_CASE"] pub enum KeyType { #[default] @@ -172,3 +173,29 @@ impl<'a> KeyValueStoreRepository<'a> { Ok(row.and_then(|row| row.value_bool)) } } + +#[cfg(test)] +mod test { + use super::*; + use strum::IntoEnumIterator; + use util::assert_matches; + + use crate::{mock::MockDataInserts, test_db::setup_all}; + + #[actix_rt::test] + async fn key_type_enum() { + let (_, connection, _, _) = setup_all("key_type_enum", MockDataInserts::none()).await; + + let repo = KeyValueStoreRepository::new(&connection); + // Try upsert all variants, confirm that diesel enums match postgres + for variant in KeyType::iter() { + let result = repo.upsert_one(&KeyValueStoreRow { + id: variant.clone(), + ..Default::default() + }); + assert_eq!(result, Ok(())); + + assert_matches!(repo.get_row(variant.clone()), Ok(Some(_))); + } + } +} From 296ef6b7597e00b7bb466af4be269b1a4d522b8e Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 3 Feb 2025 18:26:16 +1300 Subject: [PATCH 3/4] Load plugins via processor --- server/server/src/lib.rs | 5 ++++- server/service/src/plugin/mod.rs | 15 +++++++++++-- .../src/processors/general_processor.rs | 8 +++---- .../src/processors/load_plugin/load_plugin.rs | 22 ++++++++++++++----- server/service/src/sync/synchroniser.rs | 3 +++ 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/server/server/src/lib.rs b/server/server/src/lib.rs index 01a923e333..76196ad98a 100644 --- a/server/server/src/lib.rs +++ b/server/server/src/lib.rs @@ -147,11 +147,14 @@ pub async fn start_server( } // PLUGIN CONTEXT - PluginContext { service_provider: service_provider.clone(), } .bind(); + service_provider + .plugin_service + .reload_all_plugins(&service_context) + .unwrap(); // SET LOG CALLBACK FOR WASM FUNCTIONS info!("Setting wasm function log callback.."); diff --git a/server/service/src/plugin/mod.rs b/server/service/src/plugin/mod.rs index 7e767ff09c..8399e7fcfe 100644 --- a/server/service/src/plugin/mod.rs +++ b/server/service/src/plugin/mod.rs @@ -3,6 +3,7 @@ use thiserror::Error; use crate::{ backend_plugin::plugin_provider::{PluginBundle, PluginInstance}, + processors::ProcessorType, service_provider::ServiceContext, settings::Settings, UploadedFile, UploadedFileJsonError, @@ -43,6 +44,15 @@ pub trait PluginServiceTrait: Sync + Send { uploaded_file.as_json_file(settings) } + fn reload_all_plugins(&self, ctx: &ServiceContext) -> Result<(), RepositoryError> { + let repo = BackendPluginRowRepository::new(&ctx.connection); + for row in repo.all()? { + PluginInstance::bind(row); + } + + Ok(()) + } + fn install_uploaded_plugin( &self, ctx: &ServiceContext, @@ -56,10 +66,11 @@ pub trait PluginServiceTrait: Sync + Send { for row in plugin_bundle.backend_plugins.clone() { backend_repo.upsert_one(row.clone())?; - // Bind plugin to provider (this would ideally happen via processor, going over changelog) - PluginInstance::bind(row) } + ctx.processors_trigger + .trigger_processor(ProcessorType::LoadPlugin); + Ok(plugin_bundle) }) .map_err(|error| error.to_inner_error()) diff --git a/server/service/src/processors/general_processor.rs b/server/service/src/processors/general_processor.rs index 265972d4da..4f30b2371a 100644 --- a/server/service/src/processors/general_processor.rs +++ b/server/service/src/processors/general_processor.rs @@ -15,11 +15,11 @@ use super::{contact_form::QueueContactEmailProcessor, load_plugin::LoadPlugin}; #[derive(Error, Debug)] pub(crate) enum ProcessorError { - #[error("{0:?} not found: {1:?}")] + #[error("{0} not found: {1}")] RecordNotFound(String, String), - #[error("{0:?}")] - DatabaseError(RepositoryError), - #[error("{0:?}")] + #[error("Database error")] + DatabaseError(#[from] RepositoryError), + #[error("Error in email service {0:?}")] EmailServiceError(EmailServiceError), } diff --git a/server/service/src/processors/load_plugin/load_plugin.rs b/server/service/src/processors/load_plugin/load_plugin.rs index 4740ac0a5e..fcbe544e51 100644 --- a/server/service/src/processors/load_plugin/load_plugin.rs +++ b/server/service/src/processors/load_plugin/load_plugin.rs @@ -1,6 +1,11 @@ -use repository::{ChangelogRow, ChangelogTableName, KeyType, StorageConnection}; +use repository::{ + BackendPluginRowRepository, ChangelogRow, ChangelogTableName, KeyType, StorageConnection, +}; -use crate::processors::general_processor::{Processor, ProcessorError}; +use crate::{ + backend_plugin::plugin_provider::PluginInstance, + processors::general_processor::{Processor, ProcessorError}, +}; const DESCRIPTION: &str = "Load plugins"; @@ -11,13 +16,20 @@ impl Processor for LoadPlugin { DESCRIPTION.to_string() } - /// Only runs once because contact form is create only - /// Changelog will only be processed once fn try_process_record( &self, connection: &StorageConnection, changelog: &ChangelogRow, ) -> Result, ProcessorError> { + let plugin = BackendPluginRowRepository::new(connection) + .find_one_by_id(&changelog.record_id)? + .ok_or(ProcessorError::RecordNotFound( + "Backend plugin".to_string(), + changelog.record_id.clone(), + ))?; + + PluginInstance::bind(plugin); + Ok(Some("success".to_string())) } @@ -26,6 +38,6 @@ impl Processor for LoadPlugin { } fn cursor_type(&self) -> KeyType { - KeyType::ContactFormProcessorCursor + KeyType::LoadPluginProcessorCursor } } diff --git a/server/service/src/sync/synchroniser.rs b/server/service/src/sync/synchroniser.rs index 894be4704a..180be0170e 100644 --- a/server/service/src/sync/synchroniser.rs +++ b/server/service/src/sync/synchroniser.rs @@ -289,6 +289,9 @@ impl Synchroniser { ctx.processors_trigger .trigger_processor(ProcessorType::ContactFormEmail); + ctx.processors_trigger + .trigger_processor(ProcessorType::LoadPlugin); + Ok(()) } } From 629cd1d13f716032319e4eb9bf1231b46a72ef46 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 3 Feb 2025 19:20:45 +1300 Subject: [PATCH 4/4] Add snippet for adding new KeyValueStore key --- server/repository/src/db_diesel/key_value_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/repository/src/db_diesel/key_value_store.rs b/server/repository/src/db_diesel/key_value_store.rs index 91773d2a19..b60927d917 100644 --- a/server/repository/src/db_diesel/key_value_store.rs +++ b/server/repository/src/db_diesel/key_value_store.rs @@ -16,7 +16,7 @@ table! { } } -// Database: https://github.com/openmsupply/open-msupply/blob/d6645711184c63593949c3e8b6dc96b5a5ded39f/server/repository/migrations/postgres/2022-02-11T15-00_create_key_value_store/up.sql#L2-L16 +// Snippet for adding new, including migration : https://github.com/msupply-foundation/open-msupply/wiki/Snippets "New Key Type for KeyValueStore" #[derive(DbEnum, Debug, Clone, PartialEq, Eq, Default)] #[cfg_attr(test, derive(strum::EnumIter))] #[DbValueStyle = "SCREAMING_SNAKE_CASE"]