From 0060779abc9e57abb47a632eab27afb40c985bd9 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Date: Fri, 25 Nov 2022 04:18:40 +0530 Subject: [PATCH] FFI support for ain integration * Parse args and run in a separate thread * Native helper struct for static FFI invocation * Wait until node starts when run as daemon * Flags for success, daemon and displaying help * Graceful shutdown on interrupt signal --- .gitignore | 5 ++ Cargo.lock | 76 +++++++++++++++++++++++++-- Makefile | 9 ++++ meta/meta-node/Cargo.toml | 13 ++++- meta/meta-node/build.rs | 32 ++++++++++++ meta/meta-node/src/command.rs | 45 +++++++++++++--- meta/meta-node/src/lib.rs | 98 +++++++++++++++++++++++++++++++++++ meta/meta-node/src/main.rs | 13 ++++- meta/meta-node/src/native.rs | 48 +++++++++++++++++ meta/meta-node/src/service.rs | 13 +++-- 10 files changed, 332 insertions(+), 20 deletions(-) create mode 100644 Makefile create mode 100644 meta/meta-node/src/lib.rs create mode 100644 meta/meta-node/src/native.rs diff --git a/.gitignore b/.gitignore index 2f77fb6f..111d9c2e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,11 @@ target/ # These are backup files generated by rustfmt **/*.rs.bk +# Rust package files +pkg/ +# ain depends integration artifacts +.stamp_* +master-* ### Substrate .local diff --git a/Cargo.lock b/Cargo.lock index cb585701..032fa361 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -797,6 +797,16 @@ dependencies = [ "cc", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "concurrent-queue" version = "1.2.4" @@ -1128,6 +1138,47 @@ dependencies = [ "zeroize", ] +[[package]] +name = "cxx" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0492b760d2a9b9c5d1e2132db2dcbba07a2045df6f14694b136eec5b2053ef" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-gen" +version = "0.7.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac757cbb702ed1bec5525e3acda1d5420dbf7163fc2941eaa9cc9c760fbbb9a4" +dependencies = [ + "codespan-reporting", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebfd09b0a600cf97b53c16eec6591893cbcaffe3f98c17c4e89b3b3a672c33a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ccac323f5afe0678c7129b8346c2be2d2213d8c58507facfb4987d109abf1b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.3.2" @@ -3462,6 +3513,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -3677,6 +3737,8 @@ version = "0.1.0" dependencies = [ "async-trait", "clap", + "cxx", + "cxx-gen", "fc-cli", "fc-consensus", "fc-db", @@ -3690,10 +3752,12 @@ dependencies = [ "fp-storage", "futures", "jsonrpsee", + "lazy_static", "log", "meta-runtime", "pallet-transaction-payment-rpc", "parity-scale-codec", + "proc-macro2", "sc-basic-authorship", "sc-cli", "sc-client-api", @@ -3720,6 +3784,7 @@ dependencies = [ "sp-timestamp", "substrate-build-script-utils", "substrate-frame-rpc-system", + "tokio", ] [[package]] @@ -7555,9 +7620,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.1" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" +checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", "bytes", @@ -7565,7 +7630,6 @@ dependencies = [ "memchr", "mio", "num_cpus", - "once_cell", "parking_lot 0.12.1", "pin-project-lite 0.2.9", "signal-hook-registry", @@ -7871,6 +7935,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "unicode-xid" version = "0.2.3" diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..83463704 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +CARGO ?= cargo +TARGET ?= + +build-native-pkg: + CRATE_CC_NO_DEFAULTS=1 $(CARGO) build --package meta-node --release $(if $(TARGET),--target $(TARGET),) + mkdir -p pkg/metachain/include pkg/metachain/lib + cp target/$(if $(TARGET),$(TARGET)/,)release/libmeta_node.a pkg/metachain/lib/libmetachain.a + cp target/libmc.hpp pkg/metachain/include/ + cp target/libmc.cpp pkg/metachain/ diff --git a/meta/meta-node/Cargo.toml b/meta/meta-node/Cargo.toml index 005fbd1a..c35d3bc6 100644 --- a/meta/meta-node/Cargo.toml +++ b/meta/meta-node/Cargo.toml @@ -4,12 +4,22 @@ version = "0.1.0" edition = "2021" build = "build.rs" +[lib] +crate-type = ["staticlib"] + +[[bin]] +name = "meta-node" +path = "src/main.rs" + [dependencies] async-trait = "0.1" clap = { version = "3.1", features = ["derive"] } +cxx = "1.0" futures = "0.3.21" +lazy_static = "1.4.0" log = "0.4.17" serde_json = "1.0" +tokio = { version = "1.22", features = ["sync"] } # Parity codec = { version = "3.1.5", features = ["derive"], package = "parity-scale-codec" } jsonrpsee = { version = "0.14.0", features = ["server"] } @@ -58,4 +68,5 @@ meta-runtime = { package = "meta-runtime", path = "../meta-runtime" } [build-dependencies] substrate-build-script-utils = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.27" } - +cxx-gen = "0.7" +proc-macro2 = "1.0" diff --git a/meta/meta-node/build.rs b/meta/meta-node/build.rs index e3bfe311..5bfffca2 100644 --- a/meta/meta-node/build.rs +++ b/meta/meta-node/build.rs @@ -1,7 +1,39 @@ +use proc_macro2::TokenStream; use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed}; +use std::env; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::PathBuf; + fn main() { generate_cargo_keys(); rerun_if_git_head_changed(); + + let mut root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let parent = root.clone(); + root.pop(); + root.pop(); + let lib_path = &parent.join("src").join("lib.rs"); + println!("cargo:rerun-if-changed={}", lib_path.display()); + let target_dir = &root.join("target"); + + let mut content = String::new(); + File::open(lib_path) + .unwrap() + .read_to_string(&mut content) + .unwrap(); + let tt: TokenStream = content.parse().unwrap(); + let codegen = cxx_gen::generate_header_and_cc(tt, &cxx_gen::Opt::default()).unwrap(); + + let cpp_stuff = String::from_utf8(codegen.implementation).unwrap(); + File::create(target_dir.join("libmc.hpp")) + .unwrap() + .write_all(&codegen.header) + .unwrap(); + File::create(target_dir.join("libmc.cpp")) + .unwrap() + .write_all(cpp_stuff.as_bytes()) + .unwrap(); } diff --git a/meta/meta-node/src/command.rs b/meta/meta-node/src/command.rs index e0f50333..63653fee 100644 --- a/meta/meta-node/src/command.rs +++ b/meta/meta-node/src/command.rs @@ -3,10 +3,12 @@ use crate::{ cli::{Cli, Subcommand}, service::{self, db_config_dir}, }; -use clap::Parser; use fc_db::frontier_database_dir; use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::{DatabaseSource, PartialComponents}; +use tokio::sync::mpsc; + +use std::thread; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -51,10 +53,8 @@ impl SubstrateCli for Cli { } } -/// Parse and run command line arguments -pub fn run() -> sc_cli::Result<()> { - let cli = Cli::parse(); - +/// Parse and run command line arguments in a separate thread and return handle +pub fn run(cli: Cli) -> sc_cli::Result<()> { match &cli.subcommand { Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::BuildSpec(cmd)) => { @@ -154,10 +154,39 @@ pub fn run() -> sc_cli::Result<()> { }) } None => { + // Metachain should block until the network starts before handing over + // control to the native chain. One way to accomplish this is by moving + // everything over to a separate thread, wait for the result of node start, + // and let native chain take control after. + let (tx, mut rx) = mpsc::channel(1); let runner = cli.create_runner(&cli.run.base)?; - runner.run_node_until_exit(|config| async move { - service::new_full(config, &cli).map_err(sc_cli::Error::Service) - }) + let (signal_tx, mut signal_rx) = mpsc::channel(1); + crate::native::setup_interrupt_handler(signal_tx); + let handle = thread::spawn(move || { + runner.async_run(|config| { + match service::new_full(config, &cli) { + Ok((starter, manager)) => { + starter.start_network(); + let _ = tx.blocking_send(None); // network has started, return control to native chain + Ok((async move { + signal_rx.recv().await; // wait for signal from native chain + Ok(()) + }, manager)) + }, + Err(e) => { + let _ = tx.blocking_send(Some(e.to_string())); // failed to start network + Err(sc_cli::Error::Service(e)) + }, + } + }) + }); + + // Wait for startup failure, if any + if let Some(Some(msg)) = rx.blocking_recv() { + return Err(msg.into()); + } + + crate::native::store_handle(handle) } } } diff --git a/meta/meta-node/src/lib.rs b/meta/meta-node/src/lib.rs new file mode 100644 index 00000000..d175fcd8 --- /dev/null +++ b/meta/meta-node/src/lib.rs @@ -0,0 +1,98 @@ +mod chain_spec; +mod native; +#[macro_use] +mod service; +mod cli; +mod command; +mod rpc; + +use std::error::Error; + +use clap::{CommandFactory, ErrorKind, FromArgMatches}; +use cli::Cli; +use cxx::{CxxString, CxxVector}; + +// NOTE: Struct definitions for FFI should match those defined in libain-rs/protobuf + +#[cxx::bridge] +mod ffi { + pub struct DmcTx { + pub from: String, + pub to: String, + pub amount: i64, + } + + pub struct DmcBlock { + pub payload: Vec, + } + + struct ExecResult { + is_help: bool, + success: bool, + daemon: bool, + } + + extern "Rust" { + fn mint_block(dmc_txs: &CxxVector) -> Result; + fn parse_args_and_run(args: &CxxVector) -> ExecResult; + fn interrupt_dmc() -> Result<()>; + } +} + +#[inline] +fn mint_block(dmc_txs: &CxxVector) -> Result> { + self::native::NATIVE_HELPER + .read().unwrap().as_ref().expect("uninitialized native helper") + .mint_block(dmc_txs.iter()) +} + +#[inline] +fn interrupt_dmc() -> sc_cli::Result<()> { + log::info!("Signaling metachain node to terminate"); + let _ = self::native::SIGNAL_HANDLER.read().unwrap().as_ref().expect("uninitialized signal handler") + .blocking_send(()); + log::info!("Waiting for processes to finish"); + self::native::THREAD_HANDLE.write().unwrap().take().expect("thread not spawned yet?") + .join().unwrap() +} + +fn parse_args_and_run(args: &CxxVector) -> ffi::ExecResult { + let args: Vec = args + .iter() + .map(|s| s.to_string_lossy().into_owned()) + .collect(); + let res = Cli::command().try_get_matches_from(args) + .and_then(|mut m| Cli::from_arg_matches_mut(&mut m)); + + let cli = match res { + Ok(c) => c, + Err(e) => { + let _ = e.print(); + let is_help = match e.kind() { + ErrorKind::DisplayHelp | ErrorKind::DisplayVersion => true, + _ => false, + }; + return ffi::ExecResult { + is_help, + success: is_help, + daemon: false, + } + }, + }; + + let mut result = ffi::ExecResult { + is_help: false, + success: false, + daemon: cli.subcommand.is_none(), + }; + match command::run(cli) { + Ok(_) => { + // Deciding to run the node, give control back to defid without joining handle + // FIXME: Figure out how to send SIGINT on defid exiting for cleanly shutting down + result.success = true; + }, + Err(e) => eprintln!("{:?}", e), + } + + result +} diff --git a/meta/meta-node/src/main.rs b/meta/meta-node/src/main.rs index 9acec91f..8a7e94b8 100644 --- a/meta/meta-node/src/main.rs +++ b/meta/meta-node/src/main.rs @@ -2,6 +2,8 @@ #![warn(missing_docs)] #![warn(unused_extern_crates)] +use clap::Parser; + mod chain_spec; #[macro_use] mod service; @@ -9,6 +11,15 @@ mod cli; mod command; mod rpc; +mod native { + pub fn init_helper(_client: std::sync::Arc) {} + pub fn setup_interrupt_handler(_sender: tokio::sync::mpsc::Sender<()>) {} + pub fn store_handle(handle: std::thread::JoinHandle>) -> sc_cli::Result<()> { + handle.join().unwrap() + } +} + fn main() -> sc_cli::Result<()> { - command::run() + let cli = cli::Cli::parse(); + command::run(cli) } diff --git a/meta/meta-node/src/native.rs b/meta/meta-node/src/native.rs new file mode 100644 index 00000000..af2b0e15 --- /dev/null +++ b/meta/meta-node/src/native.rs @@ -0,0 +1,48 @@ +use crate::ffi::{DmcTx, DmcBlock}; +use crate::service::FullClient; + +use tokio::sync::mpsc::Sender; + +use std::error::Error; +use std::sync::{Arc, RwLock}; +use std::thread::JoinHandle; + +lazy_static::lazy_static! { + pub static ref SIGNAL_HANDLER: RwLock>> = RwLock::new(None); + pub static ref NATIVE_HELPER: RwLock> = RwLock::new(None); + pub static ref THREAD_HANDLE: RwLock>>> = RwLock::new(None); +} + +pub struct NativeHelper { + client: Arc, +} + +impl NativeHelper { + pub fn mint_block<'a, I>(&self, _dmc_txs: I) -> Result> + where I: Iterator + { + let _ = &self.client; + Ok(DmcBlock { + payload: vec![1, 2, 3, 4, 5], + }) + } +} + +/// Initialize the native chain helper for FFI usage +pub fn init_helper(client: Arc) { + log::info!("Initializing FFI helper"); + *NATIVE_HELPER.write().unwrap() = Some(NativeHelper { + client, + }); +} + +/// Setup interrupt handler so that native chain can ask metachain to stop gracefully +pub fn setup_interrupt_handler(sender: Sender<()>) { + *SIGNAL_HANDLER.write().unwrap() = Some(sender); +} + +/// Store thread handle so that we can block later for the thread to exit +pub fn store_handle(handle: JoinHandle>) -> sc_cli::Result<()> { + *THREAD_HANDLE.write().unwrap() = Some(handle); + Ok(()) // NOTE: Returns a result so that it's compatible with library and executable +} diff --git a/meta/meta-node/src/service.rs b/meta/meta-node/src/service.rs index 417db6f1..1cd9f0e8 100644 --- a/meta/meta-node/src/service.rs +++ b/meta/meta-node/src/service.rs @@ -15,7 +15,7 @@ use sc_client_api::BlockchainEvents; use sc_executor::NativeElseWasmExecutor; use sc_keystore::LocalKeystore; use sc_service::{ - error::Error as ServiceError, BasePath, Configuration, PartialComponents, TaskManager, + error::Error as ServiceError, BasePath, Configuration, NetworkStarter, PartialComponents, TaskManager, }; use sc_telemetry::{Telemetry, TelemetryWorker}; use sp_core::U256; @@ -27,8 +27,7 @@ use fc_mapping_sync::{MappingSyncWorker, SyncStrategy}; use fc_rpc::{EthTask, OverrideHandle}; use fc_rpc_core::types::{FeeHistoryCache, FeeHistoryCacheLimit, FilterPool}; // Runtime -use crate::cli::Cli; -use crate::cli::Sealing; +use crate::cli::{Cli, Sealing}; use meta_runtime::{opaque::Block, RuntimeApi}; // Our native executor instance. @@ -188,7 +187,7 @@ fn remote_keystore(_url: &str) -> Result, &'static str> { } /// Builds a new service for a full client. -pub fn new_full(mut config: Configuration, cli: &Cli) -> Result { +pub fn new_full(mut config: Configuration, cli: &Cli) -> Result<(NetworkStarter, TaskManager), ServiceError> { // Use ethereum style for subscription ids config.rpc_id_provider = Some(Box::new(fc_rpc::EthereumSubIdProvider)); @@ -210,6 +209,8 @@ pub fn new_full(mut config: Configuration, cli: &Cli) -> Result keystore_container.set_remote_keystore(k), @@ -392,9 +393,7 @@ pub fn new_full(mut config: Configuration, cli: &Cli) -> Result