From b58637d63d9e21f7242de56480ead892c9c048ad Mon Sep 17 00:00:00 2001 From: spacesops Date: Tue, 5 May 2026 22:57:56 -0400 Subject: [PATCH 1/3] fix: use configured RPC proxy credentials and print startup URL Wire spaced RPC user/password from CLI into AppState so /rpc/spaced no longer uses hardcoded credentials, and add a clear startup console URL that includes the port. Co-authored-by: Cursor --- subs/src/main.rs | 37 +++++++++++++++++++++++++++++++++---- subs/src/routes/console.rs | 7 ++++++- subs/src/state.rs | 16 ++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/subs/src/main.rs b/subs/src/main.rs index 51e99d7..0517ff7 100644 --- a/subs/src/main.rs +++ b/subs/src/main.rs @@ -85,7 +85,7 @@ async fn main() -> Result<()> { tracing_subscriber::registry() .with( tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "subsd=info,tower_http=debug".into()), + .unwrap_or_else(|_| "subs=info,tower_http=debug".into()), ) .with(tracing_subscriber::fmt::layer()) .init(); @@ -150,7 +150,16 @@ async fn run_normal(cli: Cli) -> Result<()> { operator.load_all_spaces().await?; // Build app state and run server - run_server(operator, config, cli.port, Some(rpc_url.clone()), None).await + run_server( + operator, + config, + cli.port, + Some(rpc_url.clone()), + cli.rpc_user.clone(), + cli.rpc_password.clone(), + None, + ) + .await } #[cfg(feature = "test-rig")] @@ -226,10 +235,19 @@ async fn run_server( config: ConfigStore, port: u16, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, bitcoin_rpc_url: Option, ) -> Result<()> { // Build app state - let state = AppState::with_rpc_urls(operator, config, spaced_rpc_url, bitcoin_rpc_url); + let state = AppState::with_rpc_urls( + operator, + config, + spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, + bitcoin_rpc_url, + ); run_server_inner(state, port).await } @@ -244,7 +262,16 @@ async fn run_server_with_testrig( test_rig: std::sync::Arc, ) -> Result<()> { // Build app state with test rig - let state = AppState::with_test_rig(operator, config, Some(spaced_rpc_url), Some(bitcoin_rpc_url), Some(certrelay_url), test_rig); + let state = AppState::with_test_rig( + operator, + config, + Some(spaced_rpc_url), + Some("user".to_string()), + Some("pass".to_string()), + Some(bitcoin_rpc_url), + Some(certrelay_url), + test_rig, + ); run_server_inner(state, port).await } @@ -266,6 +293,8 @@ async fn run_server_inner(state: AppState, port: u16) -> Result<()> { // Start server let addr = SocketAddr::from(([0, 0, 0, 0], port)); tracing::info!("Starting server on http://{}", addr); + tracing::info!("Server URL: http://127.0.0.1:{}", port); + println!("Server URL: http://127.0.0.1:{} (port {})", port, port); let listener = tokio::net::TcpListener::bind(addr).await?; axum::serve(listener, app) diff --git a/subs/src/routes/console.rs b/subs/src/routes/console.rs index 9e1376a..9e585ba 100644 --- a/subs/src/routes/console.rs +++ b/subs/src/routes/console.rs @@ -84,7 +84,12 @@ pub async fn proxy_spaced( .as_ref() .ok_or_else(|| json_error(StatusCode::SERVICE_UNAVAILABLE, "Spaced RPC URL not configured"))?; - proxy_rpc_call(rpc_url, &request, Some(("user", "pass"))).await + let auth = state + .spaced_rpc_user + .as_deref() + .map(|user| (user, state.spaced_rpc_password.as_deref().unwrap_or(""))); + + proxy_rpc_call(rpc_url, &request, auth).await } /// POST /rpc/bitcoin - Proxy RPC call to bitcoind (test-rig only) diff --git a/subs/src/state.rs b/subs/src/state.rs index 2868d71..5f03320 100644 --- a/subs/src/state.rs +++ b/subs/src/state.rs @@ -16,6 +16,10 @@ pub struct AppState { pub config: Arc, /// Spaced RPC URL for the console pub spaced_rpc_url: Option, + /// Spaced RPC username for proxied calls + pub spaced_rpc_user: Option, + /// Spaced RPC password for proxied calls + pub spaced_rpc_password: Option, /// Bitcoin RPC URL (only available in test-rig mode) pub bitcoin_rpc_url: Option, /// Certrelay URL (only available in test-rig mode) @@ -31,12 +35,16 @@ impl AppState { operator: Operator, config: ConfigStore, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, _bitcoin_rpc_url: Option, ) -> Self { Self { operator: Arc::new(operator), config: Arc::new(config), spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, bitcoin_rpc_url: None, certrelay_url: None, } @@ -47,12 +55,16 @@ impl AppState { operator: Operator, config: ConfigStore, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, bitcoin_rpc_url: Option, ) -> Self { Self { operator: Arc::new(operator), config: Arc::new(config), spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, bitcoin_rpc_url, certrelay_url: None, test_rig: None, @@ -64,6 +76,8 @@ impl AppState { operator: Operator, config: ConfigStore, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, bitcoin_rpc_url: Option, certrelay_url: Option, test_rig: Arc, @@ -72,6 +86,8 @@ impl AppState { operator: Arc::new(operator), config: Arc::new(config), spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, bitcoin_rpc_url, certrelay_url, test_rig: Some(test_rig), From 55b0aa613180a1f69f5e336d2f5c1441c21ba5f9 Mon Sep 17 00:00:00 2001 From: spacesops Date: Thu, 21 May 2026 15:25:46 -0400 Subject: [PATCH 2/3] Prep for Docker w/env --- .env.example | 30 +++++ .gitignore | 5 + Cargo.toml | 12 +- README.md | 95 ++++++++++++++ config-origins/Cargo.toml | 10 ++ config-origins/src/lib.rs | 185 +++++++++++++++++++++++++++ examples/registry-server/Cargo.toml | 1 + examples/registry-server/README.md | 4 +- examples/registry-server/src/main.rs | 21 ++- prover/Cargo.toml | 2 + prover/src/env.rs | 71 ++++++++++ prover/src/lib.rs | 1 + prover/src/main.rs | 44 +++++-- subs/Cargo.toml | 2 + subs/src/env.rs | 165 ++++++++++++++++++++++++ subs/src/main.rs | 46 +++++-- 16 files changed, 666 insertions(+), 28 deletions(-) create mode 100644 .env.example create mode 100644 config-origins/Cargo.toml create mode 100644 config-origins/src/lib.rs create mode 100644 prover/src/env.rs create mode 100644 subs/src/env.rs diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1f72bbd --- /dev/null +++ b/.env.example @@ -0,0 +1,30 @@ +# Copy to .env and adjust, or export variables in your shell. +# CLI flags take precedence over environment variables. + +# --- subs --- +SUBS_PORT=7777 +SUBS_DATA_DIR=./data +SUBS_WALLET=my-wallet +SUBS_SPACED_RPC_URL=http://127.0.0.1:7225 +# SUBS_SPACED_RPC_USER=testuser +# SUBS_SPACED_RPC_PASSWORD=secret +# SUBS_SPACED_RPC_COOKIE=/path/to/.cookie +# SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888 +# SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8080 +# SUBS_ENV_FILE=.env +# SUBS_TEST_RIG=1 +# SUBS_TEST_RIG_DIR=./testrig-data + +# --- subs-prover --- +# SUBS_PROVER_SERVER=1 +SUBS_PROVER_PORT=8888 +# SUBS_PROVER_ENV_FILE=.env +# SUBS_PROVER_INPUT=request.json +# SUBS_PROVER_OUTPUT=receipt.bin + +# --- registry-server (example) --- +REGISTRY_SERVER_PORT=8080 +# REGISTRY_SERVER_ENV_FILE=.env + +# --- logging (all components) --- +# RUST_LOG=subs=info,subs_prover=info,registry_server=info,tower_http=debug diff --git a/.gitignore b/.gitignore index 7d9e179..3571e4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.env +.env.* +!.env.example .DS_Store Cargo.lock .vscode @@ -11,3 +14,5 @@ target/ .idea testrig-data data +datamad +NOTES.md diff --git a/Cargo.toml b/Cargo.toml index fa35aea..7b15882 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,13 @@ [workspace] resolver = "2" -members = ["core", "prover", "types", "subs", "examples/registry-server"] +members = [ + "config-origins", + "core", + "prover", + "types", + "subs", + "examples/registry-server", +] [workspace.dependencies] # Internal crates @@ -29,7 +36,8 @@ serde_json = "1.0" anyhow = "1.0" hex = "0.4" borsh = { version = "1.5", default-features = false, features = ["derive"] } -clap = { version = "4.5", features = ["derive"] } +clap = { version = "4.5", features = ["derive", "env"] } +dotenvy = "0.15" tokio = { version = "1" } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/README.md b/README.md index 02f781e..6e3d4db 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,101 @@ cargo install --path prover For operators, use `--features cuda` on `subs-prover` for nvidia machines to enable GPU acceleration. +## Configuration + +Each binary accepts the same settings via **CLI flags**, **environment variables**, or a **`.env` file** in the current working directory. Command-line flags override environment variables. + +Load a custom env file path with: + +- `subs`: `SUBS_ENV_FILE=/path/to/subs.env` +- `subs-prover`: `SUBS_PROVER_ENV_FILE=/path/to/prover.env` +- `registry-server`: `REGISTRY_SERVER_ENV_FILE=/path/to/registry.env` + +See [.env.example](.env.example) for a full template. + +### `subs` + +| Variable | CLI flag | Description | +|----------|----------|-------------| +| `SUBS_PORT` | `--port` | HTTP server port (default `7777`) | +| `SUBS_DATA_DIR` | `--data-dir` | Data directory (default `./data`) | +| `SUBS_WALLET` | `--wallet` | Wallet name for signing | +| `SUBS_SPACED_RPC_URL` | `--rpc-url` | `spaced` RPC URL | +| `SUBS_SPACED_RPC_USER` | `--rpc-user` | `spaced` RPC username | +| `SUBS_SPACED_RPC_PASSWORD` | `--rpc-password` | `spaced` RPC password | +| `SUBS_SPACED_RPC_COOKIE` | `--rpc-cookie` | `spaced` RPC cookie file path | +| `SUBS_PROVER_ENDPOINT` | *(Settings UI)* | Prover URL written to `config.db` at startup | +| `SUBS_REGISTRY_ENDPOINT` | *(Settings UI)* | Registry URL written to `config.db` at startup | +| `SUBS_TEST_RIG` | `--test-rig` | Enable test rig (`1`, `true`, `yes`) | +| `SUBS_TEST_RIG_DIR` | `--test-rig-dir` | Test rig data directory | + +### `subs-prover` + +| Variable | CLI flag | Description | +|----------|----------|-------------| +| `SUBS_PROVER_SERVER` | `--server` | Run as HTTP server (`1`, `true`, `yes`) | +| `SUBS_PROVER_PORT` | `--server-port` | Server port (default `8888`) | +| `SUBS_PROVER_INPUT` | `-i` / `--input` | Input file (prove/compress subcommands) | +| `SUBS_PROVER_OUTPUT` | `-o` / `--output` | Output file (prove/compress subcommands) | +| `SUBS_PROVER_BENCH_EXISTING` | `--existing` | Bench: existing handle count | +| `SUBS_PROVER_BENCH_INSERT` | `--insert` | Bench: handles to insert | + +### `registry-server` + +| Variable | CLI flag | Description | +|----------|----------|-------------| +| `REGISTRY_SERVER_PORT` | `--port` | HTTP server port (default `8080`) | + +### Examples + +Using `export`: + +```bash +export SUBS_SPACED_RPC_URL=http://127.0.0.1:7225 +export SUBS_WALLET=my-wallet +export SUBS_DATA_DIR=./data +export SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888 +subs +``` + +Using a `.env` file: + +```bash +cp .env.example .env +# edit .env, then: +subs +``` + +```bash +# subs-prover from .env +export SUBS_PROVER_SERVER=1 +export SUBS_PROVER_PORT=8888 +subs-prover +``` + +```bash +# registry-server +export REGISTRY_SERVER_PORT=8080 +registry-server +``` + +Log verbosity uses the standard `RUST_LOG` variable (e.g. `RUST_LOG=subs=debug,tower_http=debug`). + +On startup, each binary prints its **effective configuration** to the console with the **origin** of each value: `param` (CLI flag), `environment` (`export`), `.env` (dotenv file), or `default`. Sensitive values (passwords) are shown as `(set)` without revealing the secret. Example: + +``` +subs configuration: + (loaded env file: .env) + port = 7777 (.env) + data_dir = ./datamad (.env) + wallet = mad (environment) + rpc_url = http://127.0.0.1:7225 (.env) + rpc_password = (set) (.env) + server_url = http://127.0.0.1:7777 (derived from port) +``` + +CLI flags override environment variables; process environment overrides `.env` for the same key. + ## Usage ### 1. Start the prover server diff --git a/config-origins/Cargo.toml b/config-origins/Cargo.toml new file mode 100644 index 0000000..46dd8b2 --- /dev/null +++ b/config-origins/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "config-origins" +version = "0.1.0" +edition = "2021" +description = "Startup configuration logging with value origins for subs binaries" +publish = false + +[dependencies] +clap = { workspace = true } +dotenvy = { workspace = true } diff --git a/config-origins/src/lib.rs b/config-origins/src/lib.rs new file mode 100644 index 0000000..b471ce3 --- /dev/null +++ b/config-origins/src/lib.rs @@ -0,0 +1,185 @@ +//! Helpers for loading `.env` files and logging effective configuration with origins. + +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::path::{Path, PathBuf}; + +use clap::parser::ValueSource; +use clap::ArgMatches; + +/// Where a configuration value came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConfigOrigin { + /// Command-line flag or positional argument. + Param, + /// Process environment (e.g. `export VAR=...`) before `.env` was applied. + Environment, + /// `.env` file (or file pointed to by `*_ENV_FILE`). + DotEnv, + /// Built-in default when nothing else was provided. + Default, +} + +impl fmt::Display for ConfigOrigin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Param => write!(f, "param"), + Self::Environment => write!(f, "environment"), + Self::DotEnv => write!(f, ".env"), + Self::Default => write!(f, "default"), + } + } +} + +/// Result of loading a dotenv file. +#[derive(Debug, Clone)] +pub struct DotenvLoad { + /// Path loaded, if any. + pub env_file: Option, + /// Variable names whose values were applied from the file (not pre-set in the process env). + pub keys_from_dotenv: HashSet, +} + +/// Snapshot of the process environment before loading dotenv. +type EnvSnapshot = HashMap; + +/// Load variables from a `.env` file before CLI parsing. +/// +/// If `env_file_var` is set in the environment, that path is used; otherwise tries `.env` +/// in the current working directory. Existing process environment variables are not overridden. +pub fn load_dotenv(env_file_var: &str) -> DotenvLoad { + let before = snapshot_env(); + let (env_file, file_keys) = resolve_env_file(env_file_var); + + if let Some(ref path) = env_file { + let _ = dotenvy::from_filename(path); + } else { + let _ = dotenvy::dotenv(); + } + + let mut keys_from_dotenv = HashSet::new(); + for key in file_keys { + if std::env::var(&key).is_ok() && !before.contains_key(&key) { + keys_from_dotenv.insert(key); + } + } + + DotenvLoad { + env_file, + keys_from_dotenv, + } +} + +fn snapshot_env() -> EnvSnapshot { + std::env::vars().collect() +} + +fn resolve_env_file(env_file_var: &str) -> (Option, HashSet) { + if let Ok(path) = std::env::var(env_file_var) { + if !path.is_empty() { + let p = PathBuf::from(&path); + let keys = parse_dotenv_keys(&p).unwrap_or_default(); + return (Some(p), keys); + } + } + + let dot_env = PathBuf::from(".env"); + if dot_env.is_file() { + let keys = parse_dotenv_keys(&dot_env).unwrap_or_default(); + (Some(dot_env), keys) + } else { + (None, HashSet::new()) + } +} + +/// Parse variable names from a dotenv file (ignores comments and blank lines). +fn parse_dotenv_keys(path: &Path) -> std::io::Result> { + let content = std::fs::read_to_string(path)?; + let mut keys = HashSet::new(); + for line in content.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let line = line.strip_prefix("export ").unwrap_or(line).trim(); + if let Some((key, _)) = line.split_once('=') { + let key = key.trim(); + if !key.is_empty() { + keys.insert(key.to_string()); + } + } + } + Ok(keys) +} + +/// Origin for a clap argument that may also use an environment variable. +pub fn origin_from_clap( + matches: &ArgMatches, + field_id: &str, + env_var: Option<&str>, + dotenv: &DotenvLoad, +) -> ConfigOrigin { + match matches.value_source(field_id) { + Some(ValueSource::CommandLine) => ConfigOrigin::Param, + Some(ValueSource::EnvVariable) => { + env_var + .and_then(|v| origin_for_env_var(v, dotenv)) + .unwrap_or(ConfigOrigin::Environment) + } + Some(ValueSource::DefaultValue) => ConfigOrigin::Default, + _ => ConfigOrigin::Default, + } +} + +/// Origin for a setting that is only available via environment (not a CLI flag). +pub fn origin_for_env_var(env_var: &str, dotenv: &DotenvLoad) -> Option { + if std::env::var(env_var).is_err() { + return None; + } + if dotenv.keys_from_dotenv.contains(env_var) { + Some(ConfigOrigin::DotEnv) + } else { + Some(ConfigOrigin::Environment) + } +} + +/// Print a startup configuration section to stdout. +pub fn log_section(component: &str, dotenv: &DotenvLoad) { + println!("{component} configuration:"); + if let Some(path) = &dotenv.env_file { + println!(" (loaded env file: {})", path.display()); + } +} + +/// Print one configuration line. +pub fn log_entry(name: &str, value: impl fmt::Display, origin: ConfigOrigin) { + println!(" {name} = {value} ({origin})"); +} + +/// Print one optional configuration line. +pub fn log_entry_optional( + name: &str, + value: Option, + origin: Option, + secret: bool, +) { + match (value, origin) { + (Some(v), Some(o)) => { + if secret { + log_entry(name, "(set)", o); + } else { + log_entry(name, v, o); + } + } + _ => println!(" {name} = (not set)"), + } +} + +/// Display value for sensitive settings. +pub fn display_secret(value: Option<&str>) -> String { + if value.is_some() && !value.unwrap_or("").is_empty() { + "(set)".to_string() + } else { + "(not set)".to_string() + } +} diff --git a/examples/registry-server/Cargo.toml b/examples/registry-server/Cargo.toml index a4cfa73..aa03e64 100644 --- a/examples/registry-server/Cargo.toml +++ b/examples/registry-server/Cargo.toml @@ -10,6 +10,7 @@ name = "registry-server" path = "src/main.rs" [dependencies] +config-origins = { path = "../../config-origins" } axum = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] } tower-http = { workspace = true } diff --git a/examples/registry-server/README.md b/examples/registry-server/README.md index 4e7f418..0162a25 100644 --- a/examples/registry-server/README.md +++ b/examples/registry-server/README.md @@ -23,8 +23,10 @@ This architecture keeps subsd private (it holds wallet keys) while the registry # Build cargo build --release -p registry-server -# Run +# Run (CLI or environment) registry-server --port 8080 +# REGISTRY_SERVER_PORT=8080 registry-server +# Loads .env from the current directory if present ``` Then configure subsd to use this registry: diff --git a/examples/registry-server/src/main.rs b/examples/registry-server/src/main.rs index 3ce88fb..b96fbb1 100644 --- a/examples/registry-server/src/main.rs +++ b/examples/registry-server/src/main.rs @@ -40,7 +40,8 @@ use axum::{ routing::{get, post}, Json, Router, }; -use clap::Parser; +use clap::{CommandFactory, FromArgMatches, Parser}; +use config_origins::{load_dotenv, log_entry, log_section, origin_from_clap}; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use tower_http::cors::{Any, CorsLayer}; @@ -54,7 +55,7 @@ use tower_http::trace::TraceLayer; )] struct Cli { /// Server port - #[arg(short, long, default_value = "8080")] + #[arg(short, long, env = "REGISTRY_SERVER_PORT", default_value = "8080")] port: u16, } @@ -82,6 +83,8 @@ enum RegistrationStatus { #[tokio::main] async fn main() -> anyhow::Result<()> { + let dotenv = load_dotenv("REGISTRY_SERVER_ENV_FILE"); + // Initialize tracing tracing_subscriber::fmt() .with_env_filter( @@ -90,7 +93,19 @@ async fn main() -> anyhow::Result<()> { ) .init(); - let cli = Cli::parse(); + let matches = Cli::command().get_matches(); + let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()); + + log_section("registry-server", &dotenv); + log_entry( + "port", + cli.port, + origin_from_clap(&matches, "port", Some("REGISTRY_SERVER_PORT"), &dotenv), + ); + println!( + " server_url = http://127.0.0.1:{} (derived from port)", + cli.port + ); let state = Arc::new(AppState { registrations: RwLock::new(Vec::new()), diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 1f06717..c9e53d6 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -15,10 +15,12 @@ path = "src/main.rs" risc0-zkvm = { workspace = true, features = ["prove"] } libveritas = { workspace = true } libveritas_zk = { workspace = true } +config-origins = { path = "../config-origins" } subs-types = { workspace = true } spacedb = { workspace = true } clap = { workspace = true } +dotenvy = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } anyhow = { workspace = true } diff --git a/prover/src/env.rs b/prover/src/env.rs new file mode 100644 index 0000000..95a94a2 --- /dev/null +++ b/prover/src/env.rs @@ -0,0 +1,71 @@ +//! Environment variable and `.env` file loading. + +use clap::ArgMatches; +use config_origins::{ + self as origins, origin_for_env_var, origin_from_clap, DotenvLoad, +}; + +pub use config_origins::load_dotenv; + +/// Log effective `subs-prover` configuration for server mode. +pub fn log_server_startup(matches: &ArgMatches, dotenv: &DotenvLoad, server: bool, port: u16) { + origins::log_section("subs-prover", dotenv); + origins::log_entry( + "server", + server, + origin_from_clap(matches, "server", Some("SUBS_PROVER_SERVER"), dotenv), + ); + origins::log_entry( + "server_port", + port, + origin_from_clap(matches, "server_port", Some("SUBS_PROVER_PORT"), dotenv), + ); + println!(" server_url = http://127.0.0.1:{} (derived from server_port)", port); +} + +/// Log configuration for a prove/compress subcommand. +pub fn log_subcommand_startup( + sub: &ArgMatches, + dotenv: &DotenvLoad, + sub_name: &str, + input: Option<&std::path::Path>, + output: Option<&std::path::Path>, +) { + origins::log_section("subs-prover", dotenv); + println!(" command = {sub_name} (param)"); + + log_io_path(sub, "input", "SUBS_PROVER_INPUT", input, dotenv); + log_io_path(sub, "output", "SUBS_PROVER_OUTPUT", output, dotenv); +} + +/// Log configuration for the bench subcommand. +pub fn log_bench_startup(dotenv: &DotenvLoad, sub: &ArgMatches, existing: usize, insert: usize) { + origins::log_section("subs-prover", dotenv); + println!(" command = bench (param)"); + origins::log_entry( + "bench_existing", + existing, + origin_from_clap(sub, "existing", Some("SUBS_PROVER_BENCH_EXISTING"), dotenv), + ); + origins::log_entry( + "bench_insert", + insert, + origin_from_clap(sub, "insert", Some("SUBS_PROVER_BENCH_INSERT"), dotenv), + ); +} + +fn log_io_path( + sub: &ArgMatches, + field_id: &str, + env_var: &str, + value: Option<&std::path::Path>, + dotenv: &DotenvLoad, +) { + let display = value.map(|p| p.display().to_string()); + let origin = match sub.value_source(field_id) { + Some(_) => Some(origin_from_clap(sub, field_id, Some(env_var), dotenv)), + None if display.is_some() => origin_for_env_var(env_var, dotenv), + None => None, + }; + origins::log_entry_optional(field_id, display.as_deref(), origin, false); +} diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 66ae237..2dbae01 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -2,6 +2,7 @@ //! //! Provides the `Prover` struct for generating STARK proofs and SNARK compression. +pub mod env; pub mod server; use std::time::Instant; diff --git a/prover/src/main.rs b/prover/src/main.rs index e3c73b8..62bef76 100644 --- a/prover/src/main.rs +++ b/prover/src/main.rs @@ -21,7 +21,7 @@ use std::io::{self, Read, Write}; use std::path::PathBuf; use anyhow::Result; -use clap::{Parser, Subcommand}; +use clap::{CommandFactory, FromArgMatches, Parser, Subcommand}; use subs_prover::Prover; use subs_types::{CompressInput, ProvingRequest}; @@ -33,11 +33,11 @@ use subs_types::{CompressInput, ProvingRequest}; )] struct Cli { /// Run as an HTTP server that accepts proving requests - #[arg(long)] + #[arg(long, env = "SUBS_PROVER_SERVER")] server: bool, /// Server port (for --server mode) - #[arg(long, default_value = "8888")] + #[arg(long, env = "SUBS_PROVER_PORT", default_value = "8888")] server_port: u16, #[command(subcommand)] @@ -49,55 +49,79 @@ enum Commands { /// Prove a ProvingRequest (Step or Fold) Prove { /// Input file (JSON ProvingRequest). If not provided, reads from stdin. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_INPUT")] input: Option, /// Output file for receipt. If not provided, writes to stdout. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_OUTPUT")] output: Option, }, /// Compress a STARK proof to SNARK (Groth16) Compress { /// Input file (JSON CompressInput). If not provided, reads from stdin. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_INPUT")] input: Option, /// Output file for receipt. If not provided, writes to stdout. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_OUTPUT")] output: Option, }, /// Benchmark: estimate proving cost for inserting handles into a tree Bench { /// Number of existing handles in the tree - #[arg(long, default_value = "10000")] + #[arg(long, env = "SUBS_PROVER_BENCH_EXISTING", default_value = "10000")] existing: usize, /// Number of new handles to insert - #[arg(long, default_value = "100")] + #[arg(long, env = "SUBS_PROVER_BENCH_INSERT", default_value = "100")] insert: usize, }, } #[tokio::main] async fn main() -> Result<()> { - let cli = Cli::parse(); + let dotenv = subs_prover::env::load_dotenv("SUBS_PROVER_ENV_FILE"); + let matches = Cli::command().get_matches(); + let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()); if cli.server { + subs_prover::env::log_server_startup(&matches, &dotenv, cli.server, cli.server_port); subs_prover::server::run_server(cli.server_port).await?; return Ok(()); } match cli.cmd { Some(Commands::Prove { input, output }) => { + if let Some((_, sub)) = matches.subcommand() { + subs_prover::env::log_subcommand_startup( + sub, + &dotenv, + "prove", + input.as_deref(), + output.as_deref(), + ); + } let input_data = read_input(input)?; let request: ProvingRequest = serde_json::from_slice(&input_data)?; let receipt = prove(&request)?; write_output(output, &receipt)?; } Some(Commands::Compress { input, output }) => { + if let Some((_, sub)) = matches.subcommand() { + subs_prover::env::log_subcommand_startup( + sub, + &dotenv, + "compress", + input.as_deref(), + output.as_deref(), + ); + } let input_data = read_input(input)?; let compress_input: CompressInput = serde_json::from_slice(&input_data)?; let receipt = compress(&compress_input)?; write_output(output, &receipt)?; } Some(Commands::Bench { existing, insert }) => { + if let Some((_, sub)) = matches.subcommand() { + subs_prover::env::log_bench_startup(&dotenv, sub, existing, insert); + } run_bench(existing, insert)?; } None => { diff --git a/subs/Cargo.toml b/subs/Cargo.toml index 3c1f312..27e9fc1 100644 --- a/subs/Cargo.toml +++ b/subs/Cargo.toml @@ -10,6 +10,7 @@ name = "subs" path = "src/main.rs" [dependencies] +config-origins = { path = "../config-origins" } subs-core = { workspace = true } subs-types = { workspace = true } @@ -20,6 +21,7 @@ tower-http = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } clap = { workspace = true } +dotenvy = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } anyhow = { workspace = true } diff --git a/subs/src/env.rs b/subs/src/env.rs new file mode 100644 index 0000000..02b3476 --- /dev/null +++ b/subs/src/env.rs @@ -0,0 +1,165 @@ +//! Environment variable and `.env` file loading. + +use std::path::Path; + +use anyhow::Result; +use clap::ArgMatches; +use config_origins::{ + self as origins, display_secret, origin_for_env_var, origin_from_clap, DotenvLoad, +}; + +use crate::config::ConfigStore; + +pub use config_origins::load_dotenv; + +/// Parsed configuration values used for startup logging. +pub struct StartupValues<'a> { + pub port: u16, + pub data_dir: &'a Path, + pub wallet: Option<&'a str>, + pub rpc_url: Option<&'a str>, + pub rpc_user: Option<&'a str>, + pub rpc_password: Option<&'a str>, + pub rpc_cookie: Option<&'a Path>, + #[cfg(feature = "test-rig")] + pub test_rig: bool, + #[cfg(feature = "test-rig")] + pub test_rig_dir: &'a Path, +} + +/// Log effective `subs` configuration and each value's origin. +pub fn log_startup(matches: &ArgMatches, dotenv: &DotenvLoad, cfg: StartupValues<'_>) { + origins::log_section("subs", dotenv); + + origins::log_entry( + "port", + cfg.port, + origin_from_clap(matches, "port", Some("SUBS_PORT"), dotenv), + ); + origins::log_entry( + "data_dir", + cfg.data_dir.display(), + origin_from_clap(matches, "data_dir", Some("SUBS_DATA_DIR"), dotenv), + ); + + log_field( + matches, + "wallet", + "SUBS_WALLET", + cfg.wallet, + dotenv, + false, + ); + log_field( + matches, + "rpc_url", + "SUBS_SPACED_RPC_URL", + cfg.rpc_url, + dotenv, + false, + ); + log_field( + matches, + "rpc_user", + "SUBS_SPACED_RPC_USER", + cfg.rpc_user, + dotenv, + false, + ); + log_field( + matches, + "rpc_password", + "SUBS_SPACED_RPC_PASSWORD", + cfg.rpc_password, + dotenv, + true, + ); + let rpc_cookie = cfg + .rpc_cookie + .map(|p| p.display().to_string()); + log_field( + matches, + "rpc_cookie", + "SUBS_SPACED_RPC_COOKIE", + rpc_cookie.as_deref(), + dotenv, + false, + ); + + log_env_only("prover_endpoint", "SUBS_PROVER_ENDPOINT", dotenv, false); + log_env_only("registry_endpoint", "SUBS_REGISTRY_ENDPOINT", dotenv, false); + + #[cfg(feature = "test-rig")] + { + origins::log_entry( + "test_rig", + cfg.test_rig, + origin_from_clap(matches, "test_rig", Some("SUBS_TEST_RIG"), dotenv), + ); + origins::log_entry( + "test_rig_dir", + cfg.test_rig_dir.display(), + origin_from_clap(matches, "test_rig_dir", Some("SUBS_TEST_RIG_DIR"), dotenv), + ); + } + + println!( + " server_url = http://127.0.0.1:{} (derived from port)", + cfg.port + ); +} + +fn log_field( + matches: &ArgMatches, + field_id: &str, + env_var: &str, + value: Option<&str>, + dotenv: &DotenvLoad, + secret: bool, +) { + let origin = match matches.value_source(field_id) { + Some(_) => Some(origin_from_clap(matches, field_id, Some(env_var), dotenv)), + None if value.is_some() && origin_for_env_var(env_var, dotenv).is_some() => { + origin_for_env_var(env_var, dotenv) + } + None => None, + }; + + if secret { + let display = display_secret(value); + if let Some(o) = origin { + origins::log_entry(field_id, display, o); + } else { + println!(" {field_id} = {display}"); + } + } else { + origins::log_entry_optional(field_id, value, origin, false); + } +} + +fn log_env_only(name: &str, env_var: &str, dotenv: &DotenvLoad, secret: bool) { + let value = std::env::var(env_var).ok(); + let origin = origin_for_env_var(env_var, dotenv); + if secret { + origins::log_entry_optional(name, value.as_deref().map(|_| "(set)"), origin, true); + } else { + origins::log_entry_optional(name, value.as_deref(), origin, false); + } +} + +/// Apply optional runtime settings from the environment into `config.db`. +pub fn apply_runtime_config_from_env(config: &ConfigStore) -> Result<()> { + if let Ok(url) = std::env::var("SUBS_PROVER_ENDPOINT") { + let url = url.trim(); + if !url.is_empty() { + config.set_prover_endpoint(url)?; + } + } + if let Ok(url) = std::env::var("SUBS_REGISTRY_ENDPOINT") { + let url = url.trim(); + if !url.is_empty() { + config.set_registry_endpoint(url)?; + } + } + Ok(()) +} diff --git a/subs/src/main.rs b/subs/src/main.rs index 0517ff7..13b78e5 100644 --- a/subs/src/main.rs +++ b/subs/src/main.rs @@ -14,6 +14,7 @@ mod background; mod config; +mod env; mod routes; mod state; @@ -24,7 +25,7 @@ use std::net::SocketAddr; use std::path::PathBuf; use anyhow::Result; -use clap::Parser; +use clap::{CommandFactory, FromArgMatches, Parser}; use subs_core::Operator; use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; @@ -41,46 +42,48 @@ use crate::state::AppState; )] struct Cli { /// Server port - #[arg(short, long, default_value = "7777")] + #[arg(short, long, env = "SUBS_PORT", default_value = "7777")] port: u16, /// Data directory for spaces - #[arg(short, long, default_value = "./data")] + #[arg(short, long, env = "SUBS_DATA_DIR", default_value = "./data")] data_dir: PathBuf, /// Wallet name for signing operations (not required with --test-rig) - #[arg(short, long, required_unless_present = "test_rig")] + #[arg(short, long, env = "SUBS_WALLET", required_unless_present = "test_rig")] wallet: Option, /// Spaces RPC URL (not required with --test-rig) - #[arg(short, long, required_unless_present = "test_rig")] + #[arg(short, long, env = "SUBS_SPACED_RPC_URL", required_unless_present = "test_rig")] rpc_url: Option, /// RPC username (optional) - #[arg(long)] + #[arg(long, env = "SUBS_SPACED_RPC_USER")] rpc_user: Option, /// RPC password (optional) - #[arg(long)] + #[arg(long, env = "SUBS_SPACED_RPC_PASSWORD")] rpc_password: Option, /// RPC cookie file path (optional) - #[arg(long)] + #[arg(long, env = "SUBS_SPACED_RPC_COOKIE")] rpc_cookie: Option, /// Enable test rig mode (starts bitcoind + spaced automatically) #[cfg(feature = "test-rig")] - #[arg(long)] + #[arg(long, env = "SUBS_TEST_RIG")] test_rig: bool, /// Directory for test rig data (persistent across restarts) #[cfg(feature = "test-rig")] - #[arg(long, default_value = "./testrig-data")] + #[arg(long, env = "SUBS_TEST_RIG_DIR", default_value = "./testrig-data")] test_rig_dir: PathBuf, } #[tokio::main] async fn main() -> Result<()> { + let dotenv = env::load_dotenv("SUBS_ENV_FILE"); + // Initialize tracing tracing_subscriber::registry() .with( @@ -90,7 +93,25 @@ async fn main() -> Result<()> { .with(tracing_subscriber::fmt::layer()) .init(); - let cli = Cli::parse(); + let matches = Cli::command().get_matches(); + let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()); + env::log_startup( + &matches, + &dotenv, + env::StartupValues { + port: cli.port, + data_dir: &cli.data_dir, + wallet: cli.wallet.as_deref(), + rpc_url: cli.rpc_url.as_deref(), + rpc_user: cli.rpc_user.as_deref(), + rpc_password: cli.rpc_password.as_deref(), + rpc_cookie: cli.rpc_cookie.as_deref(), + #[cfg(feature = "test-rig")] + test_rig: cli.test_rig, + #[cfg(feature = "test-rig")] + test_rig_dir: &cli.test_rig_dir, + }, + ); #[cfg(feature = "test-rig")] { @@ -141,6 +162,7 @@ async fn run_normal(cli: Cli) -> Result<()> { // Create config store let config_path = cli.data_dir.join("config.db"); let config = ConfigStore::open(&config_path)?; + env::apply_runtime_config_from_env(&config)?; // Create operator let operator = Operator::new(cli.data_dir, wallet, rpc) @@ -202,6 +224,7 @@ async fn run_with_test_rig(cli: Cli) -> Result { // Create config store let config_path = cli.data_dir.join("config.db"); let config = ConfigStore::open(&config_path)?; + env::apply_runtime_config_from_env(&config)?; // Use default wallet from test rig let wallet = "wallet_99"; @@ -294,7 +317,6 @@ async fn run_server_inner(state: AppState, port: u16) -> Result<()> { let addr = SocketAddr::from(([0, 0, 0, 0], port)); tracing::info!("Starting server on http://{}", addr); tracing::info!("Server URL: http://127.0.0.1:{}", port); - println!("Server URL: http://127.0.0.1:{} (port {})", port, port); let listener = tokio::net::TcpListener::bind(addr).await?; axum::serve(listener, app) From 925af026f5006ba03ebd35dbfff8e0f6f4ea0c76 Mon Sep 17 00:00:00 2001 From: spacesops Date: Fri, 22 May 2026 14:19:56 -0400 Subject: [PATCH 3/3] v0.1.0 none --- .dockerignore | 12 ++++ .env.example | 8 ++- .gitignore | 6 +- Dockerfile | 132 ++++++++++++++++++++++++++++++++++ README.md | 143 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 58 +++++++++++++++ docker/entrypoint.sh | 121 +++++++++++++++++++++++++++++++ subs/src/main.rs | 4 ++ subs/src/routes/console.rs | 54 ++++++++++---- subs/src/state.rs | 9 +++ 10 files changed, 528 insertions(+), 19 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7469e63 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +target/ +.git/ +.idea/ +.vscode/ +testrig-data/ +data/ +datamad/ +NOTES.md +*.md +!README.md +.env +.DS_Store diff --git a/.env.example b/.env.example index 1f72bbd..021d8e6 100644 --- a/.env.example +++ b/.env.example @@ -9,8 +9,12 @@ SUBS_SPACED_RPC_URL=http://127.0.0.1:7225 # SUBS_SPACED_RPC_USER=testuser # SUBS_SPACED_RPC_PASSWORD=secret # SUBS_SPACED_RPC_COOKIE=/path/to/.cookie -# SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888 -# SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8080 +# In Docker, subs-prover and registry-server run in the same container as subs by default. +SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888 +SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8080 +# SUBS_START_PROVER=1 +# SUBS_START_REGISTRY=1 +# SUBS_PROVER_SERVER=1 # SUBS_ENV_FILE=.env # SUBS_TEST_RIG=1 # SUBS_TEST_RIG_DIR=./testrig-data diff --git a/.gitignore b/.gitignore index 3571e4d..a6e2917 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ target/ *.sdb .idea testrig-data -data -datamad +data/ +datamad/ +datamadd/ NOTES.md +.cargo/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9888080 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,132 @@ +# syntax=docker/dockerfile:1 + +# Rust toolchain on Alpine (musl) for release binaries. +FROM rust:1-alpine3.21 AS builder-base + +ARG CARGO_BUILD_JOBS=1 + +RUN apk add --no-cache \ + build-base \ + musl-dev \ + git \ + openssl-dev \ + openssl-libs-static \ + pkgconf \ + clang \ + llvm-dev \ + lld \ + libatomic \ + ca-certificates + +WORKDIR /app +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV RUSTFLAGS="-C link-arg=-fuse-ld=lld" + +# subs + registry share a target/ tree (small compared to subs-prover). +FROM builder-base AS builder-subs + +ARG ENABLE_REGISTRY=true + +COPY . . + +RUN set -eux; \ + cargo build --release -p subs; \ + if [ "$ENABLE_REGISTRY" != "false" ]; then \ + cargo build --release -p registry-server; \ + fi; \ + mkdir -p /out; \ + cp target/release/subs /out/; \ + if [ "$ENABLE_REGISTRY" != "false" ]; then \ + cp target/release/registry-server /out/; \ + fi; \ + cargo clean + +# subs-prover (RISC Zero) in a fresh stage so target/ does not stack on top of subs. +FROM builder-base AS builder-prover + +ARG ENABLE_PROVER=true +ARG GPU_ACCELERATION=none +ARG TARGETARCH + +COPY . . + +RUN set -eux; \ + if [ "$ENABLE_PROVER" = "false" ]; then \ + mkdir -p /out; \ + exit 0; \ + fi; \ + if [ "$TARGETARCH" = "arm64" ]; then \ + export CFLAGS="-mno-outline-atomics"; \ + export CXXFLAGS="-mno-outline-atomics"; \ + export CMAKE_C_FLAGS="-mno-outline-atomics"; \ + export CMAKE_CXX_FLAGS="-mno-outline-atomics"; \ + export RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-feature=-outline-atomics"; \ + else \ + export RUSTFLAGS="-C link-arg=-fuse-ld=lld"; \ + fi; \ + case "$GPU_ACCELERATION" in \ + none) cargo build --release -p subs-prover ;; \ + metal) cargo build --release -p subs-prover --features metal ;; \ + cuda) cargo build --release -p subs-prover --features cuda ;; \ + *) echo "Invalid GPU_ACCELERATION=$GPU_ACCELERATION (expected none, metal, or cuda)" >&2; exit 1 ;; \ + esac; \ + mkdir -p /out; \ + cp target/release/subs-prover /out/; \ + cargo clean + +FROM alpine:3.21 + +ARG ENABLE_PROVER=true +ARG ENABLE_REGISTRY=true +ARG GPU_ACCELERATION=none + +RUN apk add --no-cache ca-certificates libgcc tini \ + && addgroup -S subs \ + && adduser -S subs -G subs + +COPY --from=builder-subs /out/subs /usr/local/bin/subs + +RUN --mount=type=bind,from=builder-subs,source=/out,target=/subs-out \ + --mount=type=bind,from=builder-prover,source=/out,target=/prover-out \ + set -eux; \ + if [ "$ENABLE_REGISTRY" != "false" ] && [ -f /subs-out/registry-server ]; then \ + cp /subs-out/registry-server /usr/local/bin/registry-server; \ + fi; \ + if [ "$ENABLE_PROVER" != "false" ] && [ -f /prover-out/subs-prover ]; then \ + cp /prover-out/subs-prover /usr/local/bin/subs-prover; \ + fi; \ + : > /etc/subs-image.env; \ + if [ "$ENABLE_PROVER" != "false" ]; then \ + echo "SUBS_START_PROVER=1" >> /etc/subs-image.env; \ + echo "SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888" >> /etc/subs-image.env; \ + echo "SUBS_PROVER_SERVER=1" >> /etc/subs-image.env; \ + echo "SUBS_PROVER_GPU_ACCELERATION=${GPU_ACCELERATION}" >> /etc/subs-image.env; \ + else \ + echo "SUBS_START_PROVER=0" >> /etc/subs-image.env; \ + fi; \ + if [ "$ENABLE_REGISTRY" != "false" ]; then \ + echo "SUBS_START_REGISTRY=1" >> /etc/subs-image.env; \ + echo "SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8080" >> /etc/subs-image.env; \ + else \ + echo "SUBS_START_REGISTRY=0" >> /etc/subs-image.env; \ + fi + +COPY docker/entrypoint.sh /entrypoint.sh + +RUN chmod +x /entrypoint.sh \ + && mkdir -p /data \ + && chown -R subs:subs /data + +WORKDIR /data +USER subs + +ENV SUBS_DATA_DIR=/data +ENV SUBS_PORT=7777 +ENV SUBS_PROVER_PORT=8888 +ENV REGISTRY_SERVER_PORT=8080 + +EXPOSE 7777 8888 8080 + +ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"] +CMD ["subs"] diff --git a/README.md b/README.md index 6e3d4db..51f9c2e 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,149 @@ subs configuration: CLI flags override environment variables; process environment overrides `.env` for the same key. +## Docker + +The image is built from **Rust on Alpine** (musl). By default it includes `subs`, `subs-prover`, and `registry-server`; use build args to omit optional components. An entrypoint dispatches by component name or `SUBS_COMPONENT`. + +When all components are included, starting `subs` also starts **subs-prover** and **registry-server** in the same container: + +| Service | Default URL | Disable with | +|---------|-------------|--------------| +| subs-prover | `http://127.0.0.1:8888` (`SUBS_PROVER_ENDPOINT`) | `SUBS_START_PROVER=0` | +| registry-server | `http://127.0.0.1:8080` (`SUBS_REGISTRY_ENDPOINT`) | `SUBS_START_REGISTRY=0` | + +**Note:** The image build includes RISC Zero proving when `ENABLE_PROVER` is enabled; use `GPU_ACCELERATION` to select CPU (`none`), Apple Metal (`metal`), or NVIDIA CUDA (`cuda`) for `subs-prover`. + +### Build + +Full image (subs + prover + registry): + +```bash +docker build -t subs:latest . +``` + +Subs only (faster build; skips RISC Zero prover and registry): + +```bash +docker build -t subs:slim \ + --build-arg ENABLE_PROVER=false \ + --build-arg ENABLE_REGISTRY=false . +``` + +Omit only the prover: + +```bash +docker build -t subs:no-prover --build-arg ENABLE_PROVER=false . +``` + +Omit only the registry: + +```bash +docker build -t subs:no-registry --build-arg ENABLE_REGISTRY=false . +``` + +Build args: + +| Build arg | Default | Description | +|-----------|---------|-------------| +| `ENABLE_PROVER` | `true` | Set to `false` to skip building/shipping `subs-prover` | +| `ENABLE_REGISTRY` | `true` | Set to `false` to skip building/shipping `registry-server` | +| `GPU_ACCELERATION` | `none` | `subs-prover` features: `none` (CPU), `metal`, or `cuda` | +| `CARGO_BUILD_JOBS` | `1` | Parallel `rustc` jobs in the builder (raise only if Docker has enough RAM) | + +**Memory:** A full image with `subs-prover` often needs **8 GB+** RAM for the Docker builder VM. If the build fails with `cannot allocate memory`, increase **Docker Desktop → Settings → Resources → Memory**, keep `CARGO_BUILD_JOBS=1` (default), or build without the prover. + +**Disk:** `subs-prover` (RISC Zero) can use **20–40 GB** under `target/` during the build. If you see `No space left on device (os error 28)`, free Docker space and raise the disk limit: + +```bash +docker system df +docker builder prune -af # drops build cache (safe before a clean rebuild) +``` + +Docker Desktop → **Settings → Resources → Disk image size** → **64 GB+** (or **Clean / Purge data** if the VM is full), then rebuild. + +**Linker (`__aarch64_cas4_sync` / `__aarch64_swp4_sync`):** On Alpine **arm64**, the prover build uses `-mno-outline-atomics` / `-C target-feature=-outline-atomics` (see Dockerfile `builder-prover` and `.cargo/config.toml`). If you changed those flags, rebuild without cache: `docker buildx build --no-cache-filter builder-prover ...`. + +```bash +docker build -t subs:slim --build-arg ENABLE_PROVER=false . +``` + +```bash +# NVIDIA CUDA prover (Linux hosts with GPU) +docker build -t subs:cuda --build-arg GPU_ACCELERATION=cuda . + +# Apple Metal prover (macOS/arm64 builds) +docker build -t subs:metal --build-arg GPU_ACCELERATION=metal . +``` + +### Run `subs` + +Point at a `spaced` instance reachable from the container (use `host.docker.internal` on Docker Desktop for a node on the host): + +```bash +docker run --rm \ + -p 7777:7777 -p 8888:8888 -p 8080:8080 \ + -v subs-data:/data \ + -e SUBS_SPACED_RPC_URL=http://host.docker.internal:7225 \ + -e SUBS_WALLET=my-wallet \ + -e SUBS_SPACED_RPC_USER=testuser \ + -e SUBS_SPACED_RPC_PASSWORD=secret \ + subs:latest subs +``` + +(`SUBS_PROVER_ENDPOINT` and `SUBS_REGISTRY_ENDPOINT` default to `http://127.0.0.1:8888` and `http://127.0.0.1:8080` in the image.) + +Or mount a `.env` file: + +```bash +docker run --rm -p 7777:7777 \ + -v "$(pwd)/.env:/data/.env:ro" \ + -v subs-data:/data \ + -e SUBS_ENV_FILE=/data/.env \ + subs:latest +``` + +### Run `subs-prover` only + +The default `subs` command already starts subs-prover in the same container. To run the prover alone: + +```bash +docker run --rm -p 8888:8888 \ + -e SUBS_START_PROVER=0 \ + -e SUBS_START_REGISTRY=0 \ + subs:latest subs-prover --server +``` + +### Run `registry-server` only + +The default `subs` command already starts registry-server in the same container. To run registry alone: + +```bash +docker run --rm -p 8080:8080 \ + -e SUBS_START_PROVER=0 \ + -e SUBS_START_REGISTRY=0 \ + subs:latest registry-server +``` + +### Docker Compose + +Starts `subs` with embedded subs-prover (8888) and registry-server (8080) in the same container: + +```bash +cp .env.example .env +# Set SUBS_SPACED_RPC_URL=http://host.docker.internal:7225 and SUBS_WALLET=... +docker compose up --build +``` + +Optional standalone services: + +```bash +docker compose --profile prover-only up --build +docker compose --profile registry-only up --build +``` + +Open http://localhost:7777 for the operator UI. Prover and registry APIs are at http://localhost:8888 and http://localhost:8080. Compose sets `SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888` and `SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8080` by default. + ## Usage ### 1. Start the prover server diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..df09679 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,58 @@ +services: + subs: + build: . + image: subs:latest + command: ["subs"] + env_file: + - path: .env + required: false + environment: + SUBS_DATA_DIR: /data + # subs-prover and registry-server start in the same container (see docker/entrypoint.sh). + SUBS_PROVER_ENDPOINT: http://127.0.0.1:8888 + SUBS_REGISTRY_ENDPOINT: http://127.0.0.1:8080 + SUBS_START_PROVER: "1" + SUBS_START_REGISTRY: "1" + SUBS_PROVER_SERVER: "1" + SUBS_PROVER_PORT: "8888" + REGISTRY_SERVER_PORT: "8080" + RUST_LOG: subs=info,subs_prover=info,registry_server=info,tower_http=debug + volumes: + - subs-data:/data + ports: + - "${SUBS_PORT:-7777}:7777" + - "${SUBS_PROVER_PORT:-8888}:8888" + - "${REGISTRY_SERVER_PORT:-8080}:8080" + + # Optional: run a single component alone (embedded services disabled). + subs-prover: + build: . + image: subs:latest + command: ["subs-prover", "--server"] + environment: + SUBS_START_PROVER: "0" + SUBS_START_REGISTRY: "0" + SUBS_PROVER_SERVER: "1" + SUBS_PROVER_PORT: "8888" + RUST_LOG: subs_prover=info,tower_http=debug + ports: + - "${SUBS_PROVER_PORT:-8888}:8888" + profiles: + - prover-only + + registry-server: + build: . + image: subs:latest + command: ["registry-server"] + environment: + SUBS_START_PROVER: "0" + SUBS_START_REGISTRY: "0" + REGISTRY_SERVER_PORT: "8080" + RUST_LOG: registry_server=info,tower_http=debug + ports: + - "${REGISTRY_SERVER_PORT:-8080}:8080" + profiles: + - registry-only + +volumes: + subs-data: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..c2356a0 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,121 @@ +#!/bin/sh +# Dispatch to subs, subs-prover, or registry-server. +# By default, starting subs also starts co-located services when their binaries exist. +# Usage: +# docker run subs [flags...] +# docker run subs-prover --server +# docker run registry-server --port 8080 + +set -eu + +PROVER_PORT="${SUBS_PROVER_PORT:-8888}" +REGISTRY_PORT="${REGISTRY_SERVER_PORT:-8080}" + +# Apply image defaults from build (only for unset variables). +load_image_defaults() { + if [ ! -f /etc/subs-image.env ]; then + return 0 + fi + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|\#*) continue ;; + esac + key="${line%%=*}" + value="${line#*=}" + eval "if [ -z \"\${$key+x}\" ]; then export $key=\"$value\"; fi" + done < /etc/subs-image.env +} + +load_image_defaults + +if [ -n "${SUBS_PROVER_GPU_ACCELERATION:-}" ]; then + echo "entrypoint: subs-prover GPU acceleration: ${SUBS_PROVER_GPU_ACCELERATION}" +fi + +require_binary() { + if [ ! -x "$1" ]; then + echo "entrypoint: $1 is not available in this image (rebuild with ENABLE_PROVER/ENABLE_REGISTRY enabled)" >&2 + exit 1 + fi +} + +# Start subs-prover in the background (co-located with subs). +start_prover_server() { + if [ "${SUBS_START_PROVER:-1}" = "0" ] || [ ! -x /usr/local/bin/subs-prover ]; then + return 0 + fi + echo "entrypoint: starting subs-prover on 127.0.0.1:${PROVER_PORT}" + SUBS_PROVER_SERVER=1 SUBS_PROVER_PORT="${PROVER_PORT}" \ + /usr/local/bin/subs-prover --server --server-port "${PROVER_PORT}" & +} + +# Start registry-server in the background (co-located with subs). +start_registry_server() { + if [ "${SUBS_START_REGISTRY:-1}" = "0" ] || [ ! -x /usr/local/bin/registry-server ]; then + return 0 + fi + echo "entrypoint: starting registry-server on 127.0.0.1:${REGISTRY_PORT}" + /usr/local/bin/registry-server --port "${REGISTRY_PORT}" & +} + +resolve_component() { + if [ -n "${SUBS_COMPONENT:-}" ]; then + printf '%s' "$SUBS_COMPONENT" + return + fi + + if [ "$#" -gt 0 ]; then + case "$1" in + subs|subs-prover|prover|registry-server|registry) + printf '%s' "$1" + return + ;; + esac + fi + + printf '%s' "subs" +} + +COMPONENT="$(resolve_component)" + +case "$COMPONENT" in + subs) + BIN=/usr/local/bin/subs + ;; + subs-prover|prover) + BIN=/usr/local/bin/subs-prover + COMPONENT=subs-prover + require_binary "$BIN" + ;; + registry-server|registry) + BIN=/usr/local/bin/registry-server + COMPONENT=registry-server + require_binary "$BIN" + ;; + *) + echo "entrypoint: unknown SUBS_COMPONENT '$COMPONENT' (expected subs, subs-prover, or registry-server)" >&2 + exit 1 + ;; +esac + +# If the first argument was the component name, shift it off before exec. +if [ "$#" -gt 0 ]; then + case "$1" in + subs|subs-prover|prover|registry-server|registry) + shift + ;; + esac +fi + +if [ "$COMPONENT" = "subs" ]; then + start_prover_server + start_registry_server + if [ -x /usr/local/bin/subs-prover ] && [ -z "${SUBS_PROVER_ENDPOINT:-}" ]; then + export SUBS_PROVER_ENDPOINT="http://127.0.0.1:${PROVER_PORT}" + fi + if [ -x /usr/local/bin/registry-server ] && [ -z "${SUBS_REGISTRY_ENDPOINT:-}" ]; then + export SUBS_REGISTRY_ENDPOINT="http://127.0.0.1:${REGISTRY_PORT}" + fi +fi + +exec "$BIN" "$@" diff --git a/subs/src/main.rs b/subs/src/main.rs index 13b78e5..d9b418f 100644 --- a/subs/src/main.rs +++ b/subs/src/main.rs @@ -179,6 +179,7 @@ async fn run_normal(cli: Cli) -> Result<()> { Some(rpc_url.clone()), cli.rpc_user.clone(), cli.rpc_password.clone(), + cli.rpc_cookie.clone(), None, ) .await @@ -260,6 +261,7 @@ async fn run_server( spaced_rpc_url: Option, spaced_rpc_user: Option, spaced_rpc_password: Option, + spaced_rpc_cookie: Option, bitcoin_rpc_url: Option, ) -> Result<()> { // Build app state @@ -269,6 +271,7 @@ async fn run_server( spaced_rpc_url, spaced_rpc_user, spaced_rpc_password, + spaced_rpc_cookie, bitcoin_rpc_url, ); run_server_inner(state, port).await @@ -291,6 +294,7 @@ async fn run_server_with_testrig( Some(spaced_rpc_url), Some("user".to_string()), Some("pass".to_string()), + None, Some(bitcoin_rpc_url), Some(certrelay_url), test_rig, diff --git a/subs/src/routes/console.rs b/subs/src/routes/console.rs index 9e585ba..bdbd4aa 100644 --- a/subs/src/routes/console.rs +++ b/subs/src/routes/console.rs @@ -8,9 +8,35 @@ use axum::{ }; use serde::{Deserialize, Serialize}; +use reqwest::RequestBuilder; + use crate::state::AppState; use super::json_error; +/// Apply Spaced RPC credentials to an outbound request (same precedence as `build_rpc_client`). +fn apply_spaced_rpc_auth( + req: RequestBuilder, + state: &AppState, +) -> Result { + if let Some(user) = state.spaced_rpc_user.as_deref() { + return Ok(req.basic_auth(user, state.spaced_rpc_password.as_deref())); + } + if let Some(path) = state.spaced_rpc_cookie.as_ref() { + let cookie = std::fs::read_to_string(path).map_err(|e| { + json_error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read RPC cookie file: {e}"), + ) + })?; + let encoded = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + cookie.trim().as_bytes(), + ); + return Ok(req.header("Authorization", format!("Basic {encoded}"))); + } + Ok(req) +} + #[derive(Debug, Deserialize)] pub struct RpcRequest { pub method: String, @@ -84,12 +110,7 @@ pub async fn proxy_spaced( .as_ref() .ok_or_else(|| json_error(StatusCode::SERVICE_UNAVAILABLE, "Spaced RPC URL not configured"))?; - let auth = state - .spaced_rpc_user - .as_deref() - .map(|user| (user, state.spaced_rpc_password.as_deref().unwrap_or(""))); - - proxy_rpc_call(rpc_url, &request, auth).await + proxy_rpc_call(rpc_url, &request, |req| apply_spaced_rpc_auth(req, &state)).await } /// POST /rpc/bitcoin - Proxy RPC call to bitcoind (test-rig only) @@ -102,7 +123,10 @@ pub async fn proxy_bitcoin( .as_ref() .ok_or_else(|| json_error(StatusCode::SERVICE_UNAVAILABLE, "Bitcoin RPC not available (only in test-rig mode)"))?; - proxy_rpc_call(rpc_url, &request, Some(("user", "password"))).await + proxy_rpc_call(rpc_url, &request, |req| { + Ok(req.basic_auth("user", Some("password"))) + }) + .await } /// POST /rpc/mine - Mine blocks (test-rig only) @@ -135,11 +159,14 @@ pub async fn mine_blocks( Err(json_error(StatusCode::SERVICE_UNAVAILABLE, "Mining only available in test-rig mode")) } -async fn proxy_rpc_call( +async fn proxy_rpc_call( rpc_url: &str, request: &RpcRequest, - auth: Option<(&str, &str)>, -) -> Result, Response> { + apply_auth: F, +) -> Result, Response> +where + F: FnOnce(RequestBuilder) -> Result, +{ let client = reqwest::Client::new(); // Build JSON-RPC request @@ -150,14 +177,11 @@ async fn proxy_rpc_call( "params": request.params, }); - let mut req = client + let req = client .post(rpc_url) .header("Content-Type", "application/json") .json(&rpc_body); - - if let Some((user, pass)) = auth { - req = req.basic_auth(user, Some(pass)); - } + let req = apply_auth(req)?; let response = req .timeout(std::time::Duration::from_secs(30)) diff --git a/subs/src/state.rs b/subs/src/state.rs index 5f03320..3a5115a 100644 --- a/subs/src/state.rs +++ b/subs/src/state.rs @@ -1,5 +1,6 @@ //! Application state for the subsd server. +use std::path::PathBuf; use std::sync::Arc; use subs_core::Operator; @@ -20,6 +21,8 @@ pub struct AppState { pub spaced_rpc_user: Option, /// Spaced RPC password for proxied calls pub spaced_rpc_password: Option, + /// Spaced RPC cookie file for proxied calls (used when user/password not set) + pub spaced_rpc_cookie: Option, /// Bitcoin RPC URL (only available in test-rig mode) pub bitcoin_rpc_url: Option, /// Certrelay URL (only available in test-rig mode) @@ -37,6 +40,7 @@ impl AppState { spaced_rpc_url: Option, spaced_rpc_user: Option, spaced_rpc_password: Option, + spaced_rpc_cookie: Option, _bitcoin_rpc_url: Option, ) -> Self { Self { @@ -45,6 +49,7 @@ impl AppState { spaced_rpc_url, spaced_rpc_user, spaced_rpc_password, + spaced_rpc_cookie, bitcoin_rpc_url: None, certrelay_url: None, } @@ -57,6 +62,7 @@ impl AppState { spaced_rpc_url: Option, spaced_rpc_user: Option, spaced_rpc_password: Option, + spaced_rpc_cookie: Option, bitcoin_rpc_url: Option, ) -> Self { Self { @@ -65,6 +71,7 @@ impl AppState { spaced_rpc_url, spaced_rpc_user, spaced_rpc_password, + spaced_rpc_cookie, bitcoin_rpc_url, certrelay_url: None, test_rig: None, @@ -78,6 +85,7 @@ impl AppState { spaced_rpc_url: Option, spaced_rpc_user: Option, spaced_rpc_password: Option, + spaced_rpc_cookie: Option, bitcoin_rpc_url: Option, certrelay_url: Option, test_rig: Arc, @@ -88,6 +96,7 @@ impl AppState { spaced_rpc_url, spaced_rpc_user, spaced_rpc_password, + spaced_rpc_cookie, bitcoin_rpc_url, certrelay_url, test_rig: Some(test_rig),