From 0dfdc951f3aa40120dbd8d7f6f0891778a7b1bda Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 24 Sep 2024 23:07:16 -0400 Subject: [PATCH 01/19] minimal code change to get windmill worker on windows for bun and python + rustfmt --- backend/windmill-api/src/jobs.rs | 4 +- backend/windmill-api/src/resources.rs | 5 +- backend/windmill-common/src/worker.rs | 9 ++- backend/windmill-indexer/src/indexer_ee.rs | 2 +- .../windmill-worker/src/ansible_executor.rs | 4 +- backend/windmill-worker/src/bun_executor.rs | 39 ++++++++- backend/windmill-worker/src/go_executor.rs | 8 ++ backend/windmill-worker/src/lib.rs | 2 +- .../windmill-worker/src/python_executor.rs | 60 ++++++++++---- backend/windmill-worker/src/rust_executor.rs | 47 ++++++++++- backend/windmill-worker/src/worker.rs | 79 ++++++++++++------- 11 files changed, 203 insertions(+), 56 deletions(-) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index e0c118c7b339d..82067d8642978 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -4644,8 +4644,8 @@ async fn get_job_update( .fetch_optional(&db) .await?; - let progress: Option = if get_progress == Some(true){ - sqlx::query_scalar!( + let progress: Option = if get_progress == Some(true) { + sqlx::query_scalar!( "SELECT scalar_int FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &w_id, job_id, diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 1fb5b122592e7..bb881d5369b65 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -58,7 +58,10 @@ pub fn workspaced_service() -> Router { .route("/type/exists/:name", get(exists_resource_type)) .route("/type/update/:name", post(update_resource_type)) .route("/type/delete/:name", delete(delete_resource_type)) - .route("/file_resource_type_to_file_ext_map", get(file_resource_ext_to_resource_type)) + .route( + "/file_resource_type_to_file_ext_map", + get(file_resource_ext_to_resource_type), + ) .route("/type/create", post(create_resource_type)) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ce07438dd57d3..11e03e27a8641 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -195,7 +195,11 @@ fn normalize_path(path: &Path) -> PathBuf { } ret } -pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &str, content: &str) -> error::Result { +pub fn write_file_at_user_defined_location( + job_dir: &str, + user_defined_path: &str, + content: &str, +) -> error::Result { let job_dir = Path::new(job_dir); let user_path = PathBuf::from(user_defined_path); @@ -210,7 +214,8 @@ pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &st return Err(std::io::Error::new( std::io::ErrorKind::PermissionDenied, "Path is outside the allowed job directory.", - ).into()); + ) + .into()); } if let Some(parent_dir) = full_path.parent() { diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs index 79bcbcff778a6..da92ff0ef82fd 100644 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ b/backend/windmill-indexer/src/indexer_ee.rs @@ -1,6 +1,6 @@ +use anyhow::anyhow; use sqlx::{Pool, Postgres}; use windmill_common::error::Error; -use anyhow::anyhow; #[derive(Clone)] pub struct IndexReader; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 785edfa85b781..324d06ff5d950 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -14,8 +14,8 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - get_reserved_variables, handle_child, read_and_check_result, - start_child_process, transform_json, + get_reserved_variables, handle_child, read_and_check_result, start_child_process, + transform_json, }, python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, HOME_ENV, PATH_ENV, TZ_ENV, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 95ec6e3f9fec0..4e7bad2aaaaf7 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -19,7 +19,7 @@ use crate::{ }, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, - NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, + NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, SYSTEM_ROOT, TZ_ENV, }; use tokio::{fs::File, process::Command}; @@ -110,6 +110,9 @@ pub async fn gen_bun_lockfile( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child_cmd, &*BUN_PATH).await?; if let Some(db) = db { @@ -240,6 +243,9 @@ pub async fn install_bun_lockfile( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut npm_logs = if npm_mode { "NPM mode\n".to_string() } else { @@ -446,6 +452,10 @@ pub async fn generate_wrapper_mjs( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let child_process = start_child_process(child, &*BUN_PATH).await?; handle_child( job_id, @@ -489,6 +499,10 @@ pub async fn generate_bun_bundle( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child, &*BUN_PATH).await?; if let Some(db) = db { handle_child( @@ -710,6 +724,10 @@ async fn compute_bundle_local_and_remote_path( let hash = windmill_common::utils::calculate_hash(&input_src); let local_path = format!("{BUN_BUNDLE_CACHE_DIR}/{hash}"); + + #[cfg(windows)] + let local_path = local_path.replace("/tmp", r"C:\tmp").replace("/", r"\"); + let remote_path = format!("{BUN_BUNDLE_OBJECT_STORE_PREFIX}{hash}"); (local_path, remote_path) } @@ -802,14 +820,25 @@ pub async fn handle_bun_job( )); } - let mut gbuntar_name = None; + let mut gbuntar_name: Option = None; if has_bundle_cache { + #[cfg(unix)] let target = format!("{job_dir}/main.js"); + #[cfg(unix)] std::os::unix::fs::symlink(&local_path, &target).map_err(|e| { error::Error::ExecutionErr(format!( "could not copy cached binary from {local_path} to {job_dir}/main: {e:?}" )) })?; + + #[cfg(windows)] + let target = format!("{job_dir}\\main.js"); + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&local_path, &target).map_err(|e| { + error::Error::ExecutionErr(format!( + "could not copy cached binary from {local_path} to {job_dir}\\main: {e:?}" + )) + })?; } else if let Some(codebase) = codebase.as_ref() { pull_codebase(&job.workspace_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o.as_ref() { @@ -1295,7 +1324,7 @@ try {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let cmd = if annotation.nodejs_mode { + let mut cmd = if annotation.nodejs_mode { let script_path = format!("{job_dir}/wrapper.mjs"); let mut bun_cmd = Command::new(&*NODE_BIN_PATH); @@ -1336,6 +1365,10 @@ try {{ .stderr(Stdio::piped()); bun_cmd }; + + #[cfg(windows)] + cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process( cmd, if annotation.nodejs_mode { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 202584c653e82..1dbf14e627865 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -221,12 +221,20 @@ func Run(req Req) (interface{{}}, error){{ } } else { let target = format!("{job_dir}/main"); + #[cfg(unix)] std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) })?; + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bin_path, &target).map_err(|e| { + Error::ExecutionErr(format!( + "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + )) + })?; + create_args_and_out_file(client, job, job_dir, db).await?; cache_logs }; diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index cf5c601c90d6d..d933847d4adb9 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -5,6 +5,7 @@ mod mssql_executor; #[cfg(feature = "enterprise")] mod snowflake_executor; +mod ansible_executor; mod bash_executor; mod bun_executor; pub mod common; @@ -25,7 +26,6 @@ mod rust_executor; mod worker; mod worker_flow; mod worker_lockfiles; -mod ansible_executor; pub use worker::*; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 51584eb037d95..a477feeeff20e 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -60,7 +60,7 @@ use crate::{ }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, TZ_ENV, + PIP_INDEX_URL, SYSTEM_ROOT, TZ_ENV, }; pub async fn create_dependencies_dir(job_dir: &str) { @@ -425,6 +425,13 @@ mount {{ ), )?; } else { + #[cfg(windows)] + reserved_variables.insert( + "PYTHONPATH".to_string(), + additional_python_paths_folders.replace(":", ";"), + ); + + #[cfg(unix)] reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders); } @@ -470,6 +477,10 @@ mount {{ .args(vec!["-u", "-m", "wrapper"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(python_cmd, PYTHON_PATH.as_str()).await? }; @@ -1006,7 +1017,12 @@ pub async fn handle_python_reqs( start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string(); + #[cfg(unix)] let req = format!("'{}'", req); + + #[cfg(windows)] + let req = format!("{}", req); + let mut command_args = vec![ PYTHON_PATH.as_str(), "-m", @@ -1062,19 +1078,35 @@ pub async fn handle_python_reqs( tracing::debug!("pip install command: {:?}", command_args); - let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); - flock_cmd - .env_clear() - .envs(envs) - .args([ - "-x", - &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), - "--command", - &command_args.join(" "), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + #[cfg(unix)] + { + let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); + flock_cmd + .env_clear() + .envs(envs) + .args([ + "-x", + &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + "--command", + &command_args.join(" "), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + } + + #[cfg(windows)] + { + let mut pip_cmd = Command::new(PYTHON_PATH.as_str()); + pip_cmd + .env_clear() + .envs(envs) + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .args(&command_args[1..]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(pip_cmd, PYTHON_PATH.as_str()).await? + } }; let child = handle_child( diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 3477eed51bc9a..91088e6889187 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -19,17 +19,30 @@ use crate::{ start_child_process, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - RUST_CACHE_DIR, TZ_ENV, + RUST_CACHE_DIR, SYSTEM_ROOT, TZ_ENV, }; const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); +#[cfg(unix)] lazy_static::lazy_static! { static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string()); static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string()); static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string())); } +#[cfg(windows)] +lazy_static::lazy_static! { + static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); + static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { + format!("{}\\.cargo", *HOME_DIR) + }); + static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { + format!("{}\\.rustup", *HOME_DIR) + }); + static ref CARGO_PATH: String = format!("{}/bin/cargo.exe", *CARGO_HOME); +} + const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; fn gen_cargo_crate(code: &str, job_dir: &str) -> anyhow::Result<()> { @@ -124,6 +137,14 @@ pub async fn generate_cargo_lockfile( .args(vec!["generate-lockfile"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + { + gen_lockfile_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + gen_lockfile_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -169,9 +190,20 @@ pub async fn build_rust_crate( .env("HOME", HOME_ENV.as_str()) .env("CARGO_HOME", CARGO_HOME.as_str()) .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) - .args(vec!["build", "--release"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + build_rust_cmd.args(vec!["build", "--release"]); + + #[cfg(windows)] + { + build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + build_rust_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } + let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -273,12 +305,20 @@ pub async fn handle_rust_job( let cache_logs = if cache { let target = format!("{job_dir}/main"); + #[cfg(unix)] std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) })?; + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bin_path, &target).map_err(|e| { + Error::ExecutionErr(format!( + "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + )) + })?; + create_args_and_out_file(client, job, job_dir, db).await?; cache_logs } else { @@ -353,6 +393,9 @@ pub async fn handle_rust_job( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(run_rust, compiled_executable_name).await? }; handle_child( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 1e5ca26024855..f3ae69d1e6b97 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -31,15 +31,18 @@ use windmill_common::METRICS_ENABLED; use reqwest::Response; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sqlx::{types::Json, Pool, Postgres}; +#[cfg(feature = "benchmark")] +use std::sync::atomic::AtomicUsize; use std::{ - collections::{hash_map::DefaultHasher, HashMap}, fs::DirBuilder, hash::Hash, sync::{ + collections::{hash_map::DefaultHasher, HashMap}, + fs::DirBuilder, + hash::Hash, + sync::{ atomic::{AtomicBool, AtomicU16, Ordering}, Arc, - }, time::Duration + }, + time::Duration, }; -#[cfg(feature = "benchmark")] -use std::sync::atomic::AtomicUsize; - use uuid::Uuid; @@ -55,8 +58,8 @@ use windmill_common::{ }; use windmill_queue::{ - append_logs, canceled_job_to_result, empty_result, pull, push, CanceledBy, - PushArgs, PushIsolationLevel, HTTP_CLIENT, + append_logs, canceled_job_to_result, empty_result, pull, push, CanceledBy, PushArgs, + PushIsolationLevel, HTTP_CLIENT, }; #[cfg(feature = "prometheus")] @@ -81,16 +84,31 @@ use tokio::{ use rand::Rng; - use crate::{ - ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ + ansible_executor::handle_ansible_job, + bash_executor::{handle_bash_job, handle_powershell_job}, + bun_executor::handle_bun_job, + common::{ build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, NO_LOGS_AT_ALL, SLOW_LOGS, - }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_job_error, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{ + }, + deno_executor::handle_deno_job, + go_executor::handle_go_job, + graphql_executor::do_graphql, + handle_job_error, + js_eval::{eval_fetch_timeout, transpile_ts}, + mysql_executor::do_mysql, + pg_executor::do_postgresql, + php_executor::handle_php_job, + python_executor::handle_python_job, + result_processor::{handle_receive_completed_job, process_result}, + rust_executor::handle_rust_job, + worker_flow::{ handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, Step, - }, worker_lockfiles::{ + }, + worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, - } + }, }; #[cfg(feature = "enterprise")] @@ -328,6 +346,8 @@ lazy_static::lazy_static! { pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); + #[cfg(windows)] + pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() @@ -573,11 +593,9 @@ type GGauge = (); #[derive(Clone)] pub struct JobCompletedSender(Sender, Option, Option); - #[derive(Clone)] pub struct SameWorkerSender(pub Sender, pub Arc); - pub struct SameWorkerPayload { pub job_id: Uuid, pub recoverable: bool, @@ -611,7 +629,6 @@ impl SameWorkerSender { } } - // on linux, we drop caches every DROP_CACHE_PERIOD to avoid OOM killer believing we are using too much memory just because we create lots of files when executing jobs #[cfg(any(target_os = "linux"))] pub async fn drop_cache() { @@ -1429,7 +1446,11 @@ pub async fn run_worker { - - last_executed_job = None; jobs_executed += 1; tracing::debug!("started handling of job {}", job.id); - if matches!(job.job_kind, JobKind::Script | JobKind::Preview) { - if !dedicated_workers.is_empty() { let key_o = if is_flow_worker { job.flow_step_id.as_ref().map(|x| x.to_string()) @@ -1653,7 +1673,7 @@ pub async fn run_worker( Ok(()) } - pub fn build_envs( envs: Option>, ) -> windmill_common::error::Result> { @@ -2673,7 +2692,11 @@ async fn handle_code_execution_job( ); let shared_mount = if job.same_worker && job.language != Some(ScriptLang::Deno) { - let folder = if job.language == Some(ScriptLang::Go) { "/go" } else { "" }; + let folder = if job.language == Some(ScriptLang::Go) { + "/go" + } else { + "" + }; format!( r#" mount {{ @@ -2837,7 +2860,6 @@ mount {{ .await } Some(ScriptLang::Ansible) => { - handle_ansible_job( requirements_o, job_dir, @@ -2851,7 +2873,8 @@ mount {{ &inner_content, base_internal_url, envs, - ).await + ) + .await } _ => panic!("unreachable, language is not supported: {language:#?}"), }; From 3ed1d5f93c3a53ab741edd8c8c701b487663b351 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Wed, 25 Sep 2024 16:24:13 -0400 Subject: [PATCH 02/19] adding support for powershell --- backend/windmill-worker/src/bash_executor.rs | 75 +++++++++++++++++++- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 25d55af57a4e8..a1b677b6cc733 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -28,7 +28,7 @@ use crate::{ start_child_process, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, + POWERSHELL_CACHE_DIR, POWERSHELL_PATH, SYSTEM_ROOT, TZ_ENV, }; lazy_static::lazy_static! { @@ -195,7 +195,7 @@ pub async fn handle_powershell_job( worker_name: &str, envs: HashMap, ) -> Result, Error> { - let pwsh_args = { + let mut pwsh_args = { let args = build_args_map(job, client, db).await?.map(Json); let job_args = if args.is_some() { args.as_ref() @@ -222,13 +222,19 @@ pub async fn handle_powershell_job( .collect::>() }; + #[cfg(windows)] + let split_char = '\\'; + + #[cfg(unix)] + let split_char = '/'; + let installed_modules = fs::read_dir(POWERSHELL_CACHE_DIR)? .filter_map(|x| { x.ok().map(|x| { x.path() .display() .to_string() - .split('/') + .split(split_char) .last() .unwrap_or_default() .to_lowercase() @@ -284,6 +290,7 @@ pub async fn handle_powershell_job( append_logs(&job.id, &job.workspace_id, logs2, db).await; // make sure default (only allhostsallusers) modules are loaded, disable autoload (cache can be large to explore especially on cloud) and add /tmp/windmill/cache to PSModulePath + #[cfg(unix)] let profile = format!( "$PSModuleAutoloadingPreference = 'None' $PSModulePathBackup = $env:PSModulePath @@ -292,6 +299,17 @@ Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{}:$PSModulePathBackup\"", POWERSHELL_CACHE_DIR ); + + #[cfg(windows)] + let profile = format!( + "#$PSModuleAutoloadingPreference = 'None' +$PSModulePathBackup = $env:PSModulePath +#$env:PSModulePath = \"C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\Modules;C:\\Program Files\\PowerShell\\7\\Modules\" +Get-Module -ListAvailable | Import-Module +$env:PSModulePath = \"{};$PSModulePathBackup\"", + POWERSHELL_CACHE_DIR + ); + // make sure param() is first let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); let content: String = if let Some(param_match) = param_match { @@ -307,11 +325,29 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", }; write_file(job_dir, "main.ps1", content.as_str())?; + + #[cfg(unix)] write_file( job_dir, "wrapper.sh", &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n{} -F ./main.ps1 \"$@\" 2>&1 | tee bp\nwait $!", POWERSHELL_PATH.as_str()), )?; + + #[cfg(windows)] + write_file( + job_dir, + "wrapper.ps1", + &format!( + " param([string[]]$args)\n\ + $ErrorActionPreference = 'Stop'\n\ + $pipe = New-TemporaryFile\n\ + & \"{}\" -File ./main.ps1 @args 2>&1 | Tee-Object -FilePath $pipe\n\ + Get-Content -Path $pipe | Select-Object -Last 1 | Set-Content -Path './result2.out'\n\ + Remove-Item $pipe\n", + POWERSHELL_PATH.as_str() + ), + )?; + let token = client.get_token().await; let mut reserved_variables = get_reserved_variables(job, &token, db).await?; reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); @@ -320,6 +356,7 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", let _ = write_file(job_dir, "result.out", "")?; let _ = write_file(job_dir, "result2.out", "")?; + #[cfg(unix)] let child = if !*DISABLE_NSJAIL { let _ = write_file( job_dir, @@ -366,6 +403,38 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", .stderr(Stdio::piped()) .spawn()? }; + + #[cfg(windows)] + pwsh_args.insert(0, r".\wrapper.ps1".to_string()); + + #[cfg(windows)] + let child = Command::new(POWERSHELL_PATH.as_str()) + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "PATHEXT", + ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL", + ) + .args( + pwsh_args + .into_iter() + .map(|arg| arg.replace("--", "-")) + .collect::>(), + ) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + handle_child( &job.id, db, From 6551494a680317cbd19e6b6f074c539a4dcb89d2 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Wed, 25 Sep 2024 16:56:33 -0400 Subject: [PATCH 03/19] compiling error on unix --- backend/windmill-worker/src/worker.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index f3ae69d1e6b97..322d0b41d3956 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -346,7 +346,6 @@ lazy_static::lazy_static! { pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); - #[cfg(windows)] pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") From 3c3fdbe2f88844a1e00de88f040ee42d85374a82 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 11:53:54 -0400 Subject: [PATCH 04/19] rust linting comments --- backend/windmill-worker/src/bash_executor.rs | 142 +++++++++--------- backend/windmill-worker/src/bun_executor.rs | 18 ++- .../windmill-worker/src/python_executor.rs | 5 +- backend/windmill-worker/src/rust_executor.rs | 5 +- backend/windmill-worker/src/worker.rs | 8 +- 5 files changed, 100 insertions(+), 78 deletions(-) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 196dc19d0d5b3..ffa1c8f867d88 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -28,9 +28,12 @@ use crate::{ start_child_process, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - POWERSHELL_CACHE_DIR, POWERSHELL_PATH, SYSTEM_ROOT, TZ_ENV, + POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + lazy_static::lazy_static! { pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap(); @@ -195,7 +198,7 @@ pub async fn handle_powershell_job( worker_name: &str, envs: HashMap, ) -> Result, Error> { - let mut pwsh_args = { + let pwsh_args = { let args = build_args_map(job, client, db).await?.map(Json); let job_args = if args.is_some() { args.as_ref() @@ -356,84 +359,85 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", let _ = write_file(job_dir, "result.out", "")?; let _ = write_file(job_dir, "result2.out", "")?; + let child; + #[cfg(unix)] - let child = if !*DISABLE_NSJAIL { - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{SHARED_MOUNT}", shared_mount) - .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), - )?; - let mut cmd_args = vec![ - "--config", - "run.config.proto", - "--", - BIN_BASH.as_str(), - "wrapper.sh", - ]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - Command::new(NSJAIL_PATH.as_str()) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("TZ", TZ_ENV.as_str()) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(cmd_args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - let mut cmd_args = vec!["wrapper.sh"]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - Command::new(BIN_BASH.as_str()) + { + child = if !*DISABLE_NSJAIL { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), + )?; + let mut cmd_args = vec![ + "--config", + "run.config.proto", + "--", + BIN_BASH.as_str(), + "wrapper.sh", + ]; + cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); + Command::new(NSJAIL_PATH.as_str()) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("TZ", TZ_ENV.as_str()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(cmd_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + let mut cmd_args = vec!["wrapper.sh"]; + cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); + Command::new(BIN_BASH.as_str()) + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("TZ", TZ_ENV.as_str()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .args(cmd_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + } + + #[cfg(windows)] + { + let mut cmd_args = vec![r".\wrapper.ps1".to_string()]; + cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-"))); + + child = Command::new(POWERSHELL_PATH.as_str()) .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) - .env("TZ", TZ_ENV.as_str()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "PATHEXT", + ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL", + ) .args(cmd_args) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .spawn()? - }; - - #[cfg(windows)] - pwsh_args.insert(0, r".\wrapper.ps1".to_string()); - - #[cfg(windows)] - let child = Command::new(POWERSHELL_PATH.as_str()) - .current_dir(job_dir) - .env_clear() - .envs(envs) - .envs(reserved_variables) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .env("SystemRoot", SYSTEM_ROOT.as_str()) - .env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), - ) - .env( - "PATHEXT", - ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL", - ) - .args( - pwsh_args - .into_iter() - .map(|arg| arg.replace("--", "-")) - .collect::>(), - ) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .spawn()?; + } handle_child( &job.id, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 54f5091e9ef6d..cba2087d104fc 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -19,9 +19,12 @@ use crate::{ }, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, - NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, SYSTEM_ROOT, TZ_ENV, + NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + use tokio::{fs::File, process::Command}; use tokio::io::AsyncReadExt; @@ -1325,7 +1328,7 @@ try {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let mut cmd = if annotation.nodejs_mode { + let cmd = if annotation.nodejs_mode { let script_path = format!("{job_dir}/wrapper.mjs"); let mut bun_cmd = Command::new(&*NODE_BIN_PATH); @@ -1338,6 +1341,10 @@ try {{ .args(vec!["--preserve-symlinks", &script_path]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd } else { let script_path = format!("{job_dir}/wrapper.mjs"); @@ -1364,12 +1371,13 @@ try {{ .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd }; - #[cfg(windows)] - cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); - start_child_process( cmd, if annotation.nodejs_mode { diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index a477feeeff20e..224e7443cdb69 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -60,9 +60,12 @@ use crate::{ }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, SYSTEM_ROOT, TZ_ENV, + PIP_INDEX_URL, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + pub async fn create_dependencies_dir(job_dir: &str) { DirBuilder::new() .recursive(true) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 91088e6889187..103cb9cac027e 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -19,9 +19,12 @@ use crate::{ start_child_process, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - RUST_CACHE_DIR, SYSTEM_ROOT, TZ_ENV, + RUST_CACHE_DIR, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); #[cfg(unix)] diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 39fd8625fa5fd..93da00dd83949 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -347,8 +347,6 @@ lazy_static::lazy_static! { pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); - pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); - static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() .and_then(|x| x.parse::().ok()) @@ -389,6 +387,12 @@ lazy_static::lazy_static! { } + +#[cfg(windows)] +lazy_static::lazy_static! { + pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); +} + //only matter if CLOUD_HOSTED pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB From 674a11dbf0176749020a946a3894d3fccfed5580 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 12:04:37 -0400 Subject: [PATCH 05/19] comments hugo: PSModulePath --- backend/windmill-worker/src/bash_executor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index ffa1c8f867d88..29531cf9d5696 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -305,9 +305,9 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", #[cfg(windows)] let profile = format!( - "#$PSModuleAutoloadingPreference = 'None' + "$PSModuleAutoloadingPreference = 'None' $PSModulePathBackup = $env:PSModulePath -#$env:PSModulePath = \"C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\Modules;C:\\Program Files\\PowerShell\\7\\Modules\" +$env:PSModulePath = \"C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\Modules;C:\\Program Files\\PowerShell\\7\\Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{};$PSModulePathBackup\"", POWERSHELL_CACHE_DIR From fb8132158fad1e41e7ef818a695b62b178f7aa4d Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 14:36:01 -0400 Subject: [PATCH 06/19] comments ruben, refactor to simplify --- backend/windmill-worker/src/bash_executor.rs | 133 +++++++++--------- backend/windmill-worker/src/bun_executor.rs | 26 ++-- backend/windmill-worker/src/go_executor.rs | 11 +- .../windmill-worker/src/python_executor.rs | 10 +- backend/windmill-worker/src/rust_executor.rs | 41 +++--- 5 files changed, 106 insertions(+), 115 deletions(-) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 6e4c0d2ac5650..ce191230b63fb 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -364,85 +364,82 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", let _ = write_file(job_dir, "result.out", "")?; let _ = write_file(job_dir, "result2.out", "")?; - let child; + let child = if !*DISABLE_NSJAIL { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), + )?; + let mut cmd_args = vec![ + "--config", + "run.config.proto", + "--", + BIN_BASH.as_str(), + "wrapper.sh", + ]; + cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); + Command::new(NSJAIL_PATH.as_str()) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("TZ", TZ_ENV.as_str()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(cmd_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + let mut cmd; + let mut cmd_args; - #[cfg(unix)] - { - child = if !*DISABLE_NSJAIL { - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) - .replace("{SHARED_MOUNT}", shared_mount) - .replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR), - )?; - let mut cmd_args = vec![ - "--config", - "run.config.proto", - "--", - BIN_BASH.as_str(), - "wrapper.sh", - ]; + #[cfg(unix)] + { + cmd_args = vec!["wrapper.sh"]; cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - Command::new(NSJAIL_PATH.as_str()) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("TZ", TZ_ENV.as_str()) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(cmd_args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - let mut cmd_args = vec!["wrapper.sh"]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - Command::new(BIN_BASH.as_str()) - .current_dir(job_dir) - .env_clear() - .envs(envs) - .envs(reserved_variables) - .env("TZ", TZ_ENV.as_str()) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; - } + cmd = Command::new(BIN_BASH.as_str()); + } - #[cfg(windows)] - { - let mut cmd_args = vec![r".\wrapper.ps1".to_string()]; - cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-"))); + #[cfg(windows)] + { + cmd_args = vec![r".\wrapper.ps1".to_string()]; + cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-"))); + cmd = Command::new(POWERSHELL_PATH.as_str()); + } - child = Command::new(POWERSHELL_PATH.as_str()) - .current_dir(job_dir) + cmd.current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) + .env("TZ", TZ_ENV.as_str()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .env("SystemRoot", SYSTEM_ROOT.as_str()) - .env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), - ) - .env( - "PATHEXT", - ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL", - ) - .args(cmd_args) + .args(&cmd_args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - } + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "PATHEXT", + std::env::var("PATHEXT").unwrap_or_else(|_| { + String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL") + }), + ); + } + + cmd.spawn()? + }; handle_child( &job.id, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7251a93bdec4a..d50f63384dc57 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -838,21 +838,23 @@ pub async fn handle_bun_job( let mut gbuntar_name: Option = None; if has_bundle_cache { - #[cfg(unix)] - let target = format!("{job_dir}/main.js"); - #[cfg(unix)] - std::os::unix::fs::symlink(&local_path, &target).map_err(|e| { - error::Error::ExecutionErr(format!( - "could not copy cached binary from {local_path} to {job_dir}/main: {e:?}" - )) - })?; + let target; + let symlink; + #[cfg(unix)] + { + target = format!("{job_dir}/main.js"); + symlink = std::os::unix::fs::symlink(&local_path, &target); + } #[cfg(windows)] - let target = format!("{job_dir}\\main.js"); - #[cfg(windows)] - std::os::windows::fs::symlink_dir(&local_path, &target).map_err(|e| { + { + target = format!("{job_dir}\\main.js"); + symlink = std::os::windows::fs::symlink_dir(&local_path, &target); + } + + symlink.map_err(|e| { error::Error::ExecutionErr(format!( - "could not copy cached binary from {local_path} to {job_dir}\\main: {e:?}" + "could not copy cached binary from {local_path} to {job_dir}/main: {e:?}" )) })?; } else if let Some(codebase) = codebase.as_ref() { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index ef43ba4c5a6ca..c171e52156ff3 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -226,14 +226,11 @@ func Run(req Req) (interface{{}}, error){{ } else { let target = format!("{job_dir}/main"); #[cfg(unix)] - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { - Error::ExecutionErr(format!( - "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" - )) - })?; - + let symlink = std::os::unix::fs::symlink(&bin_path, &target); #[cfg(windows)] - std::os::windows::fs::symlink_dir(&bin_path, &target).map_err(|e| { + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 64199abf2f84c..a5717ed12bb08 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -402,6 +402,9 @@ except BaseException as e: let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?; let additional_python_paths_folders = additional_python_paths.iter().join(":"); + #[cfg(windows)] + let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); + if !*DISABLE_NSJAIL { let shared_deps = additional_python_paths .into_iter() @@ -433,13 +436,6 @@ mount {{ ), )?; } else { - #[cfg(windows)] - reserved_variables.insert( - "PYTHONPATH".to_string(), - additional_python_paths_folders.replace(":", ";"), - ); - - #[cfg(unix)] reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders); } diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index e06f0d78f64c1..0ab4a44e3ba89 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -28,23 +28,25 @@ use crate::SYSTEM_ROOT; const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); -#[cfg(unix)] lazy_static::lazy_static! { - static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string()); - static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string()); - static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string())); + static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); + static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); + static ref RUSTUP_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); + static ref CARGO_PATH: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_PATH_DEFAULT.clone() }); } #[cfg(windows)] lazy_static::lazy_static! { - static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); - static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { - format!("{}\\.cargo", *HOME_DIR) - }); - static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { - format!("{}\\.rustup", *HOME_DIR) - }); - static ref CARGO_PATH: String = format!("{}/bin/cargo.exe", *CARGO_HOME); + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); + static ref CARGO_PATH_DEFAULT: String = format!("{}/bin/cargo.exe", *CARGO_HOME); +} + +#[cfg(unix)] +lazy_static::lazy_static! { + static ref CARGO_HOME_DEFAULT: String = "usr/local/cargo".to_string(); + static ref RUSTUP_HOME_DEFAULT: String = "/usr/local/rustup".to_string(); + static ref CARGO_PATH_DEFAULT: String = "/usr/local/cargo/bin/cargo".to_string(); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -197,11 +199,10 @@ pub async fn build_rust_crate( .env("HOME", HOME_ENV.as_str()) .env("CARGO_HOME", CARGO_HOME.as_str()) .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .args(vec!["build", "--release"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - build_rust_cmd.args(vec!["build", "--release"]); - #[cfg(windows)] { build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); @@ -314,15 +315,13 @@ pub async fn handle_rust_job( let cache_logs = if cache { let target = format!("{job_dir}/main"); - #[cfg(unix)] - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { - Error::ExecutionErr(format!( - "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" - )) - })?; + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); #[cfg(windows)] - std::os::windows::fs::symlink_dir(&bin_path, &target).map_err(|e| { + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) From ceb221a29058a396975b4c8b67dad8e3f93c444e Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 15:03:16 -0400 Subject: [PATCH 07/19] adding build workflow --- .github/workflows/build_windows_worker.yml | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/build_windows_worker.yml diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml new file mode 100644 index 0000000000000..b24f87f669797 --- /dev/null +++ b/.github/workflows/build_windows_worker.yml @@ -0,0 +1,48 @@ +name: Backend only integration tests + +on: + push: + branches: + - "alp/build_windows" + paths: + - "backend/**" + - ".github/workflows/build_windows_worker.yml" + pull_request: + types: [opened, synchronize, reopened] + paths: + - "backend/**" + - ".github/workflows/backend-test.yml" + +jobs: + cargo_test: + runs-on: ubicloud-standard-8 + container: + image: ghcr.io/windmill-labs/backend-tests + services: + postgres: + image: postgres + env: + POSTGRES_DB: windmill + POSTGRES_PASSWORD: changeme + + options: >- + --health-cmd pg_isready --health-interval 10s --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: 1.80.0 + # - uses: Swatinem/rust-cache@v2 + # with: + # workspaces: | + # backend + # backend -> target + - name: cargo build windows + timeout-minutes: 15 + run: + mkdir frontend/build && cd backend && touch + windmill-api/openapi-deref.yaml && rustup target add x86_64-pc-windows-gnu && + DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill + DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release + --all -- --nocapture From b6f8d462ade892bc361939f3e2fa0f993f4b6854 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 15:08:17 -0400 Subject: [PATCH 08/19] editing workflow --- .github/workflows/build_windows_worker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index b24f87f669797..90bfa1d72db56 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -1,4 +1,4 @@ -name: Backend only integration tests +name: Build Windows Worker on: push: @@ -44,5 +44,5 @@ jobs: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && rustup target add x86_64-pc-windows-gnu && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill - DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release - --all -- --nocapture + DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release + From 3b02fbb38e48ca9137bcf6436ac475d9c5b3c03f Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 15:12:59 -0400 Subject: [PATCH 09/19] editing workflow --- .github/workflows/build_windows_worker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 90bfa1d72db56..da7cec7e14c92 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -42,7 +42,7 @@ jobs: timeout-minutes: 15 run: mkdir frontend/build && cd backend && touch - windmill-api/openapi-deref.yaml && rustup target add x86_64-pc-windows-gnu && + windmill-api/openapi-deref.yaml && apt-get install gcc-mingw-w64 && rustup target add x86_64-pc-windows-gnu && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release - + From b3dad9e93e25be36d5c9de83930beb3e4d2acec0 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 15:17:23 -0400 Subject: [PATCH 10/19] editing workflow --- .github/workflows/build_windows_worker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index da7cec7e14c92..48e296280c19e 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -14,7 +14,7 @@ on: - ".github/workflows/backend-test.yml" jobs: - cargo_test: + cargo_build_windos: runs-on: ubicloud-standard-8 container: image: ghcr.io/windmill-labs/backend-tests @@ -42,7 +42,7 @@ jobs: timeout-minutes: 15 run: mkdir frontend/build && cd backend && touch - windmill-api/openapi-deref.yaml && apt-get install gcc-mingw-w64 && rustup target add x86_64-pc-windows-gnu && + windmill-api/openapi-deref.yaml && apt-get install gcc-mingw-w64 --yes && rustup target add x86_64-pc-windows-gnu && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release From 9b84cb990f3026800b462e77135c301b954a9587 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 15:23:42 -0400 Subject: [PATCH 11/19] editing workflow --- .github/workflows/build_windows_worker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 48e296280c19e..64efd2d01597c 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -14,7 +14,7 @@ on: - ".github/workflows/backend-test.yml" jobs: - cargo_build_windos: + cargo_build_windows: runs-on: ubicloud-standard-8 container: image: ghcr.io/windmill-labs/backend-tests @@ -42,7 +42,7 @@ jobs: timeout-minutes: 15 run: mkdir frontend/build && cd backend && touch - windmill-api/openapi-deref.yaml && apt-get install gcc-mingw-w64 --yes && rustup target add x86_64-pc-windows-gnu && + windmill-api/openapi-deref.yaml && apt-get install gcc-mingw-w64 python-is-python3 --yes && rustup target add x86_64-pc-windows-gnu && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release From 11e8ca70fb066082d29bd2c8e3eb0bb9e8060429 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 15:55:10 -0400 Subject: [PATCH 12/19] editing workflow --- .github/workflows/build_windows_worker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 64efd2d01597c..265c2b33ff7e4 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -43,6 +43,6 @@ jobs: run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && apt-get install gcc-mingw-w64 python-is-python3 --yes && rustup target add x86_64-pc-windows-gnu && - DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill + DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill OPENSSL_DIR="/usr/lib/ssl" DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release From 951bd8efe60b66cb827949bacc6481cb7f7f8900 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 26 Sep 2024 20:08:58 -0400 Subject: [PATCH 13/19] skip migration env, ee fixes --- backend/src/main.rs | 12 ++++++++++-- backend/windmill-common/src/worker.rs | 3 +++ backend/windmill-worker/src/bun_executor.rs | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index aa6993cf92d57..a5f0145bee47c 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -373,8 +373,16 @@ async fn windmill_main() -> anyhow::Result<()> { let is_agent = mode == Mode::Agent; if !is_agent { - // migration code to avoid break - windmill_api::migrate_db(&db).await?; + let skip_migration = std::env::var("SKIP_MIGRATION") + .map(|val| val == "true") + .unwrap_or(false); + + if !skip_migration { + // migration code to avoid break + windmill_api::migrate_db(&db).await?; + } else { + tracing::info!("SKIP_MIGRATION set, skipping db migration...") + } } let (killpill_tx, mut killpill_rx) = tokio::sync::broadcast::channel::<()>(2); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 9464eccdcc466..385fac616313c 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -449,10 +449,13 @@ pub async fn save_cache( fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { use std::fs::{File, Permissions}; use std::io::Write; + + #[cfg(unix)] use std::os::unix::fs::PermissionsExt; let mut file = File::create(main_path)?; file.write_all(byts)?; + #[cfg(unix)] file.set_permissions(Permissions::from_mode(0o755))?; file.flush()?; Ok(()) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index d50f63384dc57..fc9ab61952a8b 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -557,6 +557,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(fs::read(bun_cache_path)?.into(), job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(&bun_cache_path, dst).await?; } } else if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS @@ -570,6 +571,7 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(bytes, job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(bun_cache_path, dst).await?; } From 5102e0a00adde3057e2200d522776100f14cd5a8 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 27 Sep 2024 12:31:02 -0400 Subject: [PATCH 14/19] improvements powershell --- backend/windmill-worker/src/bash_executor.rs | 27 +++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index ce191230b63fb..271cac5b9efcb 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -312,7 +312,7 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", let profile = format!( "$PSModuleAutoloadingPreference = 'None' $PSModulePathBackup = $env:PSModulePath -$env:PSModulePath = \"C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\Modules;C:\\Program Files\\PowerShell\\7\\Modules\" +$env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{};$PSModulePathBackup\"", POWERSHELL_CACHE_DIR @@ -426,6 +426,31 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", #[cfg(windows)] { cmd.env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "LOCALAPPDATA", + std::env::var("LOCALAPPDATA") + .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), + ) + .env( + "ProgramData", + std::env::var("ProgramData") + .unwrap_or_else(|_| String::from("C:\\ProgramData")), + ) + .env( + "ProgramFiles", + std::env::var("ProgramFiles") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "ProgramFiles(x86)", + std::env::var("ProgramFiles(x86)") + .unwrap_or_else(|_| String::from("C:\\Program Files (x86)")), + ) + .env( + "ProgramW6432", + std::env::var("ProgramW6432") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) .env( "TMP", std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), From 01d7c27e6ebd9dd26c1fda2e8e7e2239a40c3ee8 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 27 Sep 2024 15:28:04 -0400 Subject: [PATCH 15/19] testing windows runner --- .github/workflows/build_windows_worker.yml | 36 ++++++++++------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 265c2b33ff7e4..856e6d954506c 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -7,15 +7,22 @@ on: paths: - "backend/**" - ".github/workflows/build_windows_worker.yml" - pull_request: - types: [opened, synchronize, reopened] - paths: - - "backend/**" - - ".github/workflows/backend-test.yml" + # pull_request: + # types: [opened, synchronize, reopened] + # paths: + # - "backend/**" + # - ".github/workflows/backend-test.yml" + +env: + CARGO_INCREMENTAL: 0 + DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill + DISABLE_EMBEDDING: true + RUST_LOG: info + jobs: cargo_build_windows: - runs-on: ubicloud-standard-8 + runs-on: windows-latest container: image: ghcr.io/windmill-labs/backend-tests services: @@ -30,19 +37,10 @@ jobs: --health-retries 5 steps: - uses: actions/checkout@v4 - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - toolchain: 1.80.0 - # - uses: Swatinem/rust-cache@v2 - # with: - # workspaces: | - # backend - # backend -> target - name: cargo build windows timeout-minutes: 15 - run: - mkdir frontend/build && cd backend && touch - windmill-api/openapi-deref.yaml && apt-get install gcc-mingw-w64 python-is-python3 --yes && rustup target add x86_64-pc-windows-gnu && - DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill OPENSSL_DIR="/usr/lib/ssl" - DISABLE_EMBEDDING=true RUST_LOG=info cargo build --target x86_64-pc-windows-gnu --release + run: cargo build --release + # mkdir frontend/build && cd backend && touch + # windmill-api/openapi-deref.yaml && + From e49ba39d97199cbd8aef6950a3e504c49d01c72d Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 27 Sep 2024 15:31:53 -0400 Subject: [PATCH 16/19] testing windows runner --- .github/workflows/build_windows_worker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 856e6d954506c..c5edcc2891f41 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -23,8 +23,6 @@ env: jobs: cargo_build_windows: runs-on: windows-latest - container: - image: ghcr.io/windmill-labs/backend-tests services: postgres: image: postgres From d5f3e75e60c31609b5da75bde0976618264b11cb Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 27 Sep 2024 16:08:16 -0400 Subject: [PATCH 17/19] testing windows runner --- .github/workflows/build_windows_worker.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index c5edcc2891f41..775c2bb14e98e 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -23,16 +23,6 @@ env: jobs: cargo_build_windows: runs-on: windows-latest - services: - postgres: - image: postgres - env: - POSTGRES_DB: windmill - POSTGRES_PASSWORD: changeme - - options: >- - --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@v4 - name: cargo build windows From d1b5abdd9e2fe900de9dba66a9099691fbb26017 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 27 Sep 2024 16:14:00 -0400 Subject: [PATCH 18/19] testing windows runner --- .github/workflows/build_windows_worker.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 775c2bb14e98e..1527a05409a59 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -27,8 +27,5 @@ jobs: - uses: actions/checkout@v4 - name: cargo build windows timeout-minutes: 15 - run: cargo build --release - # mkdir frontend/build && cd backend && touch - # windmill-api/openapi-deref.yaml && - + run: mkdir frontend/build && cd backend && type NUL > windmill-api/openapi-deref.yaml && cargo build --release From 2f80ba1db052f4e0b36158c3ef07fdb59b5942e2 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 27 Sep 2024 16:18:42 -0400 Subject: [PATCH 19/19] testing windows runner --- .github/workflows/build_windows_worker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 1527a05409a59..8a943ce9940ed 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -27,5 +27,5 @@ jobs: - uses: actions/checkout@v4 - name: cargo build windows timeout-minutes: 15 - run: mkdir frontend/build && cd backend && type NUL > windmill-api/openapi-deref.yaml && cargo build --release + run: mkdir frontend/build && cd backend && New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force && cargo build --release