diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml new file mode 100644 index 0000000000000..83e0855a5079d --- /dev/null +++ b/.github/workflows/build_windows_worker.yml @@ -0,0 +1,56 @@ +name: Build and Publish Windows Worker + +on: + push: + tags: + - "v*" + +env: + CARGO_INCREMENTAL: 0 + SQLX_OFFLINE: true + DISABLE_EMBEDDING: true + RUST_LOG: info + +jobs: + cargo_build_windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Read EE repo commit hash + shell: pwsh + run: | + $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt + echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Checkout windmill-ee-private repository + uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + shell: bash + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Cargo build windows + timeout-minutes: 60 + run: | + $env:OPENSSL_DIR = "${Env:ProgramFiles}\OpenSSL" + mkdir frontend/build && cd backend + New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy + + - name: Rename binary with corresponding architecture + run: | + ren ./backend/target/release/windmill.exe ./backend/target/release/windmill-ee.exe + + - name: Attach binary to release + uses: softprops/action-gh-release@v2 + with: + files: | + ./backend/target/release/windmill-ee.exe diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 151d6eb562c01..f008f2f34a32b 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -399,15 +399,11 @@ fn parse_ansible_options(opts: &Vec) -> AnsiblePlaybookOptions { if c > 0 && c <= 6 { ret.verbosity = Some("v".repeat(c.min(6))); } - } } - _ => () - + _ => (), } } - - } } @@ -422,10 +418,10 @@ fn count_consecutive_vs(s: &str) -> usize { if c == 'v' { current_count += 1; if current_count == 6 { - return 6; // Stop early if we reach 6 + return 6; // Stop early if we reach 6 } } else { - current_count = 0; // Reset count if the character is not 'v' + current_count = 0; // Reset count if the character is not 'v' } max_count = max_count.max(current_count); } diff --git a/backend/src/main.rs b/backend/src/main.rs index 0a59e4af07033..74b1269f7cf01 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -373,8 +373,16 @@ async fn windmill_main() -> anyhow::Result<()> { let is_agent = mode == Mode::Agent; if !is_agent { - // migration code to avoid break - windmill_api::migrate_db(&db).await?; + let skip_migration = std::env::var("SKIP_MIGRATION") + .map(|val| val == "true") + .unwrap_or(false); + + if !skip_migration { + // migration code to avoid break + windmill_api::migrate_db(&db).await?; + } else { + tracing::info!("SKIP_MIGRATION set, skipping db migration...") + } } let (killpill_tx, mut killpill_rx) = tokio::sync::broadcast::channel::<()>(2); diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 156be41b26962..a431e4a2f54e1 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -11,7 +11,6 @@ use axum::http::HeaderValue; use quick_cache::sync::Cache; use serde_json::value::RawValue; use sqlx::Pool; -use windmill_common::error::JsonResult; use std::collections::HashMap; #[cfg(feature = "prometheus")] use std::sync::atomic::Ordering; @@ -19,6 +18,7 @@ use tokio::io::AsyncReadExt; #[cfg(feature = "prometheus")] use tokio::time::Instant; use tower::ServiceBuilder; +use windmill_common::error::JsonResult; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ format_completed_job_result, format_result, CompletedJobWithFormattedResult, FormattedResult, @@ -293,11 +293,8 @@ pub fn workspace_unauthed_service() -> Router { pub fn global_root_service() -> Router { Router::new() - .route("/db_clock", get(get_db_clock)) - .route( - "/completed/count_by_tag", - get(count_by_tag), - ) + .route("/db_clock", get(get_db_clock)) + .route("/completed/count_by_tag", get(count_by_tag)) } #[derive(Deserialize)] @@ -4683,8 +4680,8 @@ async fn get_job_update( .fetch_optional(&db) .await?; - let progress: Option = if get_progress == Some(true){ - sqlx::query_scalar!( + let progress: Option = if get_progress == Some(true) { + sqlx::query_scalar!( "SELECT scalar_int FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &w_id, job_id, @@ -5115,8 +5112,6 @@ async fn get_completed_job_result( Ok(Json(result).into_response()) } - - #[derive(Deserialize)] struct CountByTagQuery { horizon_secs: Option, @@ -5130,7 +5125,7 @@ struct TagCount { } async fn count_by_tag( - ApiAuthed { email, ..}: ApiAuthed, + ApiAuthed { email, .. }: ApiAuthed, Extension(db): Extension, Query(query): Query, ) -> JsonResult> { diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 1fb5b122592e7..bb881d5369b65 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -58,7 +58,10 @@ pub fn workspaced_service() -> Router { .route("/type/exists/:name", get(exists_resource_type)) .route("/type/update/:name", post(update_resource_type)) .route("/type/delete/:name", delete(delete_resource_type)) - .route("/file_resource_type_to_file_ext_map", get(file_resource_ext_to_resource_type)) + .route( + "/file_resource_type_to_file_ext_map", + get(file_resource_ext_to_resource_type), + ) .route("/type/create", post(create_resource_type)) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 9464eccdcc466..385fac616313c 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -449,10 +449,13 @@ pub async fn save_cache( fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { use std::fs::{File, Permissions}; use std::io::Write; + + #[cfg(unix)] use std::os::unix::fs::PermissionsExt; let mut file = File::create(main_path)?; file.write_all(byts)?; + #[cfg(unix)] file.set_permissions(Permissions::from_mode(0o755))?; file.flush()?; Ok(()) diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs index 79bcbcff778a6..da92ff0ef82fd 100644 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ b/backend/windmill-indexer/src/indexer_ee.rs @@ -1,6 +1,6 @@ +use anyhow::anyhow; use sqlx::{Pool, Postgres}; use windmill_common::error::Error; -use anyhow::anyhow; #[derive(Clone)] pub struct IndexReader; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 6639189f6147c..7df394a77c59b 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -1,3 +1,4 @@ +#[cfg(unix)] use std::{ collections::HashMap, os::unix::fs::PermissionsExt, @@ -5,6 +6,13 @@ use std::{ process::Stdio, }; +#[cfg(windows)] +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + process::Stdio, +}; + use anyhow::anyhow; use itertools::Itertools; use serde_json::value::RawValue; @@ -378,6 +386,7 @@ fi let file = write_file(job_dir, "wrapper.sh", &wrapper)?; + #[cfg(unix)] file.metadata()?.permissions().set_mode(0o777); // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index e9b135d27715e..1b1fc65f17ec0 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -32,6 +32,9 @@ use crate::{ POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + lazy_static::lazy_static! { pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap(); @@ -226,13 +229,19 @@ pub async fn handle_powershell_job( .collect::>() }; + #[cfg(windows)] + let split_char = '\\'; + + #[cfg(unix)] + let split_char = '/'; + let installed_modules = fs::read_dir(POWERSHELL_CACHE_DIR)? .filter_map(|x| { x.ok().map(|x| { x.path() .display() .to_string() - .split('/') + .split(split_char) .last() .unwrap_or_default() .to_lowercase() @@ -289,14 +298,26 @@ pub async fn handle_powershell_job( append_logs(&job.id, &job.workspace_id, logs2, db).await; // make sure default (only allhostsallusers) modules are loaded, disable autoload (cache can be large to explore especially on cloud) and add /tmp/windmill/cache to PSModulePath + #[cfg(unix)] let profile = format!( "$PSModuleAutoloadingPreference = 'None' $PSModulePathBackup = $env:PSModulePath -$env:PSModulePath = ($Env:PSModulePath -split ':')[-1] +$env:PSModulePath = \"$PSHome/Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{}:$PSModulePathBackup\"", POWERSHELL_CACHE_DIR ); + + #[cfg(windows)] + let profile = format!( + "$PSModuleAutoloadingPreference = 'None' +$PSModulePathBackup = $env:PSModulePath +$env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" +Get-Module -ListAvailable | Import-Module +$env:PSModulePath = \"{};$PSModulePathBackup\"", + POWERSHELL_CACHE_DIR + ); + // make sure param() is first let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); let content: String = if let Some(param_match) = param_match { @@ -312,11 +333,29 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", }; write_file(job_dir, "main.ps1", content.as_str())?; + + #[cfg(unix)] write_file( job_dir, "wrapper.sh", &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n{} -F ./main.ps1 \"$@\" 2>&1 | tee bp\nwait $!", POWERSHELL_PATH.as_str()), )?; + + #[cfg(windows)] + write_file( + job_dir, + "wrapper.ps1", + &format!( + "param([string[]]$args)\n\ + $ErrorActionPreference = 'Stop'\n\ + $pipe = New-TemporaryFile\n\ + & \"{}\" -File ./main.ps1 @args 2>&1 | Tee-Object -FilePath $pipe\n\ + Get-Content -Path $pipe | Select-Object -Last 1 | Set-Content -Path './result2.out'\n\ + Remove-Item $pipe\n", + POWERSHELL_PATH.as_str() + ), + )?; + let token = client.get_token().await; let mut reserved_variables = get_reserved_variables(job, &token, db).await?; reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); @@ -355,10 +394,24 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", .stderr(Stdio::piped()) .spawn()? } else { - let mut cmd_args = vec!["wrapper.sh"]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - Command::new(BIN_BASH.as_str()) - .current_dir(job_dir) + let mut cmd; + let mut cmd_args; + + #[cfg(unix)] + { + cmd_args = vec!["wrapper.sh"]; + cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); + cmd = Command::new(BIN_BASH.as_str()); + } + + #[cfg(windows)] + { + cmd_args = vec![r".\wrapper.ps1".to_string()]; + cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-"))); + cmd = Command::new(POWERSHELL_PATH.as_str()); + } + + cmd.current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) @@ -366,11 +419,53 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) + .args(&cmd_args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "LOCALAPPDATA", + std::env::var("LOCALAPPDATA") + .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), + ) + .env( + "ProgramData", + std::env::var("ProgramData") + .unwrap_or_else(|_| String::from("C:\\ProgramData")), + ) + .env( + "ProgramFiles", + std::env::var("ProgramFiles") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "ProgramFiles(x86)", + std::env::var("ProgramFiles(x86)") + .unwrap_or_else(|_| String::from("C:\\Program Files (x86)")), + ) + .env( + "ProgramW6432", + std::env::var("ProgramW6432") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "PATHEXT", + std::env::var("PATHEXT").unwrap_or_else(|_| { + String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL") + }), + ); + } + + cmd.spawn()? }; + handle_child( &job.id, db, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index e17f140b3211b..e5e941a0d4e87 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -23,6 +23,9 @@ use crate::{ NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + use tokio::{fs::File, process::Command}; use tokio::io::AsyncReadExt; @@ -112,6 +115,9 @@ pub async fn gen_bun_lockfile( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child_cmd, &*BUN_PATH).await?; if let Some(db) = db { @@ -245,6 +251,9 @@ pub async fn install_bun_lockfile( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut npm_logs = if npm_mode { "NPM mode\n".to_string() } else { @@ -453,6 +462,10 @@ pub async fn generate_wrapper_mjs( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let child_process = start_child_process(child, &*BUN_PATH).await?; handle_child( job_id, @@ -498,6 +511,10 @@ pub async fn generate_bun_bundle( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child, &*BUN_PATH).await?; if let Some(db) = db { handle_child( @@ -540,7 +557,11 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(fs::read(bun_cache_path)?.into(), job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(&bun_cache_path, dst).await?; + + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?; } } else if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS .read() @@ -553,7 +574,11 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(bytes, job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(bun_cache_path, dst).await?; + + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?; } // extract_tar(bytes, job_dir).await?; @@ -722,6 +747,10 @@ async fn compute_bundle_local_and_remote_path( let hash = windmill_common::utils::calculate_hash(&input_src); let local_path = format!("{BUN_BUNDLE_CACHE_DIR}/{hash}"); + + #[cfg(windows)] + let local_path = local_path.replace("/tmp", r"C:\tmp").replace("/", r"\"); + let remote_path = format!("{BUN_BUNDLE_OBJECT_STORE_PREFIX}{hash}"); (local_path, remote_path) } @@ -815,10 +844,23 @@ pub async fn handle_bun_job( )); } - let mut gbuntar_name = None; + let mut gbuntar_name: Option = None; if has_bundle_cache { - let target = format!("{job_dir}/main.js"); - std::os::unix::fs::symlink(&local_path, &target).map_err(|e| { + let target; + let symlink; + + #[cfg(unix)] + { + target = format!("{job_dir}/main.js"); + symlink = std::os::unix::fs::symlink(&local_path, &target); + } + #[cfg(windows)] + { + target = format!("{job_dir}\\main.js"); + symlink = std::os::windows::fs::symlink_dir(&local_path, &target); + } + + symlink.map_err(|e| { error::Error::ExecutionErr(format!( "could not copy cached binary from {local_path} to {job_dir}/main: {e:?}" )) @@ -1326,6 +1368,10 @@ try {{ .args(vec!["--preserve-symlinks", &script_path]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd } else { let script_path = format!("{job_dir}/wrapper.mjs"); @@ -1352,8 +1398,13 @@ try {{ .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd }; + start_child_process( cmd, if annotation.nodejs_mode { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index b1398590052c4..c171e52156ff3 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -225,7 +225,12 @@ func Run(req Req) (interface{{}}, error){{ } } else { let target = format!("{job_dir}/main"); - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index d588ea8fc4703..791b60b9a6c6f 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -6,7 +6,11 @@ use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use sqlx::{Pool, Postgres}; +#[cfg(windows)] +use std::process::Stdio; use tokio::fs::File; +#[cfg(windows)] +use tokio::process::Command; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; @@ -49,6 +53,33 @@ use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; lazy_static::lazy_static! { pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); } + +// - kill windows process along with all child processes +#[cfg(windows)] +async fn kill_process_tree(pid: Option) -> Result<(), String> { + let pid = match pid { + Some(pid) => pid, + None => return Err("No PID provided to kill.".to_string()), + }; + + let output = Command::new("cmd") + .args(&["/C", "taskkill", "/PID", &pid.to_string(), "/T", "/F"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("Failed to execute taskkill: {}", e))?; + + if output.status.success() { + Ok(()) + } else { + Err(format!( + "Failed to kill process tree. Error: {}", + String::from_utf8_lossy(&output.stderr) + )) + } +} + /// - wait until child exits and return with exit status /// - read lines from stdout and stderr and append them to the "queue"."logs" /// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes) @@ -196,9 +227,26 @@ pub async fn handle_child( } } } - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, child.kill()).await; - kill.map(|()| Err(kill_reason)) + #[cfg(windows)] + { + let pid_to_kill = child.id(); + match kill_process_tree(pid_to_kill).await { + Ok(_) => tracing::debug!( + "successfully killed process tree with PID: {:?}", + pid_to_kill + ), + Err(e) => tracing::error!("failed to kill process tree: {:?}", e), + }; + set_reason.await; + return Ok(Err(kill_reason)); + } + + #[cfg(unix)] + { + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, child.kill()).await; + kill.map(|()| Err(kill_reason)) + } }; /* a future that reads output from the child and appends to the database */ diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 87c1c829ae125..a5717ed12bb08 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -64,6 +64,9 @@ use crate::{ PIP_INDEX_URL, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + pub async fn create_dependencies_dir(job_dir: &str) { DirBuilder::new() .recursive(true) @@ -399,6 +402,9 @@ except BaseException as e: let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?; let additional_python_paths_folders = additional_python_paths.iter().join(":"); + #[cfg(windows)] + let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); + if !*DISABLE_NSJAIL { let shared_deps = additional_python_paths .into_iter() @@ -475,6 +481,10 @@ mount {{ .args(vec!["-u", "-m", "wrapper"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(python_cmd, PYTHON_PATH.as_str()).await? }; @@ -1016,7 +1026,12 @@ pub async fn handle_python_reqs( start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string(); + #[cfg(unix)] let req = format!("'{}'", req); + + #[cfg(windows)] + let req = format!("{}", req); + let mut command_args = vec![ PYTHON_PATH.as_str(), "-m", @@ -1072,19 +1087,35 @@ pub async fn handle_python_reqs( tracing::debug!("pip install command: {:?}", command_args); - let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); - flock_cmd - .env_clear() - .envs(envs) - .args([ - "-x", - &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), - "--command", - &command_args.join(" "), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + #[cfg(unix)] + { + let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); + flock_cmd + .env_clear() + .envs(envs) + .args([ + "-x", + &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + "--command", + &command_args.join(" "), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + } + + #[cfg(windows)] + { + let mut pip_cmd = Command::new(PYTHON_PATH.as_str()); + pip_cmd + .env_clear() + .envs(envs) + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .args(&command_args[1..]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(pip_cmd, PYTHON_PATH.as_str()).await? + } }; let child = handle_child( diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index ab6fdd247f2ce..6a0090ab55984 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -23,12 +23,28 @@ use crate::{ RUST_CACHE_DIR, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); lazy_static::lazy_static! { - static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string()); - static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string()); - static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string())); + static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); + static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); + static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); + static ref CARGO_PATH: String = format!("{}/bin/cargo", CARGO_HOME.as_str()); +} + +#[cfg(windows)] +lazy_static::lazy_static! { + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); +} + +#[cfg(unix)] +lazy_static::lazy_static! { + static ref CARGO_HOME_DEFAULT: String = "/usr/local/cargo".to_string(); + static ref RUSTUP_HOME_DEFAULT: String = "/usr/local/rustup".to_string(); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -126,6 +142,14 @@ pub async fn generate_cargo_lockfile( .args(vec!["generate-lockfile"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + { + gen_lockfile_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + gen_lockfile_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -176,6 +200,16 @@ pub async fn build_rust_crate( .args(vec!["build", "--release"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + { + build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + build_rust_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } + let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -279,7 +313,13 @@ pub async fn handle_rust_job( let cache_logs = if cache { let target = format!("{job_dir}/main"); - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { + + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) @@ -360,6 +400,9 @@ pub async fn handle_rust_job( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(run_rust, compiled_executable_name).await? }; handle_child( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 6c0fce3d3bd13..918c563ea31d0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -348,7 +348,6 @@ lazy_static::lazy_static! { pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); - static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() .and_then(|x| x.parse::().ok()) @@ -389,6 +388,12 @@ lazy_static::lazy_static! { } + +#[cfg(windows)] +lazy_static::lazy_static! { + pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); +} + //only matter if CLOUD_HOSTED pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB