Skip to content

Commit

Permalink
comments ruben, refactor to simplify
Browse files Browse the repository at this point in the history
  • Loading branch information
@alpetric
alpetric committed Sep 26, 2024
1 parent e81555a commit fb81321
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 115 deletions.
133 changes: 65 additions & 68 deletions backend/windmill-worker/src/bash_executor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,85 +364,82 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
let _ = write_file(job_dir, "result.out", "")?;
let _ = write_file(job_dir, "result2.out", "")?;

let child;
let child = if !*DISABLE_NSJAIL {
let _ = write_file(
job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT
.replace("{JOB_DIR}", job_dir)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR),
)?;
let mut cmd_args = vec![
"--config",
"run.config.proto",
"--",
BIN_BASH.as_str(),
"wrapper.sh",
];
cmd_args.extend(pwsh_args.iter().map(|x| x.as_str()));
Command::new(NSJAIL_PATH.as_str())
.current_dir(job_dir)
.env_clear()
.envs(reserved_variables)
.env("TZ", TZ_ENV.as_str())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.args(cmd_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?
} else {
let mut cmd;
let mut cmd_args;

#[cfg(unix)]
{
child = if !*DISABLE_NSJAIL {
let _ = write_file(
job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT
.replace("{JOB_DIR}", job_dir)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{CACHE_DIR}", POWERSHELL_CACHE_DIR),
)?;
let mut cmd_args = vec![
"--config",
"run.config.proto",
"--",
BIN_BASH.as_str(),
"wrapper.sh",
];
#[cfg(unix)]
{
cmd_args = vec!["wrapper.sh"];
cmd_args.extend(pwsh_args.iter().map(|x| x.as_str()));
Command::new(NSJAIL_PATH.as_str())
.current_dir(job_dir)
.env_clear()
.envs(reserved_variables)
.env("TZ", TZ_ENV.as_str())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.args(cmd_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?
} else {
let mut cmd_args = vec!["wrapper.sh"];
cmd_args.extend(pwsh_args.iter().map(|x| x.as_str()));
Command::new(BIN_BASH.as_str())
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.env("TZ", TZ_ENV.as_str())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("HOME", HOME_ENV.as_str())
.args(cmd_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?
};
}
cmd = Command::new(BIN_BASH.as_str());
}

#[cfg(windows)]
{
let mut cmd_args = vec![r".\wrapper.ps1".to_string()];
cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-")));
#[cfg(windows)]
{
cmd_args = vec![r".\wrapper.ps1".to_string()];
cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-")));
cmd = Command::new(POWERSHELL_PATH.as_str());
}

child = Command::new(POWERSHELL_PATH.as_str())
.current_dir(job_dir)
cmd.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.env("TZ", TZ_ENV.as_str())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("HOME", HOME_ENV.as_str())
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"PATHEXT",
".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL",
)
.args(cmd_args)
.args(&cmd_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
}
.stderr(Stdio::piped());

#[cfg(windows)]
{
cmd.env("SystemRoot", SYSTEM_ROOT.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"PATHEXT",
std::env::var("PATHEXT").unwrap_or_else(|_| {
String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL")
}),
);
}

cmd.spawn()?
};

handle_child(
&job.id,
Expand Down
26 changes: 14 additions & 12 deletions backend/windmill-worker/src/bun_executor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -838,21 +838,23 @@ pub async fn handle_bun_job(

let mut gbuntar_name: Option<String> = None;
if has_bundle_cache {
#[cfg(unix)]
let target = format!("{job_dir}/main.js");
#[cfg(unix)]
std::os::unix::fs::symlink(&local_path, &target).map_err(|e| {
error::Error::ExecutionErr(format!(
"could not copy cached binary from {local_path} to {job_dir}/main: {e:?}"
))
})?;
let target;
let symlink;

#[cfg(unix)]
{
target = format!("{job_dir}/main.js");
symlink = std::os::unix::fs::symlink(&local_path, &target);
}
#[cfg(windows)]
let target = format!("{job_dir}\\main.js");
#[cfg(windows)]
std::os::windows::fs::symlink_dir(&local_path, &target).map_err(|e| {
{
target = format!("{job_dir}\\main.js");
symlink = std::os::windows::fs::symlink_dir(&local_path, &target);
}

symlink.map_err(|e| {
error::Error::ExecutionErr(format!(
"could not copy cached binary from {local_path} to {job_dir}\\main: {e:?}"
"could not copy cached binary from {local_path} to {job_dir}/main: {e:?}"
))
})?;
} else if let Some(codebase) = codebase.as_ref() {
Expand Down
11 changes: 4 additions & 7 deletions backend/windmill-worker/src/go_executor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,14 +226,11 @@ func Run(req Req) (interface{{}}, error){{
} else {
let target = format!("{job_dir}/main");
#[cfg(unix)]
std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| {
Error::ExecutionErr(format!(
"could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
))
})?;

let symlink = std::os::unix::fs::symlink(&bin_path, &target);
#[cfg(windows)]
std::os::windows::fs::symlink_dir(&bin_path, &target).map_err(|e| {
let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target);

symlink.map_err(|e| {
Error::ExecutionErr(format!(
"could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
))
Expand Down
10 changes: 3 additions & 7 deletions backend/windmill-worker/src/python_executor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,9 @@ except BaseException as e:
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
let additional_python_paths_folders = additional_python_paths.iter().join(":");

#[cfg(windows)]
let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";");

if !*DISABLE_NSJAIL {
let shared_deps = additional_python_paths
.into_iter()
Expand Down Expand Up @@ -433,13 +436,6 @@ mount {{
),
)?;
} else {
#[cfg(windows)]
reserved_variables.insert(
"PYTHONPATH".to_string(),
additional_python_paths_folders.replace(":", ";"),
);

#[cfg(unix)]
reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
}

Expand Down
41 changes: 20 additions & 21 deletions backend/windmill-worker/src/rust_executor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,25 @@ use crate::SYSTEM_ROOT;

const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto");

#[cfg(unix)]
lazy_static::lazy_static! {
static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string());
static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string());
static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string()));
static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable");
static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() });
static ref RUSTUP_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() });
static ref CARGO_PATH: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_PATH_DEFAULT.clone() });
}

#[cfg(windows)]
lazy_static::lazy_static! {
static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable");
static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| {
format!("{}\\.cargo", *HOME_DIR)
});
static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| {
format!("{}\\.rustup", *HOME_DIR)
});
static ref CARGO_PATH: String = format!("{}/bin/cargo.exe", *CARGO_HOME);
static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR);
static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR);
static ref CARGO_PATH_DEFAULT: String = format!("{}/bin/cargo.exe", *CARGO_HOME);
}

#[cfg(unix)]
lazy_static::lazy_static! {
static ref CARGO_HOME_DEFAULT: String = "usr/local/cargo".to_string();
static ref RUSTUP_HOME_DEFAULT: String = "/usr/local/rustup".to_string();
static ref CARGO_PATH_DEFAULT: String = "/usr/local/cargo/bin/cargo".to_string();
}

const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/";
Expand Down Expand Up @@ -197,11 +199,10 @@ pub async fn build_rust_crate(
.env("HOME", HOME_ENV.as_str())
.env("CARGO_HOME", CARGO_HOME.as_str())
.env("RUSTUP_HOME", RUSTUP_HOME.as_str())
.args(vec!["build", "--release"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());

build_rust_cmd.args(vec!["build", "--release"]);

#[cfg(windows)]
{
build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
Expand Down Expand Up @@ -314,15 +315,13 @@ pub async fn handle_rust_job(

let cache_logs = if cache {
let target = format!("{job_dir}/main");
#[cfg(unix)]
std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| {
Error::ExecutionErr(format!(
"could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
))
})?;

#[cfg(unix)]
let symlink = std::os::unix::fs::symlink(&bin_path, &target);
#[cfg(windows)]
std::os::windows::fs::symlink_dir(&bin_path, &target).map_err(|e| {
let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target);

symlink.map_err(|e| {
Error::ExecutionErr(format!(
"could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
))
Expand Down

0 comments on commit fb81321

Please sign in to comment.