From 0bf4828186a125713864989aa7665ab5de4239db Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 26 Mar 2020 01:43:45 -0600 Subject: [PATCH] rust sketch --- Cargo.toml | 5 ++ pypiscript/Cargo.toml | 11 +++ pypiscript/config-example.json | 4 + pypiscript/src/main.rs | 139 +++++++++++++++++++++++++++++++++ pypiscript/work/task.json | 13 +++ script/Cargo.toml | 13 +++ script/macros/Cargo.toml | 12 +++ script/macros/src/lib.rs | 25 ++++++ script/src/error.rs | 59 ++++++++++++++ script/src/lib.rs | 76 ++++++++++++++++++ script/src/task.rs | 135 ++++++++++++++++++++++++++++++++ 11 files changed, 492 insertions(+) create mode 100644 Cargo.toml create mode 100644 pypiscript/Cargo.toml create mode 100644 pypiscript/config-example.json create mode 100644 pypiscript/src/main.rs create mode 100644 pypiscript/work/task.json create mode 100644 script/Cargo.toml create mode 100644 script/macros/Cargo.toml create mode 100644 script/macros/src/lib.rs create mode 100644 script/src/error.rs create mode 100644 script/src/lib.rs create mode 100644 script/src/task.rs diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..5d31a439f --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] + +members = [ + "pypiscript" +] diff --git a/pypiscript/Cargo.toml b/pypiscript/Cargo.toml new file mode 100644 index 000000000..5069b08c6 --- /dev/null +++ b/pypiscript/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "pypiscript" +version = "0.1.0" +authors = ["Tom Prince "] +edition = "2018" + +[dependencies] +clap = "2.33.0" +serde = "1.0.99" +serde_derive = "1.0.99" +scriptworker_script = { path = "../script" } diff --git a/pypiscript/config-example.json b/pypiscript/config-example.json new file mode 100644 index 000000000..d2a519927 --- /dev/null +++ b/pypiscript/config-example.json @@ -0,0 +1,4 @@ +{ + "taskcluster_scope_prefix": "project:releng", + "project_config_file": "/Depot/Mozilla/scriptworker-scripts/pypiscript/passwords.yml" +} diff --git a/pypiscript/src/main.rs b/pypiscript/src/main.rs new file mode 100644 index 000000000..76337ba8d --- /dev/null +++ b/pypiscript/src/main.rs @@ -0,0 +1,139 @@ +use scriptworker_script::{Context, Error, Task}; +use serde_derive::Deserialize; +use std::collections::HashMap; +use std::os::unix::process::ExitStatusExt; +use std::process::Command; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Config { + #[serde( + alias = "project_config_file", + deserialize_with = "scriptworker_script::load_secrets" + )] + projects: HashMap, + #[serde(alias = "taskcluster_scope_prefix")] + scope_prefix: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Project { + api_token: String, + //#[serde(default = "https://test.pypi/legacy/")] + repository_url: String, +} + +#[derive(Deserialize, Debug)] +struct Attr { + project: String, +} + +#[derive(Deserialize, Debug)] +struct Extra { + action: String, +} + +fn verify_payload(config: &Config, _: &Context, task: &Task) -> Result<(), Error> { + if task.payload.extra.action != "upload" { + return Err(Error::MalformedPayload(format!( + "Unsupported action: {}", + task.payload.extra.action + ))); + } + + task.require_scopes(task.payload.upstream_artifacts.iter().map(|upstream| { + let project_name = &upstream.attributes.project; + format!("{}:pypi:project:{}", config.scope_prefix, project_name) + })) +} + +fn run_command(mut command: Command, action: &dyn Fn() -> String) -> Result<(), Error> { + println!("Running: {:?}", command); + match command.status() { + Ok(result) => { + if !result.success() { + println!( + "Failed to {}: {}", + action(), + match (result.code(), result.signal()) { + (Some(code), _) => format!("exit code {}", code), + (_, Some(signal)) => format!("exited with signal {}", signal), + (None, None) => format!("unknown exit reason"), + } + ); + return Err(Error::Failure); + } + Ok(()) + } + Err(err) => { + println!("Failed to start command: {:?}", err); + return Err(Error::Failure); + } + } +} + +impl Config { + fn get_project(&self, project_name: &str) -> Result<&Project, Error> { + self.projects + .get(project_name) + .ok_or(Error::MalformedPayload(format!( + "Unknown pypi project {}", + project_name + ))) + } +} + +#[scriptworker_script::main] +fn do_work(config: Config, context: &Context, task: Task) -> Result<(), Error> { + verify_payload(&config, &context, &task)?; + + task.payload + .upstream_artifacts + .iter() + .map(|upstream| -> Result<(), Error> { + let project_name = &upstream.attributes.project; + // Ensure project exists + config.get_project(project_name)?; + + let mut command = Command::new("twine"); + command.arg("check"); + for artifact in &upstream.paths { + command.arg(artifact.file_path(context)); + } + run_command(command, &|| format!("upload files for {}", project_name)) + }) + .fold(Ok(()), Result::or)?; + + for upstream in &task.payload.upstream_artifacts { + let project_name = &upstream.attributes.project; + let project = config.get_project(project_name)?; + + println!( + "Uploading {} from task {} to {} for project {}", + &upstream + .paths + .iter() + .map(|p| p.task_path().to_string_lossy()) + .collect::>() + .join(", "), + &upstream.task_id, + project.repository_url, + project_name + ); + + let mut command = Command::new("twine"); + command + .arg("upload") + .arg("--user") + .arg("__token__") + .arg("--repository-url") + .arg(&project.repository_url); + for artifact in &upstream.paths { + command.arg(artifact.file_path(context)); + } + command.env("TWINE_PASSWORD", &project.api_token); + run_command(command, &|| format!("upload files for {}", project_name))?; + } + Ok(()) +} diff --git a/pypiscript/work/task.json b/pypiscript/work/task.json new file mode 100644 index 000000000..6b93cca37 --- /dev/null +++ b/pypiscript/work/task.json @@ -0,0 +1,13 @@ +{ + "scopes": ["project:releng:pypi:project:redo"], + "dependencies": [], + "payload": { + "action": "upload", + "upstreamArtifacts": [{ + "taskId": "slug", + "taskType": "scriptworker", + "paths": ["public/redo-2.0.3-py2.py3-none-any.whl"], + "project": "redo" + }] + } +} diff --git a/script/Cargo.toml b/script/Cargo.toml new file mode 100644 index 000000000..840250ae7 --- /dev/null +++ b/script/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "scriptworker_script" +version = "0.1.0" +authors = ["Tom Prince "] +edition = "2018" + +[dependencies] +clap = "2.33.0" +serde_yaml = "0.8.9" +serde_json = "1.0.40" +serde = "1.0.99" +serde_derive = "1.0.99" +scriptworker_script_macros = { path = "macros" } diff --git a/script/macros/Cargo.toml b/script/macros/Cargo.toml new file mode 100644 index 000000000..5a755d4bf --- /dev/null +++ b/script/macros/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "scriptworker_script_macros" +version = "0.1.0" +authors = ["Tom Prince "] +edition = "2018" + +[dependencies] +syn = {version = "1.0.5", features=["full"]} +quote = "1.0.2" + +[lib] +proc-macro = true diff --git a/script/macros/src/lib.rs b/script/macros/src/lib.rs new file mode 100644 index 000000000..b20ac4bd7 --- /dev/null +++ b/script/macros/src/lib.rs @@ -0,0 +1,25 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; + +#[proc_macro_attribute] +#[cfg(not(test))] // Work around for rust-lang/rust#62127 +pub fn main(args: TokenStream, item: TokenStream) -> TokenStream { + let input = syn::parse_macro_input!(item as syn::ItemFn); + let args = syn::parse_macro_input!(args as syn::AttributeArgs); + + if !args.is_empty() { + panic!("???") + } + + let name = &input.sig.ident; + + let result = quote! { + #input + fn main() { + ::scriptworker_script::scriptworker_main(#name) + } + }; + result.into() +} diff --git a/script/src/error.rs b/script/src/error.rs new file mode 100644 index 000000000..a64d989e4 --- /dev/null +++ b/script/src/error.rs @@ -0,0 +1,59 @@ +use std::convert::From; + +#[derive(Clone)] +pub enum Error { + Failure, + WorkerShutdown, + MalformedPayload(String), + ResourceUnavailable, + InternalError(String), + Superseded, + IntermittentTask, +} + +impl From for Error { + fn from(err: std::io::Error) -> Error { + Error::InternalError(format!("{}", err)) + } +} + +impl From for Error { + fn from(err: serde_yaml::Error) -> Error { + Error::InternalError(format!("{}", err)) + } +} + +impl Error { + pub(crate) fn exit_code(self) -> i32 { + match self { + Self::Failure => 1, + Self::WorkerShutdown => 2, + Self::MalformedPayload(_) => 3, + Self::ResourceUnavailable => 4, + Self::InternalError(_) => 5, + Self::Superseded => 6, + Self::IntermittentTask => 7, + } + } + + #[allow(dead_code)] + pub(crate) fn description(self) -> &'static str { + match self { + Self::Failure => "failure", + Self::WorkerShutdown => "worker-shutdown", + Self::MalformedPayload(_) => "malformed-payload", + Self::ResourceUnavailable => "resource-unavailable", + Self::InternalError(_) => "internal-error", + Self::Superseded => "superseded", + Self::IntermittentTask => "intermittent-task", + } + } +} + +/* +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, + } +} +*/ diff --git a/script/src/lib.rs b/script/src/lib.rs new file mode 100644 index 000000000..9c29dae02 --- /dev/null +++ b/script/src/lib.rs @@ -0,0 +1,76 @@ +use serde::de::DeserializeOwned; +use std::path::{Path, PathBuf}; + +use clap::{App, Arg}; + +mod error; +pub use error::Error; + +pub mod task; +pub use task::Task; + +pub struct Context { + work_dir: PathBuf, +} + +fn init_config() -> Result<(T, PathBuf), Error> +where + T: DeserializeOwned, +{ + let matches = App::new("scriptworker") + .arg(Arg::with_name("CONFIG_FILE").index(1).required(true)) + .arg(Arg::with_name("WORK_DIR").index(2).required(true)) + .get_matches(); + + let config_file = matches.value_of_os("CONFIG_FILE").unwrap(); + let work_dir = Path::new(matches.value_of_os("WORK_DIR").unwrap()); + Ok(( + serde_yaml::from_reader(std::fs::File::open(config_file)?)?, + work_dir.into(), + )) +} + +pub fn load_secrets<'de, D, T>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, + T: DeserializeOwned, +{ + let secret_file_path: String = serde::Deserialize::deserialize(deserializer)?; + let secret_file = std::fs::File::open(secret_file_path) + .map_err(|_| serde::de::Error::custom("Could not open secret file."))?; + Ok(serde_yaml::from_reader(secret_file) + .map_err(|_| serde::de::Error::custom("Could not parse secrets file."))?) +} + +pub fn scriptworker_main( + do_work: impl FnOnce(Config, &Context, Task) -> Result<(), Error>, +) where + Config: DeserializeOwned, + A: DeserializeOwned, + E: DeserializeOwned, +{ + let result = (|| { + let (config, work_dir) = init_config::()?; + // TODO: logging + let task_filename = work_dir.join("task.json"); + let task = Task::::load(&task_filename)?; + + do_work(config, &Context { work_dir: work_dir }, task) + })(); + match result { + Ok(()) => std::process::exit(0), + Err(err) => { + if let Error::MalformedPayload(message) = &err { + std::println!("{}", &message) + } + if let Error::InternalError(message) = &err { + std::println!("{}", &message) + } + std::process::exit(err.exit_code()) + } + } + // TODO: Statuses +} + +#[cfg(not(test))] // Work around for rust-lang/rust#62127 +pub use scriptworker_script_macros::main; diff --git a/script/src/task.rs b/script/src/task.rs new file mode 100644 index 000000000..ee053d03e --- /dev/null +++ b/script/src/task.rs @@ -0,0 +1,135 @@ +use std::path::{Path, PathBuf}; + +use serde::de::DeserializeOwned; +use serde_derive::Deserialize; + +use crate::error::Error; +use crate::Context; + +#[derive(Deserialize, Debug)] +pub struct Empty {} + +#[derive(Debug)] +pub struct TaskArtifacts { + pub task_type: String, + pub task_id: String, + // TODO: Path + pub paths: Vec, + pub attributes: A, +} + +impl<'de, A> serde::Deserialize<'de> for TaskArtifacts +where + A: serde::Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct RawArtifacts { + pub task_type: String, + pub task_id: String, + // TODO: Path + pub paths: Vec, + #[serde(flatten)] + pub attributes: A, + } + let raw: RawArtifacts = serde::Deserialize::deserialize(deserializer)?; + let task_id = raw.task_id.clone(); + let paths = raw + .paths + .into_iter() + .map(|path| { + if path.is_absolute() { + return Err(serde::de::Error::custom( + "Cannot sepecify absolute path in upstreamArtifacts.", + )); + } else { + Ok(ArtifactPath { + task_id: task_id.clone(), + path: path, + }) + } + }) + .collect::>()?; + Ok(TaskArtifacts:: { + task_type: raw.task_type, + task_id: raw.task_id, + paths: paths, + attributes: raw.attributes, + }) + } +} + +#[derive(Debug)] +pub struct ArtifactPath { + task_id: String, + path: PathBuf, +} + +impl ArtifactPath { + pub fn task_path(&self) -> &PathBuf { + &self.path + } + pub fn file_path(&self, context: &Context) -> PathBuf { + context + .work_dir + .join("cot") + .join(&self.task_id) + .join(&self.path) + } +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct TaskPayload { + pub upstream_artifacts: Vec>, + #[serde(flatten)] + pub extra: E, +} + +#[derive(Deserialize, Debug)] +pub struct Task { + pub dependencies: Vec, + pub scopes: Vec, + pub payload: TaskPayload, +} + +impl Task { + pub(crate) fn load(filename: &Path) -> Result, Error> + where + A: DeserializeOwned, + E: DeserializeOwned, + { + let file = std::fs::File::open(filename) + .map_err(|_| Error::InternalError("Could not open task definition.".to_string()))?; + Ok(serde_json::from_reader(file).map_err(|err| { + Error::MalformedPayload(format!("Could not parse task payload: {}", err)) + })?) + } + + pub fn require_scope(&self, scope: &str) -> Result<(), Error> { + if self.scopes.iter().any(|x| x == scope) { + Ok(()) + } else { + Err(Error::MalformedPayload(format!("missing scope {}", scope))) + } + } + + pub fn require_scopes(&self, scopes: impl IntoIterator) -> Result<(), Error> { + let missing_scopes: Vec<_> = scopes + .into_iter() + .filter(|scope| self.scopes.iter().all(|x| x != scope)) + .collect(); + if missing_scopes.is_empty() { + Ok(()) + } else { + Err(Error::MalformedPayload(format!( + "missing scopes: {:?}", + missing_scopes + ))) + } + } +}