rust sketch

Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+
+members = [
+  "pypiscript"
+]

pypiscript/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "pypiscript"
+version = "0.1.0"
+authors = ["Tom Prince <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+clap = "2.33.0"
+serde = "1.0.99"
+serde_derive = "1.0.99"
+scriptworker_script = { path = "../script" }

pypiscript/config-example.json

@@ -0,0 +1,4 @@
+{
+	"taskcluster_scope_prefix": "project:releng",
+	"project_config_file": "/Depot/Mozilla/scriptworker-scripts/pypiscript/passwords.yml"
+}

pypiscript/src/main.rs

@@ -0,0 +1,139 @@
+use scriptworker_script::{Context, Error, Task};
+use serde_derive::Deserialize;
+use std::collections::HashMap;
+use std::os::unix::process::ExitStatusExt;
+use std::process::Command;
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Config {
+    #[serde(
+        alias = "project_config_file",
+        deserialize_with = "scriptworker_script::load_secrets"
+    )]
+    projects: HashMap<String, Project>,
+    #[serde(alias = "taskcluster_scope_prefix")]
+    scope_prefix: String,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Project {
+    api_token: String,
+    //#[serde(default = "https://test.pypi/legacy/")]
+    repository_url: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct Attr {
+    project: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct Extra {
+    action: String,
+}
+
+fn verify_payload(config: &Config, _: &Context, task: &Task<Attr, Extra>) -> Result<(), Error> {
+    if task.payload.extra.action != "upload" {
+        return Err(Error::MalformedPayload(format!(
+            "Unsupported action: {}",
+            task.payload.extra.action
+        )));
+    }
+
+    task.require_scopes(task.payload.upstream_artifacts.iter().map(|upstream| {
+        let project_name = &upstream.attributes.project;
+        format!("{}:pypi:project:{}", config.scope_prefix, project_name)
+    }))
+}
+
+fn run_command(mut command: Command, action: &dyn Fn() -> String) -> Result<(), Error> {
+    println!("Running: {:?}", command);
+    match command.status() {
+        Ok(result) => {
+            if !result.success() {
+                println!(
+                    "Failed to {}: {}",
+                    action(),
+                    match (result.code(), result.signal()) {
+                        (Some(code), _) => format!("exit code {}", code),
+                        (_, Some(signal)) => format!("exited with signal {}", signal),
+                        (None, None) => format!("unknown exit reason"),
+                    }
+                );
+                return Err(Error::Failure);
+            }
+            Ok(())
+        }
+        Err(err) => {
+            println!("Failed to start command: {:?}", err);
+            return Err(Error::Failure);
+        }
+    }
+}
+
+impl Config {
+    fn get_project(&self, project_name: &str) -> Result<&Project, Error> {
+        self.projects
+            .get(project_name)
+            .ok_or(Error::MalformedPayload(format!(
+                "Unknown pypi project {}",
+                project_name
+            )))
+    }
+}
+
+#[scriptworker_script::main]
+fn do_work(config: Config, context: &Context, task: Task<Attr, Extra>) -> Result<(), Error> {
+    verify_payload(&config, &context, &task)?;
+
+    task.payload
+        .upstream_artifacts
+        .iter()
+        .map(|upstream| -> Result<(), Error> {
+            let project_name = &upstream.attributes.project;
+            // Ensure project exists
+            config.get_project(project_name)?;
+
+            let mut command = Command::new("twine");
+            command.arg("check");
+            for artifact in &upstream.paths {
+                command.arg(artifact.file_path(context));
+            }
+            run_command(command, &|| format!("upload files for {}", project_name))
+        })
+        .fold(Ok(()), Result::or)?;
+
+    for upstream in &task.payload.upstream_artifacts {
+        let project_name = &upstream.attributes.project;
+        let project = config.get_project(project_name)?;
+
+        println!(
+            "Uploading {} from task {} to {} for project {}",
+            &upstream
+                .paths
+                .iter()
+                .map(|p| p.task_path().to_string_lossy())
+                .collect::<Vec<_>>()
+                .join(", "),
+            &upstream.task_id,
+            project.repository_url,
+            project_name
+        );
+
+        let mut command = Command::new("twine");
+        command
+            .arg("upload")
+            .arg("--user")
+            .arg("__token__")
+            .arg("--repository-url")
+            .arg(&project.repository_url);
+        for artifact in &upstream.paths {
+            command.arg(artifact.file_path(context));
+        }
+        command.env("TWINE_PASSWORD", &project.api_token);
+        run_command(command, &|| format!("upload files for {}", project_name))?;
+    }
+    Ok(())
+}

pypiscript/work/task.json

@@ -0,0 +1,13 @@
+{
+    "scopes": ["project:releng:pypi:project:redo"],
+    "dependencies": [],
+    "payload": {
+        "action": "upload",
+        "upstreamArtifacts": [{
+            "taskId": "slug",
+            "taskType": "scriptworker",
+            "paths": ["public/redo-2.0.3-py2.py3-none-any.whl"],
+            "project": "redo"
+        }]
+    }
+}

script/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "scriptworker_script"
+version = "0.1.0"
+authors = ["Tom Prince <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+clap = "2.33.0"
+serde_yaml = "0.8.9"
+serde_json = "1.0.40"
+serde = "1.0.99"
+serde_derive = "1.0.99"
+scriptworker_script_macros = { path = "macros" }

script/macros/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "scriptworker_script_macros"
+version = "0.1.0"
+authors = ["Tom Prince <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+syn = {version = "1.0.5", features=["full"]}
+quote = "1.0.2"
+
+[lib]
+proc-macro = true

script/macros/src/lib.rs

@@ -0,0 +1,25 @@
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use quote::quote;
+
+#[proc_macro_attribute]
+#[cfg(not(test))] // Work around for rust-lang/rust#62127
+pub fn main(args: TokenStream, item: TokenStream) -> TokenStream {
+    let input = syn::parse_macro_input!(item as syn::ItemFn);
+    let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+    if !args.is_empty() {
+        panic!("???")
+    }
+
+    let name = &input.sig.ident;
+
+    let result = quote! {
+        #input
+        fn main() {
+            ::scriptworker_script::scriptworker_main(#name)
+        }
+    };
+    result.into()
+}

script/src/error.rs

@@ -0,0 +1,59 @@
+use std::convert::From;
+
+#[derive(Clone)]
+pub enum Error {
+    Failure,
+    WorkerShutdown,
+    MalformedPayload(String),
+    ResourceUnavailable,
+    InternalError(String),
+    Superseded,
+    IntermittentTask,
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Error {
+        Error::InternalError(format!("{}", err))
+    }
+}
+
+impl From<serde_yaml::Error> for Error {
+    fn from(err: serde_yaml::Error) -> Error {
+        Error::InternalError(format!("{}", err))
+    }
+}
+
+impl Error {
+    pub(crate) fn exit_code(self) -> i32 {
+        match self {
+            Self::Failure => 1,
+            Self::WorkerShutdown => 2,
+            Self::MalformedPayload(_) => 3,
+            Self::ResourceUnavailable => 4,
+            Self::InternalError(_) => 5,
+            Self::Superseded => 6,
+            Self::IntermittentTask => 7,
+        }
+    }
+
+    #[allow(dead_code)]
+    pub(crate) fn description(self) -> &'static str {
+        match self {
+            Self::Failure => "failure",
+            Self::WorkerShutdown => "worker-shutdown",
+            Self::MalformedPayload(_) => "malformed-payload",
+            Self::ResourceUnavailable => "resource-unavailable",
+            Self::InternalError(_) => "internal-error",
+            Self::Superseded => "superseded",
+            Self::IntermittentTask => "intermittent-task",
+        }
+    }
+}
+
+/*
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f,
+    }
+}
+*/

script/src/lib.rs

@@ -0,0 +1,76 @@
+use serde::de::DeserializeOwned;
+use std::path::{Path, PathBuf};
+
+use clap::{App, Arg};
+
+mod error;
+pub use error::Error;
+
+pub mod task;
+pub use task::Task;
+
+pub struct Context {
+    work_dir: PathBuf,
+}
+
+fn init_config<T>() -> Result<(T, PathBuf), Error>
+where
+    T: DeserializeOwned,
+{
+    let matches = App::new("scriptworker")
+        .arg(Arg::with_name("CONFIG_FILE").index(1).required(true))
+        .arg(Arg::with_name("WORK_DIR").index(2).required(true))
+        .get_matches();
+
+    let config_file = matches.value_of_os("CONFIG_FILE").unwrap();
+    let work_dir = Path::new(matches.value_of_os("WORK_DIR").unwrap());
+    Ok((
+        serde_yaml::from_reader(std::fs::File::open(config_file)?)?,
+        work_dir.into(),
+    ))
+}
+
+pub fn load_secrets<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: DeserializeOwned,
+{
+    let secret_file_path: String = serde::Deserialize::deserialize(deserializer)?;
+    let secret_file = std::fs::File::open(secret_file_path)
+        .map_err(|_| serde::de::Error::custom("Could not open secret file."))?;
+    Ok(serde_yaml::from_reader(secret_file)
+        .map_err(|_| serde::de::Error::custom("Could not parse secrets file."))?)
+}
+
+pub fn scriptworker_main<Config, A, E>(
+    do_work: impl FnOnce(Config, &Context, Task<A, E>) -> Result<(), Error>,
+) where
+    Config: DeserializeOwned,
+    A: DeserializeOwned,
+    E: DeserializeOwned,
+{
+    let result = (|| {
+        let (config, work_dir) = init_config::<Config>()?;
+        // TODO: logging
+        let task_filename = work_dir.join("task.json");
+        let task = Task::<A, E>::load(&task_filename)?;
+
+        do_work(config, &Context { work_dir: work_dir }, task)
+    })();
+    match result {
+        Ok(()) => std::process::exit(0),
+        Err(err) => {
+            if let Error::MalformedPayload(message) = &err {
+                std::println!("{}", &message)
+            }
+            if let Error::InternalError(message) = &err {
+                std::println!("{}", &message)
+            }
+            std::process::exit(err.exit_code())
+        }
+    }
+    // TODO: Statuses
+}
+
+#[cfg(not(test))] // Work around for rust-lang/rust#62127
+pub use scriptworker_script_macros::main;