diff --git a/process/drivers.rs b/process/drivers.rs index adc187e1..8ad4bd28 100644 --- a/process/drivers.rs +++ b/process/drivers.rs @@ -271,6 +271,12 @@ fn get_version_run_image(oci_ref: &Reference) -> Result { ); progress.enable_steady_tick(Duration::from_millis(100)); + let should_remove = if matches!(Driver::get_run_driver(), RunDriverType::Docker) { + !Driver::list_images()?.contains(oci_ref) + } else { + false + }; + let output = Driver::run_output( &RunOpts::builder() .image(oci_ref.to_string()) @@ -284,6 +290,10 @@ fn get_version_run_image(oci_ref: &Reference) -> Result { .build(), )?; + if should_remove { + Driver::remove_image(oci_ref)?; + } + progress.finish_and_clear(); Logger::multi_progress().remove(&progress); @@ -396,6 +406,22 @@ impl RunDriver for Driver { fn run_output(opts: &RunOpts) -> Result { impl_run_driver!(run_output(opts)) } + + fn create_container(image: &Reference) -> Result { + impl_run_driver!(create_container(image)) + } + + fn remove_container(container_id: &types::ContainerId) -> Result<()> { + impl_run_driver!(remove_container(container_id)) + } + + fn remove_image(image: &Reference) -> Result<()> { + impl_run_driver!(remove_image(image)) + } + + fn list_images() -> Result> { + impl_run_driver!(list_images()) + } } macro_rules! impl_ci_driver { @@ -447,18 +473,6 @@ impl CiDriver for Driver { #[cfg(feature = "rechunk")] impl ContainerMountDriver for Driver { - fn create_container(image: &Reference) -> Result { - PodmanDriver::create_container(image) - } - - fn remove_container(container_id: &types::ContainerId) -> Result<()> { - PodmanDriver::remove_container(container_id) - } - - fn remove_image(image: &Reference) -> Result<()> { - PodmanDriver::remove_image(image) - } - fn mount_container(container_id: &types::ContainerId) -> Result { PodmanDriver::mount_container(container_id) } diff --git a/process/drivers/docker_driver.rs b/process/drivers/docker_driver.rs index 7e4af782..6598c61e 100644 --- a/process/drivers/docker_driver.rs +++ b/process/drivers/docker_driver.rs @@ -7,7 +7,7 @@ use std::{ }; use blue_build_utils::{ - constants::{BB_BUILDKIT_CACHE_GHA, CONTAINER_FILE, DOCKER_HOST, GITHUB_ACTIONS}, + constants::{BB_BUILDKIT_CACHE_GHA, DOCKER_HOST, GITHUB_ACTIONS}, credentials::Credentials, string_vec, }; @@ -16,6 +16,7 @@ use colored::Colorize; use comlexr::cmd; use log::{debug, info, trace, warn}; use miette::{bail, miette, IntoDiagnostic, Result}; +use oci_distribution::Reference; use once_cell::sync::Lazy; use semver::Version; use serde::Deserialize; @@ -30,8 +31,7 @@ use crate::{ RunOptsVolume, TagOpts, }, traits::{BuildDriver, DriverVersion, InspectDriver, RunDriver}, - types::ImageMetadata, - types::Platform, + types::{ContainerId, ImageMetadata, Platform}, }, logging::CommandLogging, signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId}, @@ -66,9 +66,13 @@ impl DockerDriver { } trace!("docker buildx ls --format={}", "{{.Name}}"); - let ls_out = cmd!("docker", "buildx", "ls", "--format={{.Name}}") - .output() - .into_diagnostic()?; + let ls_out = { + let c = cmd!("docker", "buildx", "ls", "--format={{.Name}}"); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; if !ls_out.status.success() { bail!("{}", String::from_utf8_lossy(&ls_out.stderr)); @@ -79,15 +83,18 @@ impl DockerDriver { trace!("{ls_out}"); if !ls_out.lines().any(|line| line == "bluebuild") { - trace!("docker buildx create --bootstrap --driver=docker-container --name=bluebuild"); - let create_out = cmd!( - "docker", - "buildx", - "create", - "--bootstrap", - "--driver=docker-container", - "--name=bluebuild", - ) + let create_out = { + let c = cmd!( + "docker", + "buildx", + "create", + "--bootstrap", + "--driver=docker-container", + "--name=bluebuild", + ); + trace!("{c:?}"); + c + } .output() .into_diagnostic()?; @@ -108,9 +115,13 @@ impl DriverVersion for DockerDriver { const VERSION_REQ: &'static str = ">=23"; fn version() -> Result { - let output = cmd!("docker", "version", "-f", "json") - .output() - .into_diagnostic()?; + let output = { + let c = cmd!("docker", "version", "-f", "json"); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; let version_json: DockerVersionJson = serde_json::from_slice(&output.stdout).into_diagnostic()?; @@ -127,20 +138,23 @@ impl BuildDriver for DockerDriver { warn!("Squash is deprecated for docker so this build will not squash"); } - trace!("docker build -t {} -f {CONTAINER_FILE} .", opts.image); - let status = cmd!( - "docker", - "build", - if !matches!(opts.platform, Platform::Native) => [ - "--platform", - opts.platform.to_string(), - ], - "-t", - &*opts.image, - "-f", - &*opts.containerfile, - ".", - ) + let status = { + let c = cmd!( + "docker", + "build", + if !matches!(opts.platform, Platform::Native) => [ + "--platform", + opts.platform.to_string(), + ], + "-t", + &*opts.image, + "-f", + &*opts.containerfile, + ".", + ); + trace!("{c:?}"); + c + } .status() .into_diagnostic()?; @@ -157,10 +171,13 @@ impl BuildDriver for DockerDriver { let dest_image_str = opts.dest_image.to_string(); - trace!("docker tag {} {}", opts.src_image, opts.dest_image); - let status = cmd!("docker", "tag", opts.src_image.to_string(), &dest_image_str) - .status() - .into_diagnostic()?; + let status = { + let c = cmd!("docker", "tag", opts.src_image.to_string(), &dest_image_str); + trace!("{c:?}"); + c + } + .status() + .into_diagnostic()?; if status.success() { info!("Successfully tagged {}!", dest_image_str.bold().green()); @@ -175,10 +192,13 @@ impl BuildDriver for DockerDriver { let image_str = opts.image.to_string(); - trace!("docker push {}", opts.image); - let status = cmd!("docker", "push", &image_str) - .status() - .into_diagnostic()?; + let status = { + let c = cmd!("docker", "push", &image_str); + trace!("{c:?}"); + c + } + .status() + .into_diagnostic()?; if status.success() { info!("Successfully pushed {}!", image_str.bold().green()); @@ -241,14 +261,18 @@ impl BuildDriver for DockerDriver { let (system, buildx) = std::thread::scope( |scope| -> std::thread::Result<(Result, Result)> { let system = scope.spawn(|| { - cmd!( - "docker", - "system", - "prune", - "--force", - if opts.all => "--all", - if opts.volumes => "--volumes", - ) + { + let c = cmd!( + "docker", + "system", + "prune", + "--force", + if opts.all => "--all", + if opts.volumes => "--volumes", + ); + trace!("{c:?}"); + c + } .message_status("docker system prune", "Pruning Docker System") .into_diagnostic() }); @@ -260,14 +284,18 @@ impl BuildDriver for DockerDriver { Self::setup()?; } - cmd!( - "docker", - "buildx", - "prune", - "--force", - if run_setup => "--builder=bluebuild", - if opts.all => "--all", - ) + { + let c = cmd!( + "docker", + "buildx", + "prune", + "--force", + if run_setup => "--builder=bluebuild", + if opts.all => "--all", + ); + trace!("{c:?}"); + c + } .message_status("docker buildx prune", "Pruning Docker Buildx") .into_diagnostic() }); @@ -465,6 +493,97 @@ impl RunDriver for DockerDriver { Ok(output) } + + fn create_container(image: &oci_distribution::Reference) -> Result { + trace!("DockerDriver::create_container({image})"); + + let output = { + let c = cmd!("docker", "create", image.to_string(), "bash",); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to create container from image {image}"); + } + + Ok(ContainerId( + String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?, + )) + } + + fn remove_container(container_id: &super::types::ContainerId) -> Result<()> { + trace!("DockerDriver::remove_container({container_id})"); + + let output = { + let c = cmd!("docker", "rm", container_id); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to remove container {container_id}"); + } + + Ok(()) + } + + fn remove_image(image: &oci_distribution::Reference) -> Result<()> { + trace!("DockerDriver::remove_image({image})"); + + let output = { + let c = cmd!("docker", "rmi", image.to_string()); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to remove the image {image}"); + } + + Ok(()) + } + + fn list_images() -> Result> { + #[derive(Deserialize)] + #[serde(rename_all = "PascalCase")] + struct Image { + repository: String, + tag: String, + } + + let output = { + let c = cmd!("docker", "images", "--format", "json"); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to list images"); + } + + let images: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .map(|line| serde_json::from_str::(line).into_diagnostic()) + .collect::>()?; + + images + .into_iter() + .map(|image| { + format!("{}:{}", image.repository, image.tag) + .parse::() + .into_diagnostic() + }) + .collect() + } } fn docker_run(opts: &RunOpts, cid_file: &Path) -> Command { diff --git a/process/drivers/podman_driver.rs b/process/drivers/podman_driver.rs index a76f9b58..3fe2cb77 100644 --- a/process/drivers/podman_driver.rs +++ b/process/drivers/podman_driver.rs @@ -28,11 +28,9 @@ use crate::{ signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId}, }; +use super::types::ContainerId; #[cfg(feature = "rechunk")] -use super::{ - types::{ContainerId, MountId}, - ContainerMountDriver, RechunkDriver, -}; +use super::{types::MountId, ContainerMountDriver, RechunkDriver}; #[derive(Deserialize, Debug, Clone)] #[serde(rename_all = "PascalCase")] @@ -345,56 +343,6 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result { #[cfg(feature = "rechunk")] impl ContainerMountDriver for PodmanDriver { - fn create_container(image: &Reference) -> Result { - let output = { - let c = cmd!("podman", "create", image.to_string(), "bash"); - trace!("{c:?}"); - c - } - .output() - .into_diagnostic()?; - - if !output.status.success() { - bail!("Failed to create a container from image {image}"); - } - - Ok(ContainerId( - String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?, - )) - } - - fn remove_container(container_id: &super::types::ContainerId) -> Result<()> { - let output = { - let c = cmd!("podman", "rm", container_id); - trace!("{c:?}"); - c - } - .output() - .into_diagnostic()?; - - if !output.status.success() { - bail!("Failed to remove container {container_id}"); - } - - Ok(()) - } - - fn remove_image(image: &Reference) -> Result<()> { - let output = { - let c = cmd!("podman", "rmi", image.to_string()); - trace!("{c:?}"); - c - } - .output() - .into_diagnostic()?; - - if !output.status.success() { - bail!("Failed to remove the image {image}"); - } - - Ok(()) - } - fn mount_container(container_id: &super::types::ContainerId) -> Result { let output = { let c = cmd!("podman", "mount", container_id); @@ -493,6 +441,88 @@ impl RunDriver for PodmanDriver { Ok(output) } + + fn create_container(image: &Reference) -> Result { + let output = { + let c = cmd!("podman", "create", image.to_string(), "bash"); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to create a container from image {image}"); + } + + Ok(ContainerId( + String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?, + )) + } + + fn remove_container(container_id: &super::types::ContainerId) -> Result<()> { + let output = { + let c = cmd!("podman", "rm", container_id); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to remove container {container_id}"); + } + + Ok(()) + } + + fn remove_image(image: &Reference) -> Result<()> { + let output = { + let c = cmd!("podman", "rmi", image.to_string()); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to remove the image {image}"); + } + + Ok(()) + } + + fn list_images() -> Result> { + #[derive(Deserialize)] + #[serde(rename_all = "PascalCase")] + struct Image { + names: Option>, + } + + let output = { + let c = cmd!("podman", "images", "--format", "json"); + trace!("{c:?}"); + c + } + .output() + .into_diagnostic()?; + + if !output.status.success() { + bail!("Failed to list images"); + } + + let images: Vec = serde_json::from_slice(&output.stdout).into_diagnostic()?; + + images + .into_iter() + .filter_map(|image| image.names) + .flat_map(|names| { + names + .into_iter() + .map(|name| name.parse::().into_diagnostic()) + }) + .collect() + } } fn podman_run(opts: &RunOpts, cid_file: &Path) -> Command { diff --git a/process/drivers/traits.rs b/process/drivers/traits.rs index 891b1e63..fe2b60ac 100644 --- a/process/drivers/traits.rs +++ b/process/drivers/traits.rs @@ -28,13 +28,10 @@ use super::{ }, podman_driver::PodmanDriver, skopeo_driver::SkopeoDriver, - types::ImageMetadata, + types::{ContainerId, ImageMetadata}, }; #[cfg(feature = "rechunk")] -use super::{ - opts::RechunkOpts, - types::{ContainerId, MountId}, -}; +use super::{opts::RechunkOpts, types::MountId}; trait PrivateDriver {} @@ -216,11 +213,7 @@ pub trait RunDriver: PrivateDriver { /// # Errors /// Will error if there is an issue running the container. fn run_output(opts: &RunOpts) -> Result; -} -#[allow(private_bounds)] -#[cfg(feature = "rechunk")] -pub(super) trait ContainerMountDriver: PrivateDriver { /// Creates container /// /// # Errors @@ -239,6 +232,16 @@ pub(super) trait ContainerMountDriver: PrivateDriver { /// Will error if the image remove command fails. fn remove_image(image: &Reference) -> Result<()>; + /// List all images in the local image registry. + /// + /// # Errors + /// Will error if the image list command fails. + fn list_images() -> Result>; +} + +#[allow(private_bounds)] +#[cfg(feature = "rechunk")] +pub(super) trait ContainerMountDriver: PrivateDriver { /// Mounts the container /// /// # Errors diff --git a/process/drivers/types.rs b/process/drivers/types.rs index e1cc6aaa..d76731d4 100644 --- a/process/drivers/types.rs +++ b/process/drivers/types.rs @@ -236,17 +236,14 @@ impl ImageMetadata { } } -#[cfg(feature = "rechunk")] pub struct ContainerId(pub(super) String); -#[cfg(feature = "rechunk")] impl std::fmt::Display for ContainerId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", &self.0) } } -#[cfg(feature = "rechunk")] impl AsRef for ContainerId { fn as_ref(&self) -> &std::ffi::OsStr { self.0.as_ref() diff --git a/src/commands/build.rs b/src/commands/build.rs index abaf39e9..49c7a359 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -14,8 +14,8 @@ use blue_build_process_management::{ use blue_build_recipe::Recipe; use blue_build_utils::{ constants::{ - ARCHIVE_SUFFIX, BB_BUILD_RECHUNK, BB_BUILD_RECHUNK_CLEAR_PLAN, BB_REGISTRY_NAMESPACE, - CONFIG_PATH, CONTAINER_FILE, RECIPE_FILE, RECIPE_PATH, + ARCHIVE_SUFFIX, BB_REGISTRY_NAMESPACE, CONFIG_PATH, CONTAINER_FILE, RECIPE_FILE, + RECIPE_PATH, }, cowstr, credentials::{Credentials, CredentialsArgs}, @@ -118,7 +118,7 @@ pub struct BuildCommand { /// and take up more space during build-time. /// /// NOTE: This must be run as root! - #[arg(long, group = "archive_rechunk", env = BB_BUILD_RECHUNK)] + #[arg(long, group = "archive_rechunk", env = blue_build_utils::constants::BB_BUILD_RECHUNK)] #[builder(default)] #[cfg(feature = "rechunk")] rechunk: bool, @@ -126,7 +126,7 @@ pub struct BuildCommand { /// Use a fresh rechunk plan, regardless of previous ref. /// /// NOTE: Only works with `--rechunk`. - #[arg(long, env = BB_BUILD_RECHUNK_CLEAR_PLAN)] + #[arg(long, env = blue_build_utils::constants::BB_BUILD_RECHUNK_CLEAR_PLAN)] #[builder(default)] #[cfg(feature = "rechunk")] rechunk_clear_plan: bool,