Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: Remove image for docker inspect after running image to get version
Browse files Browse the repository at this point in the history
gmpinder committed Jan 13, 2025
1 parent 890995d commit 2e63710
Showing 6 changed files with 301 additions and 138 deletions.
38 changes: 26 additions & 12 deletions process/drivers.rs
Original file line number Diff line number Diff line change
@@ -271,6 +271,12 @@ fn get_version_run_image(oci_ref: &Reference) -> Result<u64> {
);
progress.enable_steady_tick(Duration::from_millis(100));

let should_remove = if matches!(Driver::get_run_driver(), RunDriverType::Docker) {
!Driver::list_images()?.contains(oci_ref)
} else {
false
};

let output = Driver::run_output(
&RunOpts::builder()
.image(oci_ref.to_string())
@@ -284,6 +290,10 @@ fn get_version_run_image(oci_ref: &Reference) -> Result<u64> {
.build(),
)?;

if should_remove {
Driver::remove_image(oci_ref)?;
}

progress.finish_and_clear();
Logger::multi_progress().remove(&progress);

@@ -396,6 +406,22 @@ impl RunDriver for Driver {
fn run_output(opts: &RunOpts) -> Result<Output> {
impl_run_driver!(run_output(opts))
}

fn create_container(image: &Reference) -> Result<types::ContainerId> {
impl_run_driver!(create_container(image))
}

fn remove_container(container_id: &types::ContainerId) -> Result<()> {
impl_run_driver!(remove_container(container_id))
}

fn remove_image(image: &Reference) -> Result<()> {
impl_run_driver!(remove_image(image))
}

fn list_images() -> Result<Vec<Reference>> {
impl_run_driver!(list_images())
}
}

macro_rules! impl_ci_driver {
@@ -447,18 +473,6 @@ impl CiDriver for Driver {

#[cfg(feature = "rechunk")]
impl ContainerMountDriver for Driver {
fn create_container(image: &Reference) -> Result<types::ContainerId> {
PodmanDriver::create_container(image)
}

fn remove_container(container_id: &types::ContainerId) -> Result<()> {
PodmanDriver::remove_container(container_id)
}

fn remove_image(image: &Reference) -> Result<()> {
PodmanDriver::remove_image(image)
}

fn mount_container(container_id: &types::ContainerId) -> Result<types::MountId> {
PodmanDriver::mount_container(container_id)
}
231 changes: 175 additions & 56 deletions process/drivers/docker_driver.rs
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ use std::{
};

use blue_build_utils::{
constants::{BB_BUILDKIT_CACHE_GHA, CONTAINER_FILE, DOCKER_HOST, GITHUB_ACTIONS},
constants::{BB_BUILDKIT_CACHE_GHA, DOCKER_HOST, GITHUB_ACTIONS},
credentials::Credentials,
string_vec,
};
@@ -16,6 +16,7 @@ use colored::Colorize;
use comlexr::cmd;
use log::{debug, info, trace, warn};
use miette::{bail, miette, IntoDiagnostic, Result};
use oci_distribution::Reference;
use once_cell::sync::Lazy;
use semver::Version;
use serde::Deserialize;
@@ -30,8 +31,7 @@ use crate::{
RunOptsVolume, TagOpts,
},
traits::{BuildDriver, DriverVersion, InspectDriver, RunDriver},
types::ImageMetadata,
types::Platform,
types::{ContainerId, ImageMetadata, Platform},
},
logging::CommandLogging,
signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId},
@@ -66,9 +66,13 @@ impl DockerDriver {
}

trace!("docker buildx ls --format={}", "{{.Name}}");
let ls_out = cmd!("docker", "buildx", "ls", "--format={{.Name}}")
.output()
.into_diagnostic()?;
let ls_out = {
let c = cmd!("docker", "buildx", "ls", "--format={{.Name}}");
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !ls_out.status.success() {
bail!("{}", String::from_utf8_lossy(&ls_out.stderr));
@@ -79,15 +83,18 @@ impl DockerDriver {
trace!("{ls_out}");

if !ls_out.lines().any(|line| line == "bluebuild") {
trace!("docker buildx create --bootstrap --driver=docker-container --name=bluebuild");
let create_out = cmd!(
"docker",
"buildx",
"create",
"--bootstrap",
"--driver=docker-container",
"--name=bluebuild",
)
let create_out = {
let c = cmd!(
"docker",
"buildx",
"create",
"--bootstrap",
"--driver=docker-container",
"--name=bluebuild",
);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

@@ -108,9 +115,13 @@ impl DriverVersion for DockerDriver {
const VERSION_REQ: &'static str = ">=23";

fn version() -> Result<Version> {
let output = cmd!("docker", "version", "-f", "json")
.output()
.into_diagnostic()?;
let output = {
let c = cmd!("docker", "version", "-f", "json");
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

let version_json: DockerVersionJson =
serde_json::from_slice(&output.stdout).into_diagnostic()?;
@@ -127,20 +138,23 @@ impl BuildDriver for DockerDriver {
warn!("Squash is deprecated for docker so this build will not squash");
}

trace!("docker build -t {} -f {CONTAINER_FILE} .", opts.image);
let status = cmd!(
"docker",
"build",
if !matches!(opts.platform, Platform::Native) => [
"--platform",
opts.platform.to_string(),
],
"-t",
&*opts.image,
"-f",
&*opts.containerfile,
".",
)
let status = {
let c = cmd!(
"docker",
"build",
if !matches!(opts.platform, Platform::Native) => [
"--platform",
opts.platform.to_string(),
],
"-t",
&*opts.image,
"-f",
&*opts.containerfile,
".",
);
trace!("{c:?}");
c
}
.status()
.into_diagnostic()?;

@@ -157,10 +171,13 @@ impl BuildDriver for DockerDriver {

let dest_image_str = opts.dest_image.to_string();

trace!("docker tag {} {}", opts.src_image, opts.dest_image);
let status = cmd!("docker", "tag", opts.src_image.to_string(), &dest_image_str)
.status()
.into_diagnostic()?;
let status = {
let c = cmd!("docker", "tag", opts.src_image.to_string(), &dest_image_str);
trace!("{c:?}");
c
}
.status()
.into_diagnostic()?;

if status.success() {
info!("Successfully tagged {}!", dest_image_str.bold().green());
@@ -175,10 +192,13 @@ impl BuildDriver for DockerDriver {

let image_str = opts.image.to_string();

trace!("docker push {}", opts.image);
let status = cmd!("docker", "push", &image_str)
.status()
.into_diagnostic()?;
let status = {
let c = cmd!("docker", "push", &image_str);
trace!("{c:?}");
c
}
.status()
.into_diagnostic()?;

if status.success() {
info!("Successfully pushed {}!", image_str.bold().green());
@@ -241,14 +261,18 @@ impl BuildDriver for DockerDriver {
let (system, buildx) = std::thread::scope(
|scope| -> std::thread::Result<(Result<ExitStatus>, Result<ExitStatus>)> {
let system = scope.spawn(|| {
cmd!(
"docker",
"system",
"prune",
"--force",
if opts.all => "--all",
if opts.volumes => "--volumes",
)
{
let c = cmd!(
"docker",
"system",
"prune",
"--force",
if opts.all => "--all",
if opts.volumes => "--volumes",
);
trace!("{c:?}");
c
}
.message_status("docker system prune", "Pruning Docker System")
.into_diagnostic()
});
@@ -260,14 +284,18 @@ impl BuildDriver for DockerDriver {
Self::setup()?;
}

cmd!(
"docker",
"buildx",
"prune",
"--force",
if run_setup => "--builder=bluebuild",
if opts.all => "--all",
)
{
let c = cmd!(
"docker",
"buildx",
"prune",
"--force",
if run_setup => "--builder=bluebuild",
if opts.all => "--all",
);
trace!("{c:?}");
c
}
.message_status("docker buildx prune", "Pruning Docker Buildx")
.into_diagnostic()
});
@@ -465,6 +493,97 @@ impl RunDriver for DockerDriver {

Ok(output)
}

fn create_container(image: &oci_distribution::Reference) -> Result<super::types::ContainerId> {
trace!("DockerDriver::create_container({image})");

let output = {
let c = cmd!("docker", "create", image.to_string(), "bash",);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to create container from image {image}");
}

Ok(ContainerId(
String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?,
))
}

fn remove_container(container_id: &super::types::ContainerId) -> Result<()> {
trace!("DockerDriver::remove_container({container_id})");

let output = {
let c = cmd!("docker", "rm", container_id);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to remove container {container_id}");
}

Ok(())
}

fn remove_image(image: &oci_distribution::Reference) -> Result<()> {
trace!("DockerDriver::remove_image({image})");

let output = {
let c = cmd!("docker", "rmi", image.to_string());
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to remove the image {image}");
}

Ok(())
}

fn list_images() -> Result<Vec<Reference>> {
#[derive(Deserialize)]
#[serde(rename_all = "PascalCase")]
struct Image {
repository: String,
tag: String,
}

let output = {
let c = cmd!("docker", "images", "--format", "json");
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to list images");
}

let images: Vec<Image> = String::from_utf8_lossy(&output.stdout)
.lines()
.map(|line| serde_json::from_str::<Image>(line).into_diagnostic())
.collect::<Result<_>>()?;

images
.into_iter()
.map(|image| {
format!("{}:{}", image.repository, image.tag)
.parse::<Reference>()
.into_diagnostic()
})
.collect()
}
}

fn docker_run(opts: &RunOpts, cid_file: &Path) -> Command {
138 changes: 84 additions & 54 deletions process/drivers/podman_driver.rs
Original file line number Diff line number Diff line change
@@ -28,11 +28,9 @@ use crate::{
signal_handler::{add_cid, remove_cid, ContainerRuntime, ContainerSignalId},
};

use super::types::ContainerId;
#[cfg(feature = "rechunk")]
use super::{
types::{ContainerId, MountId},
ContainerMountDriver, RechunkDriver,
};
use super::{types::MountId, ContainerMountDriver, RechunkDriver};

#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "PascalCase")]
@@ -345,56 +343,6 @@ fn get_metadata_cache(opts: &GetMetadataOpts) -> Result<ImageMetadata> {

#[cfg(feature = "rechunk")]
impl ContainerMountDriver for PodmanDriver {
fn create_container(image: &Reference) -> Result<ContainerId> {
let output = {
let c = cmd!("podman", "create", image.to_string(), "bash");
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to create a container from image {image}");
}

Ok(ContainerId(
String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?,
))
}

fn remove_container(container_id: &super::types::ContainerId) -> Result<()> {
let output = {
let c = cmd!("podman", "rm", container_id);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to remove container {container_id}");
}

Ok(())
}

fn remove_image(image: &Reference) -> Result<()> {
let output = {
let c = cmd!("podman", "rmi", image.to_string());
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to remove the image {image}");
}

Ok(())
}

fn mount_container(container_id: &super::types::ContainerId) -> Result<MountId> {
let output = {
let c = cmd!("podman", "mount", container_id);
@@ -493,6 +441,88 @@ impl RunDriver for PodmanDriver {

Ok(output)
}

fn create_container(image: &Reference) -> Result<ContainerId> {
let output = {
let c = cmd!("podman", "create", image.to_string(), "bash");
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to create a container from image {image}");
}

Ok(ContainerId(
String::from_utf8(output.stdout.trim_ascii().to_vec()).into_diagnostic()?,
))
}

fn remove_container(container_id: &super::types::ContainerId) -> Result<()> {
let output = {
let c = cmd!("podman", "rm", container_id);
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to remove container {container_id}");
}

Ok(())
}

fn remove_image(image: &Reference) -> Result<()> {
let output = {
let c = cmd!("podman", "rmi", image.to_string());
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to remove the image {image}");
}

Ok(())
}

fn list_images() -> Result<Vec<Reference>> {
#[derive(Deserialize)]
#[serde(rename_all = "PascalCase")]
struct Image {
names: Option<Vec<String>>,
}

let output = {
let c = cmd!("podman", "images", "--format", "json");
trace!("{c:?}");
c
}
.output()
.into_diagnostic()?;

if !output.status.success() {
bail!("Failed to list images");
}

let images: Vec<Image> = serde_json::from_slice(&output.stdout).into_diagnostic()?;

images
.into_iter()
.filter_map(|image| image.names)
.flat_map(|names| {
names
.into_iter()
.map(|name| name.parse::<Reference>().into_diagnostic())
})
.collect()
}
}

fn podman_run(opts: &RunOpts, cid_file: &Path) -> Command {
21 changes: 12 additions & 9 deletions process/drivers/traits.rs
Original file line number Diff line number Diff line change
@@ -28,13 +28,10 @@ use super::{
},
podman_driver::PodmanDriver,
skopeo_driver::SkopeoDriver,
types::ImageMetadata,
types::{ContainerId, ImageMetadata},
};
#[cfg(feature = "rechunk")]
use super::{
opts::RechunkOpts,
types::{ContainerId, MountId},
};
use super::{opts::RechunkOpts, types::MountId};

trait PrivateDriver {}

@@ -216,11 +213,7 @@ pub trait RunDriver: PrivateDriver {
/// # Errors
/// Will error if there is an issue running the container.
fn run_output(opts: &RunOpts) -> Result<Output>;
}

#[allow(private_bounds)]
#[cfg(feature = "rechunk")]
pub(super) trait ContainerMountDriver: PrivateDriver {
/// Creates container
///
/// # Errors
@@ -239,6 +232,16 @@ pub(super) trait ContainerMountDriver: PrivateDriver {
/// Will error if the image remove command fails.
fn remove_image(image: &Reference) -> Result<()>;

/// List all images in the local image registry.
///
/// # Errors
/// Will error if the image list command fails.
fn list_images() -> Result<Vec<Reference>>;
}

#[allow(private_bounds)]
#[cfg(feature = "rechunk")]
pub(super) trait ContainerMountDriver: PrivateDriver {
/// Mounts the container
///
/// # Errors
3 changes: 0 additions & 3 deletions process/drivers/types.rs
Original file line number Diff line number Diff line change
@@ -236,17 +236,14 @@ impl ImageMetadata {
}
}

#[cfg(feature = "rechunk")]
pub struct ContainerId(pub(super) String);

#[cfg(feature = "rechunk")]
impl std::fmt::Display for ContainerId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}

#[cfg(feature = "rechunk")]
impl AsRef<std::ffi::OsStr> for ContainerId {
fn as_ref(&self) -> &std::ffi::OsStr {
self.0.as_ref()
8 changes: 4 additions & 4 deletions src/commands/build.rs
Original file line number Diff line number Diff line change
@@ -14,8 +14,8 @@ use blue_build_process_management::{
use blue_build_recipe::Recipe;
use blue_build_utils::{
constants::{
ARCHIVE_SUFFIX, BB_BUILD_RECHUNK, BB_BUILD_RECHUNK_CLEAR_PLAN, BB_REGISTRY_NAMESPACE,
CONFIG_PATH, CONTAINER_FILE, RECIPE_FILE, RECIPE_PATH,
ARCHIVE_SUFFIX, BB_REGISTRY_NAMESPACE, CONFIG_PATH, CONTAINER_FILE, RECIPE_FILE,
RECIPE_PATH,
},
cowstr,
credentials::{Credentials, CredentialsArgs},
@@ -118,15 +118,15 @@ pub struct BuildCommand {
/// and take up more space during build-time.
///
/// NOTE: This must be run as root!
#[arg(long, group = "archive_rechunk", env = BB_BUILD_RECHUNK)]
#[arg(long, group = "archive_rechunk", env = blue_build_utils::constants::BB_BUILD_RECHUNK)]
#[builder(default)]
#[cfg(feature = "rechunk")]
rechunk: bool,

/// Use a fresh rechunk plan, regardless of previous ref.
///
/// NOTE: Only works with `--rechunk`.
#[arg(long, env = BB_BUILD_RECHUNK_CLEAR_PLAN)]
#[arg(long, env = blue_build_utils::constants::BB_BUILD_RECHUNK_CLEAR_PLAN)]
#[builder(default)]
#[cfg(feature = "rechunk")]
rechunk_clear_plan: bool,

0 comments on commit 2e63710

Please sign in to comment.