Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deploy bootstrap validator #12

Closed
wants to merge 14 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
cargo-fmt/clippy. change to docker-build default
gregcusack committed Apr 8, 2024
commit 836d00617e0f817acf10bc80b52acfd91b80f04a
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -53,7 +53,6 @@ cargo run --bin cluster --
--slots-per-epoch <slots-per-epoch>
# docker config
--registry <docker-registry> # e.g. gregcusack
--docker-build
--tag <docker-image-tag> # e.g. v1
--base-image <base-image> # e.g. ubuntu:20.04
--image-name <docker-image-name> # e.g. cluster-image
54 changes: 25 additions & 29 deletions src/docker.rs
Original file line number Diff line number Diff line change
@@ -37,22 +37,21 @@ impl DockerConfig {

pub fn build_image(
&self,
solana_root_path: PathBuf,
solana_root_path: &Path,
validator_type: &ValidatorType,
) -> Result<(), Box<dyn Error>> {
match validator_type {
ValidatorType::Bootstrap => (),
ValidatorType::Standard | ValidatorType::RPC | ValidatorType::Client => {
return Err(format!(
"Build docker image for validator type: {} not supported yet",
validator_type
"Build docker image for validator type: {validator_type} not supported yet"
)
.into());
}
}
let image_name = format!("{}-{}", validator_type, self.image_name);
let docker_path = solana_root_path.join(format!("{}/{}", "docker-build", validator_type));
match self.create_base_image(solana_root_path, image_name, docker_path, validator_type) {
let image_name = format!("{validator_type}-{}", self.image_name);
let docker_path = solana_root_path.join(format!("docker-build/{validator_type}"));
match self.create_base_image(solana_root_path, image_name, &docker_path, validator_type) {
Ok(res) => {
if res.status.success() {
info!("Successfully created base Image");
@@ -68,33 +67,30 @@ impl DockerConfig {

fn create_base_image(
&self,
solana_root_path: PathBuf,
solana_root_path: &Path,
image_name: String,
docker_path: PathBuf,
docker_path: &PathBuf,
validator_type: &ValidatorType,
) -> Result<Output, Box<dyn Error>> {
let dockerfile_path = self.create_dockerfile(validator_type, docker_path, None)?;
self.create_dockerfile(validator_type, docker_path, None)?;

trace!("Tmp: {}", dockerfile_path.as_path().display());
trace!("Exists: {}", dockerfile_path.as_path().exists());
trace!("Tmp: {}", docker_path.as_path().display());
trace!("Exists: {}", docker_path.as_path().exists());

// We use std::process::Command here because Docker-rs is very slow building dockerfiles
// when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively.
// so we result to using std::process::Command
let dockerfile = dockerfile_path.join("Dockerfile");
let dockerfile = docker_path.join("Dockerfile");
let context_path = solana_root_path.display().to_string();

let progress_bar = new_spinner_progress_bar();
progress_bar.set_message(format!(
"{BUILD}Building {} docker image...",
validator_type
));
progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...",));

let command = format!(
"docker build -t {}/{}:{} -f {:?} {}",
self.registry, image_name, self.tag, dockerfile, context_path
"docker build -t {}/{image_name}:{} -f {dockerfile:?} {context_path}",
self.registry, self.tag,
);
info!("command: {}", command);

let output = match Command::new("sh")
.arg("-c")
.arg(&command)
@@ -108,7 +104,7 @@ impl DockerConfig {
Err(err) => Err(Box::new(err) as Box<dyn Error>),
};
progress_bar.finish_and_clear();
info!("{} image build complete", validator_type);
info!("{validator_type} image build complete");

output
}
@@ -120,28 +116,28 @@ impl DockerConfig {
) -> std::io::Result<()> {
let source_path = source_dir.join("src/scripts").join(file_name);
let destination_path = docker_dir.join(file_name);
fs::copy(&source_path, &destination_path)?;
fs::copy(source_path, destination_path)?;
Ok(())
}

fn create_dockerfile(
&self,
validator_type: &ValidatorType,
docker_path: PathBuf,
docker_path: &PathBuf,
content: Option<&str>,
) -> Result<PathBuf, Box<dyn Error>> {
) -> Result<(), Box<dyn Error>> {
if docker_path.exists() {
fs::remove_dir_all(&docker_path)?;
fs::remove_dir_all(docker_path)?;
}
fs::create_dir_all(&docker_path)?;
fs::create_dir_all(docker_path)?;

if let DeployMethod::Local(_) = self.deploy_method {
if validator_type == &ValidatorType::Bootstrap {
let manifest_path =
PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"));
let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"];
for file_name in files_to_copy.iter() {
Self::copy_file_to_docker(&manifest_path, &docker_path, file_name)?;
Self::copy_file_to_docker(&manifest_path, docker_path, file_name)?;
}
}
}
@@ -150,7 +146,7 @@ impl DockerConfig {
if let DeployMethod::ReleaseChannel(_) = self.deploy_method {
("solana-release", "./src/scripts".to_string())
} else {
("farf", format!("./docker-build/{}", validator_type))
("farf", format!("./docker-build/{validator_type}"))
};

let dockerfile = format!(
@@ -184,11 +180,11 @@ WORKDIR /home/solana
self.base_image
);

debug!("dockerfile: {}", dockerfile);
debug!("dockerfile: {dockerfile:?}");
std::fs::write(
docker_path.join("Dockerfile"),
content.unwrap_or(dockerfile.as_str()),
)?;
Ok(docker_path)
Ok(())
}
}
11 changes: 4 additions & 7 deletions src/genesis.rs
Original file line number Diff line number Diff line change
@@ -250,7 +250,7 @@ impl Genesis {
.for_each(|account_type| {
args.push(
self.config_dir
.join(format!("bootstrap-validator/{}.json", account_type))
.join(format!("bootstrap-validator/{account_type}.json"))
.to_string_lossy()
.to_string(),
);
@@ -269,10 +269,7 @@ impl Genesis {
args
}

pub fn setup_spl_args(
&self,
solana_root_path: &PathBuf,
) -> Result<Vec<String>, Box<dyn Error>> {
pub fn setup_spl_args(&self, solana_root_path: &Path) -> Result<Vec<String>, Box<dyn Error>> {
let fetch_spl_file = solana_root_path.join("fetch-spl.sh");
fetch_spl(&fetch_spl_file)?;

@@ -283,8 +280,8 @@ impl Genesis {

pub fn generate(
&mut self,
solana_root_path: &PathBuf,
build_path: &PathBuf,
solana_root_path: &Path,
build_path: &Path,
) -> Result<(), Box<dyn Error>> {
let mut args = self.setup_genesis_flags();
let mut spl_args = self.setup_spl_args(solana_root_path)?;
3 changes: 2 additions & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
@@ -54,6 +54,7 @@ pub enum ValidatorType {
Client,
}

pub mod docker;
pub mod genesis;
pub mod kubernetes;
pub mod release;
@@ -79,7 +80,7 @@ pub fn cat_file(path: &PathBuf) -> std::io::Result<()> {
let mut file = File::open(path)?;
let mut contents = String::new();
file.read_to_string(&mut contents)?;
info!("{:?}:\n{}", path.file_name(), contents);
info!("{:?}:\n{contents}", path.file_name());

Ok(())
}
40 changes: 15 additions & 25 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -77,7 +77,7 @@ fn parse_matches() -> clap::ArgMatches {
Arg::with_name("enable_warmup_epochs")
.long("enable-warmup-epochs")
.takes_value(true)
.possible_values(&["true", "false"])
.possible_values(["true", "false"])
.default_value("true")
.help("Genesis config. enable warmup epoch. defaults to true"),
)
@@ -90,7 +90,7 @@ fn parse_matches() -> clap::ArgMatches {
.arg(
Arg::with_name("cluster_type")
.long("cluster-type")
.possible_values(&["development", "devnet", "testnet", "mainnet-beta"])
.possible_values(["development", "devnet", "testnet", "mainnet-beta"])
.takes_value(true)
.default_value("development")
.help(
@@ -111,13 +111,9 @@ fn parse_matches() -> clap::ArgMatches {
)
//Docker config
.arg(
Arg::with_name("docker_build")
.long("docker-build")
.requires("registry_name")
.requires("image_name")
.requires("base_image")
.requires("image_tag")
.help("Build Docker images. Build new docker images"),
Arg::with_name("skip_docker_build")
.long("skip-docker-build")
.help("Skips build Docker images"),
)
.arg(
Arg::with_name("registry_name")
@@ -131,22 +127,19 @@ fn parse_matches() -> clap::ArgMatches {
.long("image-name")
.takes_value(true)
.default_value("k8s-cluster-image")
.required(true)
.help("Docker image name. Will be prepended with validator_type (bootstrap or validator)"),
)
.arg(
Arg::with_name("base_image")
.long("base-image")
.takes_value(true)
.default_value("ubuntu:20.04")
.required(true)
.help("Docker base image"),
)
.arg(
Arg::with_name("image_tag")
.long("tag")
.takes_value(true)
.required(true)
.default_value("latest")
.help("Docker image tag."),
)
@@ -225,12 +218,9 @@ async fn main() {
let build_config = BuildConfig::new(
deploy_method.clone(),
build_type,
&solana_root.get_root_path(),
matches.is_present("docker_build"),
)
.unwrap_or_else(|err| {
panic!("Error creating BuildConfig: {}", err);
});
solana_root.get_root_path(),
!matches.is_present("skip_docker_build"),
);

let genesis_flags = GenesisFlags {
hashes_per_tick: matches
@@ -285,22 +275,22 @@ async fn main() {
match build_config.prepare().await {
Ok(_) => info!("Validator setup prepared successfully"),
Err(err) => {
error!("Error: {}", err);
error!("Error: {err}");
return;
}
}

let mut genesis = Genesis::new(solana_root.get_root_path(), genesis_flags);
match genesis.generate_faucet() {
Ok(_) => (),
Ok(_) => info!("Generated faucet account"),
Err(err) => {
error!("generate faucet error! {err}");
return;
}
}

match genesis.generate_accounts(ValidatorType::Bootstrap, 1) {
Ok(_) => (),
Ok(_) => info!("Generated bootstrap account"),
Err(err) => {
error!("generate accounts error! {err}");
return;
@@ -309,9 +299,9 @@ async fn main() {

// creates genesis and writes to binary file
match genesis.generate(solana_root.get_root_path(), &build_path) {
Ok(_) => (),
Ok(_) => info!("Created genesis successfully"),
Err(err) => {
error!("generate genesis error! {}", err);
error!("generate genesis error! {err}");
return;
}
}
@@ -334,9 +324,9 @@ async fn main() {
if build_config.docker_build() {
let image_type = ValidatorType::Bootstrap;
match docker.build_image(solana_root.get_root_path(), &image_type) {
Ok(_) => info!("{} image built successfully", image_type),
Ok(_) => info!("{image_type} image built successfully"),
Err(err) => {
error!("Exiting........ {}", err);
error!("Error. Failed to build imge: {err}");
return;
}
}
30 changes: 14 additions & 16 deletions src/release.rs
Original file line number Diff line number Diff line change
@@ -30,27 +30,28 @@ pub struct BuildConfig {
build_type: BuildType,
solana_root_path: PathBuf,
docker_build: bool,
build_path: PathBuf,
}

impl BuildConfig {
pub fn new(
deploy_method: DeployMethod,
build_type: BuildType,
solana_root_path: &PathBuf,
solana_root_path: &Path,
docker_build: bool,
) -> Result<Self, Box<dyn std::error::Error>> {
) -> Self {
let build_path = match deploy_method {
DeployMethod::Local(_) => solana_root_path.join("farf/bin"),
DeployMethod::ReleaseChannel(_) => solana_root_path.join("solana-release/bin"),
};

Ok(BuildConfig {
BuildConfig {
deploy_method,
build_type,
build_path,
solana_root_path: solana_root_path.clone(),
solana_root_path: solana_root_path.to_path_buf(),
docker_build,
})
build_path,
}
}

pub fn build_path(&self) -> PathBuf {
@@ -81,7 +82,7 @@ impl BuildConfig {
async fn setup_tar_deploy(&self, release_channel: &String) -> Result<PathBuf, Box<dyn Error>> {
let file_name = "solana-release";
let tar_filename = format!("{file_name}.tar.bz2");
info!("tar file: {}", tar_filename);
info!("tar file: {tar_filename}");
self.download_release_from_channel(&tar_filename, release_channel)
.await?;

@@ -145,14 +146,14 @@ impl BuildConfig {
let tag_object = solana_repo.revparse_single(tag)?.id();
// Check if the commit associated with the tag is the same as the current commit
if tag_object == commit {
info!("The current commit is associated with tag: {}", tag);
info!("The current commit is associated with tag: {tag}");
note = tag_object.to_string();
break;
}
}

// Write to branch/tag and commit to version.yml
let content = format!("channel: devbuild {}\ncommit: {}", note, commit);
let content = format!("channel: devbuild {note}\ncommit: {commit}");
std::fs::write(self.solana_root_path.join("farf/version.yml"), content)
.expect("Failed to write version.yml");

@@ -165,22 +166,19 @@ impl BuildConfig {
tar_filename: &str,
release_channel: &String,
) -> Result<(), Box<dyn Error>> {
info!("Downloading release from channel: {}", release_channel);
info!("Downloading release from channel: {release_channel}");
let file_path = self.solana_root_path.join(tar_filename);
// Remove file
if let Err(err) = fs::remove_file(&file_path) {
if err.kind() != std::io::ErrorKind::NotFound {
return Err(format!("{}: {:?}", "Error while removing file:", err).into());
return Err(format!("{err}: {:?}", "Error while removing file:").into());
}
}

let download_url = format!(
"{}{}{}",
"https://release.solana.com/",
release_channel,
"/solana-release-x86_64-unknown-linux-gnu.tar.bz2"
"https://release.solana.com/{release_channel}/solana-release-x86_64-unknown-linux-gnu.tar.bz2"
);
info!("download_url: {}", download_url);
info!("download_url: {download_url}");

download_to_temp(
download_url.as_str(),