add and deploy bootstrap secrets (#8)

* add create and deploy bootstrap secrets

* finish up rest of nits

* fix nit and remove dependency on CARGO_MANIFEST_PATH

PROGRESS.md

@@ -19,7 +19,7 @@
     - [x] Build Bootstrap Image
     - [x] Push Image to registry
 - [ ] Create & Deploy Secrets
-    - [ ] Bootstrap
+    - [x] Bootstrap
     - [ ] Validator (regular)
     - [ ] RPC nodes
     - [ ] Client

README.md

@@ -27,13 +27,15 @@ kubectl create ns <namespace>
 cargo run --bin cluster --
     -n <namespace>
     --local-path <path-to-local-agave-monorepo>
+    --validator-lab-dir <path-to-validator-lab-directory>
 ```
 
 #### Build specific Agave release
 ```
 cargo run --bin cluster --
     -n <namespace>
-    --release-channel <agave-version: e.g. v1.17.28> # note: MUST include the "v" 
+    --release-channel <agave-version: e.g. v1.17.28> # note: MUST include the "v"
+    --validator-lab-dir <path-to-validator-lab-directory>
 ```
 
 #### Build from Local Repo and Configure Genesis and Bootstrap Validator Image
@@ -42,6 +44,7 @@ Example:
 cargo run --bin cluster -- 
     -n <namespace> 
     --local-path /home/sol/solana
+    --validator-lab-dir /home/sol/validator-lab
     # genesis config. Optional: Many of these have defaults
     --hashes-per-tick <hashes-per-tick>
     --enable-warmup-epochs <true|false>

src/docker.rs

@@ -2,7 +2,6 @@ use {
     crate::{new_spinner_progress_bar, release::DeployMethod, ValidatorType, BUILD, ROCKET},
     log::*,
     std::{
-        env,
         error::Error,
         fmt::{self, Display, Formatter},
         fs,
@@ -66,6 +65,7 @@ impl DockerConfig {
     pub fn build_image(
         &self,
         solana_root_path: &Path,
+        lab_path: &Path,
         docker_image: &DockerImage,
     ) -> Result<(), Box<dyn Error>> {
         let validator_type = docker_image.validator_type();
@@ -82,6 +82,7 @@ impl DockerConfig {
         let docker_path = solana_root_path.join(format!("docker-build/{validator_type}"));
         self.create_base_image(
             solana_root_path,
+            lab_path,
             docker_image,
             &docker_path,
             &validator_type,
@@ -93,11 +94,12 @@ impl DockerConfig {
     fn create_base_image(
         &self,
         solana_root_path: &Path,
+        lab_path: &Path,
         docker_image: &DockerImage,
         docker_path: &PathBuf,
         validator_type: &ValidatorType,
     ) -> Result<(), Box<dyn Error>> {
-        self.create_dockerfile(validator_type, docker_path, None)?;
+        self.create_dockerfile(validator_type, docker_path, lab_path, None)?;
 
         // We use std::process::Command here because Docker-rs is very slow building dockerfiles
         // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively.
@@ -108,7 +110,10 @@ impl DockerConfig {
         let progress_bar = new_spinner_progress_bar();
         progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...",));
 
-        let command = format!("docker build -t {docker_image} -f {dockerfile:?} {context_path}");
+        let command = format!(
+            "docker build -t {docker_image} -f {} {context_path}",
+            dockerfile.display()
+        );
 
         let output = match Command::new("sh")
             .arg("-c")
@@ -147,6 +152,7 @@ impl DockerConfig {
         &self,
         validator_type: &ValidatorType,
         docker_path: &PathBuf,
+        lab_path: &Path,
         content: Option<&str>,
     ) -> Result<(), Box<dyn Error>> {
         if docker_path.exists() {
@@ -156,11 +162,9 @@ impl DockerConfig {
 
         if let DeployMethod::Local(_) = self.deploy_method {
             if validator_type == &ValidatorType::Bootstrap {
-                let manifest_path =
-                    PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"));
                 let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"];
                 for file_name in files_to_copy.iter() {
-                    Self::copy_file_to_docker(&manifest_path, docker_path, file_name)?;
+                    Self::copy_file_to_docker(lab_path, docker_path, file_name)?;
                 }
             }
         }

src/genesis.rs

@@ -99,8 +99,7 @@ pub struct Genesis {
 }
 
 impl Genesis {
-    pub fn new(solana_root: &Path, flags: GenesisFlags) -> Self {
-        let config_dir = solana_root.join("config-k8s");
+    pub fn new(config_dir: PathBuf, flags: GenesisFlags) -> Self {
         if config_dir.exists() {
             std::fs::remove_dir_all(&config_dir).unwrap();
         }

src/k8s_helpers.rs

@@ -0,0 +1,30 @@
+use {
+    k8s_openapi::{api::core::v1::Secret, ByteString},
+    kube::api::ObjectMeta,
+    std::{collections::BTreeMap, error::Error, path::PathBuf},
+};
+
+fn create_secret(name: &str, data: BTreeMap<String, ByteString>) -> Secret {
+    Secret {
+        metadata: ObjectMeta {
+            name: Some(name.to_string()),
+            ..Default::default()
+        },
+        data: Some(data),
+        ..Default::default()
+    }
+}
+
+pub fn create_secret_from_files(
+    secret_name: &str,
+    key_files: &[(PathBuf, &str)], //[pathbuf, key type]
+) -> Result<Secret, Box<dyn Error>> {
+    let mut data = BTreeMap::new();
+    for (file_path, key_type) in key_files {
+        let file_content = std::fs::read(file_path)
+            .map_err(|err| format!("Failed to read file '{:?}': {}", file_path, err))?;
+        data.insert(format!("{}.json", key_type), ByteString(file_content));
+    }
+
+    Ok(create_secret(secret_name, data))
+}

src/kubernetes.rs

@@ -1,9 +1,11 @@
 use {
-    k8s_openapi::api::core::v1::Namespace,
+    crate::k8s_helpers,
+    k8s_openapi::api::core::v1::{Namespace, Secret},
     kube::{
-        api::{Api, ListParams},
+        api::{Api, ListParams, PostParams},
         Client,
     },
+    std::{error::Error, path::Path},
 };
 
 pub struct Kubernetes {
@@ -30,4 +32,30 @@ impl Kubernetes {
 
         Ok(exists)
     }
+
+    pub fn create_bootstrap_secret(
+        &self,
+        secret_name: &str,
+        config_dir: &Path,
+    ) -> Result<Secret, Box<dyn Error>> {
+        let faucet_key_path = config_dir.join("faucet.json");
+        let identity_key_path = config_dir.join("bootstrap-validator/identity.json");
+        let vote_key_path = config_dir.join("bootstrap-validator/vote-account.json");
+        let stake_key_path = config_dir.join("bootstrap-validator/stake-account.json");
+
+        let key_files = vec![
+            (faucet_key_path, "faucet"),
+            (identity_key_path, "identity"),
+            (vote_key_path, "vote"),
+            (stake_key_path, "stake"),
+        ];
+
+        k8s_helpers::create_secret_from_files(secret_name, &key_files)
+    }
+
+    pub async fn deploy_secret(&self, secret: &Secret) -> Result<Secret, kube::Error> {
+        let secrets_api: Api<Secret> =
+            Api::namespaced(self.k8s_client.clone(), self.namespace.as_str());
+        secrets_api.create(&PostParams::default(), secret).await
+    }
 }

src/lib.rs

@@ -5,7 +5,6 @@ use {
     log::*,
     reqwest::Client,
     std::{
-        env,
         fs::File,
         io::{BufReader, Cursor, Read, Write},
         path::{Path, PathBuf},
@@ -18,22 +17,16 @@ use {
 
 const UPGRADEABLE_LOADER: &str = "BPFLoaderUpgradeab1e11111111111111111111111";
 
-pub fn get_solana_root() -> PathBuf {
-    PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR")).to_path_buf()
+#[derive(Clone, Debug)]
+pub struct EnvironmentConfig<'a> {
+    pub namespace: &'a str,
+    pub lab_path: PathBuf, // path to the validator-lab directory
 }
 
 pub struct SolanaRoot {
     root_path: PathBuf,
 }
 
-impl Default for SolanaRoot {
-    fn default() -> Self {
-        Self {
-            root_path: get_solana_root(),
-        }
-    }
-}
-
 impl SolanaRoot {
     pub fn new_from_path(path: PathBuf) -> Self {
         Self { root_path: path }
@@ -58,6 +51,7 @@ pub enum ValidatorType {
 
 pub mod docker;
 pub mod genesis;
+pub mod k8s_helpers;
 pub mod kubernetes;
 pub mod release;
 

src/main.rs

@@ -11,7 +11,7 @@ use {
         },
         kubernetes::Kubernetes,
         release::{BuildConfig, BuildType, DeployMethod},
-        SolanaRoot, ValidatorType,
+        EnvironmentConfig, SolanaRoot, ValidatorType,
     },
 };
 
@@ -50,6 +50,14 @@ fn parse_matches() -> clap::ArgMatches {
                 .args(&["local_path", "release_channel"])
                 .required(true),
         )
+        .arg(
+            Arg::with_name("validator_lab_directory")
+                .long("validator-lab-dir")
+                .takes_value(true)
+                .required(true)
+                .help("Absolute path to validator lab directory. 
+                e.g. /home/sol/validator-lab"),
+        )
         // Genesis Config
         .arg(
             Arg::with_name("hashes_per_tick")
@@ -151,11 +159,6 @@ fn parse_matches() -> clap::ArgMatches {
         .get_matches()
 }
 
-#[derive(Clone, Debug)]
-pub struct EnvironmentConfig<'a> {
-    pub namespace: &'a str,
-}
-
 #[tokio::main]
 async fn main() {
     if std::env::var("RUST_LOG").is_err() {
@@ -165,6 +168,7 @@ async fn main() {
     let matches = parse_matches();
     let environment_config = EnvironmentConfig {
         namespace: matches.value_of("cluster_namespace").unwrap_or_default(),
+        lab_path: matches.value_of("validator_lab_directory").unwrap().into(),
     };
 
     let deploy_method = if let Some(local_path) = matches.value_of("local_path") {
@@ -182,7 +186,7 @@ async fn main() {
             (root, path)
         }
         DeployMethod::ReleaseChannel(_) => {
-            let root = SolanaRoot::default();
+            let root = SolanaRoot::new_from_path(environment_config.lab_path.clone());
             let path = root.get_root_path().join("solana-release/bin");
             (root, path)
         }
@@ -193,14 +197,14 @@ async fn main() {
     if let Ok(metadata) = fs::metadata(solana_root.get_root_path()) {
         if !metadata.is_dir() {
             return error!(
-                "Build path is not a directory: {:?}",
-                solana_root.get_root_path()
+                "Build path is not a directory: {}",
+                solana_root.get_root_path().display()
             );
         }
     } else {
         return error!(
-            "Build directory not found: {:?}",
-            solana_root.get_root_path()
+            "Build directory not found: {}",
+            solana_root.get_root_path().display()
         );
     }
 
@@ -285,7 +289,9 @@ async fn main() {
         }
     }
 
-    let mut genesis = Genesis::new(solana_root.get_root_path(), genesis_flags);
+    let config_directory = solana_root.get_root_path().join("config-k8s");
+    let mut genesis = Genesis::new(config_directory.clone(), genesis_flags);
+
     match genesis.generate_faucet() {
         Ok(_) => info!("Generated faucet account"),
         Err(err) => {
@@ -316,10 +322,7 @@ async fn main() {
 
     //unwraps are safe here. since their requirement is enforced by argmatches
     let docker = DockerConfig::new(
-        matches
-            .value_of("base_image")
-            .unwrap_or_default()
-            .to_string(),
+        matches.value_of("base_image").unwrap().to_string(),
         deploy_method,
     );
 
@@ -328,14 +331,15 @@ async fn main() {
         matches.value_of("registry_name").unwrap().to_string(),
         validator_type,
         matches.value_of("image_name").unwrap().to_string(),
-        matches
-            .value_of("image_tag")
-            .unwrap_or_default()
-            .to_string(),
+        matches.value_of("image_tag").unwrap().to_string(),
     );
 
     if build_config.docker_build() {
-        match docker.build_image(solana_root.get_root_path(), &docker_image) {
+        match docker.build_image(
+            solana_root.get_root_path(),
+            &environment_config.lab_path,
+            &docker_image,
+        ) {
             Ok(_) => info!("{} image built successfully", docker_image.validator_type()),
             Err(err) => {
                 error!("Exiting........ {err}");
@@ -354,4 +358,22 @@ async fn main() {
             }
         }
     }
+
+    let bootstrap_secret = match kub_controller
+        .create_bootstrap_secret("bootstrap-accounts-secret", &config_directory)
+    {
+        Ok(secret) => secret,
+        Err(err) => {
+            error!("Failed to create bootstrap secret! {}", err);
+            return;
+        }
+    };
+
+    match kub_controller.deploy_secret(&bootstrap_secret).await {
+        Ok(_) => info!("Deployed Bootstrap Secret"),
+        Err(err) => {
+            error!("{}", err);
+            return;
+        }
+    }
 }