From 67d27600794dbcf582fd9de171e45db3c365251d Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Mon, 25 Nov 2024 14:17:28 -0300 Subject: [PATCH 1/7] chore: Scale using snapshots --- Cargo.lock | 789 +++++++++++++++++++++++-- crates/operator/Cargo.toml | 2 + crates/operator/src/bin/operator.rs | 10 +- crates/operator/src/config.rs | 2 + crates/operator/src/controller.rs | 82 ++- crates/operator/src/custom_resource.rs | 69 ++- 6 files changed, 882 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7e88c1..a720fa7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -209,6 +209,380 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d3a2854c7490b4c63d2b0e8c3976d628c80afa3045d078a715b2edb2ee4e0a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper 0.14.31", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.1.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.7.7" @@ -282,6 +656,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" @@ -306,6 +686,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -438,6 +828,16 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.1.37" @@ -580,6 +980,24 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -592,6 +1010,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -661,6 +1091,16 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.9" @@ -754,18 +1194,30 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.9", "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -795,21 +1247,41 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest", - "ff", + "ff 0.13.0", "generic-array", - "group", - "pkcs8", + "group 0.13.0", + "pkcs8 0.10.2", "rand_core", - "sec1", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -886,6 +1358,16 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "ff" version = "0.13.0" @@ -931,6 +1413,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1091,13 +1579,24 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.0", "rand_core", "subtle", ] @@ -1177,6 +1676,11 @@ name = "hashbrown" version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "headers" @@ -1329,6 +1833,8 @@ name = "hydra-control-plane-operator" version = "0.1.0" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-s3", "futures", "k8s-openapi", "kube", @@ -1435,15 +1941,31 @@ dependencies = [ "headers", "http 1.1.0", "hyper 1.5.0", - "hyper-rustls", + "hyper-rustls 0.27.3", "hyper-util", "pin-project-lite", "rustls-native-certs 0.7.3", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.31", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.3" @@ -1455,11 +1977,11 @@ dependencies = [ "hyper 1.5.0", "hyper-util", "log", - "rustls", + "rustls 0.23.16", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", ] @@ -1826,11 +2348,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "once_cell", "sha2", - "signature", + "signature 2.2.0", ] [[package]] @@ -1876,15 +2398,15 @@ dependencies = [ "http-body-util", "hyper 1.5.0", "hyper-http-proxy", - "hyper-rustls", + "hyper-rustls 0.27.3", "hyper-timeout", "hyper-util", "jsonpath-rust", "k8s-openapi", "kube-core", "pem", - "rustls", - "rustls-pemfile", + "rustls 0.23.16", + "rustls-pemfile 2.2.0", "secrecy", "serde", "serde_json", @@ -2011,6 +2533,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.1", +] + [[package]] name = "matchers" version = "0.1.0" @@ -2026,6 +2557,16 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2309,12 +2850,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2", +] + [[package]] name = "pallas" version = "0.31.0" @@ -2796,14 +3354,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", +] + [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -3057,6 +3625,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -3085,7 +3659,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.5.0", - "hyper-rustls", + "hyper-rustls 0.27.3", "hyper-tls", "hyper-util", "ipnet", @@ -3097,7 +3671,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "serde", "serde_json", "serde_urlencoded", @@ -3113,6 +3687,17 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -3253,6 +3838,15 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.40" @@ -3266,6 +3860,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.16" @@ -3276,11 +3882,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + [[package]] name = "rustls-native-certs" version = "0.7.3" @@ -3288,7 +3906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", @@ -3301,12 +3919,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -3322,6 +3949,16 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -3390,16 +4027,40 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.9", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -3454,6 +4115,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.215" @@ -3617,6 +4284,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "signature" version = "2.2.0" @@ -3658,6 +4335,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -3665,7 +4352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.9", ] [[package]] @@ -3939,13 +4626,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.16", "rustls-pki-types", "tokio", ] @@ -4334,6 +5031,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -4374,6 +5077,12 @@ dependencies = [ "tonic", ] +[[package]] +name = "uuid" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" + [[package]] name = "valuable" version = "0.1.0" @@ -4392,6 +5101,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.1" @@ -4714,6 +5429,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yansi" version = "1.0.1" diff --git a/crates/operator/Cargo.toml b/crates/operator/Cargo.toml index 938f367..0de19df 100644 --- a/crates/operator/Cargo.toml +++ b/crates/operator/Cargo.toml @@ -21,3 +21,5 @@ tracing-subscriber = "0.3.18" reqwest = "0.12.9" prometheus-parse = "0.2.5" rand = "0.8.5" +aws-sdk-s3 = "1.62.0" +aws-config = "1.5.10" diff --git a/crates/operator/src/bin/operator.rs b/crates/operator/src/bin/operator.rs index 4ac729c..6a1a890 100644 --- a/crates/operator/src/bin/operator.rs +++ b/crates/operator/src/bin/operator.rs @@ -1,4 +1,6 @@ use anyhow::Result; +use aws_config::{meta::region::RegionProviderChain, BehaviorVersion}; +use aws_sdk_s3::config::Region; use futures::StreamExt; use kube::{runtime::controller::Controller, Api, Client}; use std::sync::Arc; @@ -20,7 +22,13 @@ async fn main() -> Result<()> { info!("Initiating operator."); let client = Client::try_default().await?; let config = Config::from_env(); - let context = Arc::new(K8sContext::new(client.clone(), config)); + let region_provider = RegionProviderChain::first_try(Region::new(config.bucket_region.clone())); + let shared_config = aws_config::defaults(BehaviorVersion::latest()) + .region(region_provider) + .load() + .await; + let s3_client = aws_sdk_s3::Client::new(&shared_config); + let context = Arc::new(K8sContext::new(client.clone(), config, s3_client)); // Create controller for MyApp custom resource let api: Api = Api::default_namespaced(client); diff --git a/crates/operator/src/config.rs b/crates/operator/src/config.rs index 089e8b5..c4c1f54 100644 --- a/crates/operator/src/config.rs +++ b/crates/operator/src/config.rs @@ -29,6 +29,7 @@ pub struct Config { pub dmtr_api_key: String, pub dmtr_port_name: String, pub bucket: String, + pub bucket_region: String, pub init_aws_access_key_id: String, pub init_aws_secret_access_key: String, // Autoscaler @@ -62,6 +63,7 @@ impl Config { dmtr_port_name: env::var("DMTR_PORT_NAME").expect("Missing DMTR_PORT_NAME env var."), init_image: env::var("INIT_IMAGE").expect("Missing INIT_IMAGE env var."), bucket: env::var("BUCKET").expect("Missing BUCKET env var."), + bucket_region: env::var("BUCKET_REGION").expect("Missing BUCKET_REGION env var."), init_aws_access_key_id: env::var("INIT_AWS_ACCESS_KEY_ID") .expect("Missing INIT_AWS_ACCESS_KEY_ID env var."), init_aws_secret_access_key: env::var("INIT_AWS_SECRET_ACCESS_KEY") diff --git a/crates/operator/src/controller.rs b/crates/operator/src/controller.rs index 968bf2b..36ab51b 100644 --- a/crates/operator/src/controller.rs +++ b/crates/operator/src/controller.rs @@ -111,10 +111,14 @@ pub struct K8sConstants { pub dmtrctl_image: String, pub storage_class_name: String, pub service_account_name: String, + pub available_snapshot_prefix: String, + pub used_snapshot_prefix: String, } impl Default for K8sConstants { fn default() -> Self { Self { + available_snapshot_prefix: "snapshots".to_string(), + used_snapshot_prefix: "used".to_string(), storage_class_name: "efs-sc".to_string(), config_dir: "/etc/config".to_string(), secret_dir: "/var/secret".to_string(), @@ -166,14 +170,16 @@ pub struct K8sContext { pub client: Client, pub config: Config, pub constants: K8sConstants, + pub s3_client: aws_sdk_s3::Client, } impl K8sContext { - pub fn new(client: Client, config: Config) -> Self { + pub fn new(client: Client, config: Config, s3_client: aws_sdk_s3::Client) -> Self { Self { client, config, constants: Default::default(), + s3_client, } } @@ -402,22 +408,86 @@ impl K8sContext { Ok(()) } + async fn get_snapshot(&self) -> Option { + let mut response = self + .s3_client + .list_objects_v2() + .bucket(self.config.bucket.clone()) + .prefix(self.constants.available_snapshot_prefix.clone()) + .max_keys(10) + .into_paginator() + .send(); + match response.next().await { + Some(result) => match result { + Ok(output) => match output.contents().first() { + Some(object) => object.key().map(|key| key.to_string()), + None => None, + }, + Err(e) => { + warn!(error = e.to_string(), "Error query s3 for snapshots."); + None + } + }, + None => None, + } + } + + async fn use_snapshot(&self, snapshot_key: &str) -> anyhow::Result { + let new_key = snapshot_key.replace( + &self.constants.available_snapshot_prefix, + &self.constants.used_snapshot_prefix, + ); + match self + .s3_client + .copy_object() + .copy_source(format!("{}/{}", self.config.bucket, snapshot_key)) + .bucket(self.config.bucket.clone()) + .key(new_key.clone()) + .send() + .await + { + Ok(_) => Ok(new_key), + Err(_) => bail!("Failed to move snapshot to use."), + } + } + pub async fn deploy_node(&self) -> anyhow::Result { // List available snapshots. - // Try move from available to used dir. - // If successful, start new node. - // If anything fails, at any point, deploy offline node. + let spec = match self.get_snapshot().await { + Some(snapshot_key) => { + // Try move from available to used dir. + match self.use_snapshot(&snapshot_key).await { + Ok(new_snapshot_key) => HydraDoomNodeSpec { + offline: Some(false), + asleep: None, + network_id: None, + start_chain_from: None, + resources: None, + snapshot: Some(new_snapshot_key), + }, + Err(e) => { + warn!(err = e.to_string(), "Failed to mark snapshot as used"); + HydraDoomNodeSpec::default() + } + } + } + None => HydraDoomNodeSpec::default(), + }; let api: Api = Api::default_namespaced(self.client.clone()); let name = format!( "{}{}{}", self.config.autoscaler_region_prefix, - "0", // 1 for online, 0 for offline + if spec.offline.unwrap_or(false) { + "0" + } else { + "1" + }, // 1 for online, 0 for offline random_name().to_lowercase() ); info!("Deploying new node: {}", name); let new_node = HydraDoomNode { - spec: HydraDoomNodeSpec::default(), + spec, status: None, metadata: kube::api::ObjectMeta { name: Some(name.clone()), diff --git a/crates/operator/src/custom_resource.rs b/crates/operator/src/custom_resource.rs index c4d738f..f1c6a75 100644 --- a/crates/operator/src/custom_resource.rs +++ b/crates/operator/src/custom_resource.rs @@ -87,8 +87,7 @@ impl From for ResourceRequirements { pub struct HydraDoomNodeSpec { pub offline: Option, pub network_id: Option, - pub seed_input: String, - pub commit_inputs: Vec, + pub snapshot: Option, pub start_chain_from: Option, pub asleep: Option, pub resources: Option, @@ -99,8 +98,7 @@ impl Default for HydraDoomNodeSpec { Self { offline: Some(true), network_id: None, - seed_input: "_".to_string(), - commit_inputs: vec![], + snapshot: None, start_chain_from: None, asleep: None, resources: None, @@ -360,6 +358,41 @@ impl HydraDoomNode { }) } + let mut init_container_env_vars = vec![ + EnvVar { + name: "BUCKET".to_string(), + value: Some(config.bucket.clone()), + ..Default::default() + }, + EnvVar { + name: "DATA_DIR".to_string(), + value: Some(constants.data_dir.clone()), + ..Default::default() + }, + EnvVar { + name: "AWS_REGION".to_string(), + value: Some(config.bucket_region.clone()), + ..Default::default() + }, + EnvVar { + name: "AWS_ACCESS_KEY_ID".to_string(), + value: Some(config.init_aws_access_key_id.clone()), + ..Default::default() + }, + EnvVar { + name: "AWS_SECRET_ACCESS_KEY".to_string(), + value: Some(config.init_aws_secret_access_key.clone()), + ..Default::default() + }, + ]; + if let Some(key) = &self.spec.snapshot { + init_container_env_vars.push(EnvVar { + name: "KEY".to_string(), + value: Some(key.clone()), + ..Default::default() + }); + } + Deployment { metadata: ObjectMeta { name: Some(name.clone()), @@ -386,33 +419,7 @@ impl HydraDoomNode { init_containers: Some(vec![Container { name: "init".to_string(), image: Some(config.init_image.clone()), - env: Some(vec![ - EnvVar { - name: "BUCKET".to_string(), - value: Some(config.bucket.clone()), - ..Default::default() - }, - EnvVar { - name: "KEY".to_string(), - value: Some(format!("{}.tar.gz", self.name_any())), - ..Default::default() - }, - EnvVar { - name: "DATA_DIR".to_string(), - value: Some(constants.data_dir.clone()), - ..Default::default() - }, - EnvVar { - name: "AWS_ACCESS_KEY_ID".to_string(), - value: Some(config.init_aws_access_key_id.clone()), - ..Default::default() - }, - EnvVar { - name: "AWS_SECRET_ACCESS_KEY".to_string(), - value: Some(config.init_aws_secret_access_key.clone()), - ..Default::default() - }, - ]), + env: Some(init_container_env_vars), volume_mounts: Some(vec![VolumeMount { name: "data".to_string(), mount_path: constants.data_dir.clone(), From b31d53fc2169fca1a0a15f26607252d8035b8c8b Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Mon, 25 Nov 2024 14:19:06 -0300 Subject: [PATCH 2/7] Add bucket region to bootstra --- bootstrap/stage2/deployment.tf | 5 +++++ bootstrap/stage2/main.tf | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/bootstrap/stage2/deployment.tf b/bootstrap/stage2/deployment.tf index 5e0a374..b197e3f 100644 --- a/bootstrap/stage2/deployment.tf +++ b/bootstrap/stage2/deployment.tf @@ -136,6 +136,11 @@ resource "kubernetes_deployment_v1" "operator" { value = var.bucket } + env { + name = "BUCKET_REGION" + value = var.bucket_region + } + env { name = "INIT_AWS_ACCESS_KEY_ID" value = var.init_aws_access_key_id diff --git a/bootstrap/stage2/main.tf b/bootstrap/stage2/main.tf index cd732f5..ce79b2f 100644 --- a/bootstrap/stage2/main.tf +++ b/bootstrap/stage2/main.tf @@ -118,6 +118,11 @@ variable "bucket" { default = "hydradoomsnapshots" } +variable "bucket_region" { + type = string + default = "us-east-1" +} + variable "init_aws_access_key_id" { type = string } From b8cea5bdf3ddfb3849467414eeac6eb79d6e5453 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Mon, 25 Nov 2024 14:23:54 -0300 Subject: [PATCH 3/7] Delete snapshot after use --- crates/operator/src/controller.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/operator/src/controller.rs b/crates/operator/src/controller.rs index 36ab51b..3ab387c 100644 --- a/crates/operator/src/controller.rs +++ b/crates/operator/src/controller.rs @@ -437,18 +437,27 @@ impl K8sContext { &self.constants.available_snapshot_prefix, &self.constants.used_snapshot_prefix, ); - match self + + // TODO: This should be ACID to avoid race conditions between regions. + // Copy object to used snapshots. + let _ = self .s3_client .copy_object() .copy_source(format!("{}/{}", self.config.bucket, snapshot_key)) .bucket(self.config.bucket.clone()) .key(new_key.clone()) .send() - .await - { - Ok(_) => Ok(new_key), - Err(_) => bail!("Failed to move snapshot to use."), - } + .await?; + + // Remove object from available snapshots. + self.s3_client + .delete_object() + .bucket(self.config.bucket.clone()) + .key(format!("{}/{}", self.config.bucket, snapshot_key)) + .send() + .await?; + + Ok(new_key) } pub async fn deploy_node(&self) -> anyhow::Result { From 359ae37a27c4dbb8ad2c9931f3dbbd935de971e8 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Mon, 25 Nov 2024 15:03:22 -0300 Subject: [PATCH 4/7] Choose random key to decrease possibilty of colisions --- crates/operator/src/controller.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/operator/src/controller.rs b/crates/operator/src/controller.rs index 3ab387c..1e176d7 100644 --- a/crates/operator/src/controller.rs +++ b/crates/operator/src/controller.rs @@ -9,7 +9,7 @@ use kube::{ runtime::controller::Action, Api, Client, ResourceExt, }; -use rand::{distributions::Alphanumeric, Rng}; +use rand::{distributions::Alphanumeric, seq::SliceRandom, thread_rng, Rng}; use serde_json::json; use std::{ cmp::{min, Ordering}, @@ -414,12 +414,12 @@ impl K8sContext { .list_objects_v2() .bucket(self.config.bucket.clone()) .prefix(self.constants.available_snapshot_prefix.clone()) - .max_keys(10) + .max_keys(100) .into_paginator() .send(); match response.next().await { Some(result) => match result { - Ok(output) => match output.contents().first() { + Ok(output) => match output.contents().choose(&mut thread_rng()) { Some(object) => object.key().map(|key| key.to_string()), None => None, }, @@ -438,7 +438,6 @@ impl K8sContext { &self.constants.used_snapshot_prefix, ); - // TODO: This should be ACID to avoid race conditions between regions. // Copy object to used snapshots. let _ = self .s3_client @@ -449,7 +448,8 @@ impl K8sContext { .send() .await?; - // Remove object from available snapshots. + // Remove object from available snapshots. If failed, then the snapshot was stolen by + // another agent. Return Err in that case, and continue with offline node. self.s3_client .delete_object() .bucket(self.config.bucket.clone()) From d1f38c3d91b9587ccd5bf7d6389d87d5367194a9 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Mon, 25 Nov 2024 15:14:54 -0300 Subject: [PATCH 5/7] Update bootstrap --- bootstrap/stage1/crd.tf | 57 +++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/bootstrap/stage1/crd.tf b/bootstrap/stage1/crd.tf index 3b19043..18017cf 100644 --- a/bootstrap/stage1/crd.tf +++ b/bootstrap/stage1/crd.tf @@ -1,7 +1,7 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_doom" { manifest = { "apiVersion" = "apiextensions.k8s.io/v1" - "kind" = "CustomResourceDefinition" + "kind" = "CustomResourceDefinition" "metadata" = { "name" = "hydradoomnodes.hydra.doom" } @@ -11,7 +11,7 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do "categories" = [ "hydradoom", ] - "kind" = "HydraDoomNode" + "kind" = "HydraDoomNode" "plural" = "hydradoomnodes" "shortNames" = [ "hydradoomnode", @@ -24,28 +24,28 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do "additionalPrinterColumns" = [ { "jsonPath" = ".status.nodeState" - "name" = "Node State" - "type" = "string" + "name" = "Node State" + "type" = "string" }, { "jsonPath" = ".status.gameState" - "name" = "Game State" - "type" = "string" + "name" = "Game State" + "type" = "string" }, { "jsonPath" = ".status.transactions" - "name" = "Transactions" - "type" = "string" + "name" = "Transactions" + "type" = "string" }, { "jsonPath" = ".status.localUrl" - "name" = "Local URI" - "type" = "string" + "name" = "Local URI" + "type" = "string" }, { "jsonPath" = ".status.externalUrl" - "name" = "External URI" - "type" = "string" + "name" = "External URI" + "type" = "string" }, ] "name" = "v1alpha1" @@ -57,23 +57,17 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do "properties" = { "asleep" = { "nullable" = true - "type" = "boolean" - } - "commitInputs" = { - "items" = { - "type" = "string" - } - "type" = "array" + "type" = "boolean" } "networkId" = { - "format" = "uint8" - "minimum" = 0 + "format" = "uint8" + "minimum" = 0 "nullable" = true - "type" = "integer" + "type" = "integer" } "offline" = { "nullable" = true - "type" = "boolean" + "type" = "boolean" } "resources" = { "nullable" = true @@ -115,18 +109,15 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do ] "type" = "object" } - "seedInput" = { - "type" = "string" + "snapshot" = { + "nullable" = true + "type" = "string" } "startChainFrom" = { "nullable" = true - "type" = "string" + "type" = "string" } } - "required" = [ - "commitInputs", - "seedInput", - ] "type" = "object" } "status" = { @@ -146,7 +137,7 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do } "transactions" = { "format" = "int64" - "type" = "integer" + "type" = "integer" } } "required" = [ @@ -163,10 +154,10 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do "spec", ] "title" = "HydraDoomNode" - "type" = "object" + "type" = "object" } } - "served" = true + "served" = true "storage" = true "subresources" = { "status" = {} From d2aeeb323c782fe6568e2efd2595af524f2f69a1 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Wed, 27 Nov 2024 15:19:19 -0300 Subject: [PATCH 6/7] Parametrize snapshot path --- .github/workflows/deploy.yml | 1 + bootstrap/stage2/deployment.tf | 7 ++++++- bootstrap/stage2/main.tf | 5 +++++ crates/operator/src/config.rs | 5 +++++ crates/operator/src/controller.rs | 6 ++---- playbook/doom-dev/main.tf | 6 ++++++ 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 080bbd8..20df978 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -188,6 +188,7 @@ jobs: TF_VAR_autoscaler_low_watermark: ${{ matrix.autoscaler_low_watermark }} TF_VAR_autoscaler_region_prefix: ${{ matrix.region_prefix }} TF_VAR_autoscaler_max_batch: ${{ matrix.autoscaler_max_batch }} + TF_VAR_available_snapshot_prefix: snapshots steps: - name: Filter regions diff --git a/bootstrap/stage2/deployment.tf b/bootstrap/stage2/deployment.tf index 61cb020..b8dc946 100644 --- a/bootstrap/stage2/deployment.tf +++ b/bootstrap/stage2/deployment.tf @@ -172,7 +172,12 @@ resource "kubernetes_deployment_v1" "operator" { } env { - name = "NETWORK_ID" + name = "AVAILABLE_SNAPSHOT_PREFIX" + value = var.available_snapshot_prefix + } + + env { + name = "NETWORK_ID" value = var.network_id } diff --git a/bootstrap/stage2/main.tf b/bootstrap/stage2/main.tf index a13e490..ee156a0 100644 --- a/bootstrap/stage2/main.tf +++ b/bootstrap/stage2/main.tf @@ -154,6 +154,11 @@ variable "network_id" { type = number } +variable "available_snapshot_prefix" { + type = string + default = "snapshots" +} + variable "tolerations" { type = list(object({ effect = string diff --git a/crates/operator/src/config.rs b/crates/operator/src/config.rs index 338e811..52ec28d 100644 --- a/crates/operator/src/config.rs +++ b/crates/operator/src/config.rs @@ -32,6 +32,8 @@ pub struct Config { pub init_aws_access_key_id: String, pub init_aws_secret_access_key: String, pub network_id: String, + pub available_snapshot_prefix: String, + // Autoscaler pub autoscaler_delay: Duration, pub autoscaler_low_watermark: usize, @@ -67,6 +69,9 @@ impl Config { .expect("Missing INIT_AWS_ACCESS_KEY_ID env var."), init_aws_secret_access_key: env::var("INIT_AWS_SECRET_ACCESS_KEY") .expect("Missing INIT_AWS_SECRET_ACCESS_KEY env var."), + available_snapshot_prefix: env::var("AVAILABLE_SNAPSHOT_PREFIX") + .unwrap_or("snapshots".to_string()), + autoscaler_delay: env::var("AUTOSCALER_DELAY") .map(|duration| { Duration::from_secs(duration.parse().expect("Failed to parse AUTOSCALER_DELAY")) diff --git a/crates/operator/src/controller.rs b/crates/operator/src/controller.rs index 1e176d7..fbf7853 100644 --- a/crates/operator/src/controller.rs +++ b/crates/operator/src/controller.rs @@ -111,13 +111,11 @@ pub struct K8sConstants { pub dmtrctl_image: String, pub storage_class_name: String, pub service_account_name: String, - pub available_snapshot_prefix: String, pub used_snapshot_prefix: String, } impl Default for K8sConstants { fn default() -> Self { Self { - available_snapshot_prefix: "snapshots".to_string(), used_snapshot_prefix: "used".to_string(), storage_class_name: "efs-sc".to_string(), config_dir: "/etc/config".to_string(), @@ -413,7 +411,7 @@ impl K8sContext { .s3_client .list_objects_v2() .bucket(self.config.bucket.clone()) - .prefix(self.constants.available_snapshot_prefix.clone()) + .prefix(self.config.available_snapshot_prefix.clone()) .max_keys(100) .into_paginator() .send(); @@ -434,7 +432,7 @@ impl K8sContext { async fn use_snapshot(&self, snapshot_key: &str) -> anyhow::Result { let new_key = snapshot_key.replace( - &self.constants.available_snapshot_prefix, + &self.config.available_snapshot_prefix, &self.constants.used_snapshot_prefix, ); diff --git a/playbook/doom-dev/main.tf b/playbook/doom-dev/main.tf index 8fd6bb4..0426626 100644 --- a/playbook/doom-dev/main.tf +++ b/playbook/doom-dev/main.tf @@ -120,6 +120,11 @@ variable "autoscaler_max_batch" { default = 2 } +variable "available_snapshot_prefix" { + type = string + default = "snapshots" +} + provider "kubernetes" { config_path = "~/.kube/config" config_context = var.eks_cluster_arn @@ -165,6 +170,7 @@ module "stage2" { autoscaler_region_prefix = var.autoscaler_region_prefix autoscaler_max_batch = var.autoscaler_max_batch network_id = var.network_id + available_snapshot_prefix = var.available_snapshot_prefix resources = { requests = { cpu = "500m" From 431886ed24660a2497597bbf9d8dad5470e5b2a7 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Wed, 27 Nov 2024 15:32:55 -0300 Subject: [PATCH 7/7] Parametrize init image --- .github/workflows/deploy.yml | 1 + playbook/doom-dev/main.tf | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d114511..844091f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -189,6 +189,7 @@ jobs: TF_VAR_autoscaler_region_prefix: ${{ matrix.region_prefix }} TF_VAR_autoscaler_max_batch: ${{ matrix.autoscaler_max_batch }} TF_VAR_available_snapshot_prefix: snapshots + TF_VAR_init_image: ghcr.io/cardano-scaling/hydra-control-plane-init:eca9dc5f3541a8d71573ca5b86456299ab5c6ef9 steps: - name: Filter regions diff --git a/playbook/doom-dev/main.tf b/playbook/doom-dev/main.tf index 0426626..f00db29 100644 --- a/playbook/doom-dev/main.tf +++ b/playbook/doom-dev/main.tf @@ -46,6 +46,10 @@ variable "image" { type = string } +variable "init_image" { + type = string +} + variable "hydra_node_image" { type = string description = "The Docker image to use for the Hydra node component." @@ -162,7 +166,7 @@ module "stage2" { hydra_scripts_tx_id = var.hydra_scripts_tx_id init_aws_access_key_id = var.snapshot_aws_access_key_id init_aws_secret_access_key = var.snapshot_aws_secret_access_key - init_image = "ghcr.io/demeter-run/doom-patrol-init:b7b4fc499b5274cd71b6b72f93ab4ba8199437fe" + init_image = var.init_image frontend_image = var.frontend_image frontend_replicas = var.frontend_replicas autoscaler_high_watermark = var.autoscaler_high_watermark