diff --git a/.envrc b/.envrc index 8ff688e8..29dc2412 100644 --- a/.envrc +++ b/.envrc @@ -1,4 +1,4 @@ use_flake -export RUST_LOG=homestar=debug,sqlx=warn,atuin_client=warn,libp2p=debug -export RUST_BACKTRACE=1 +export RUST_LOG=homestar_runtime=debug,atuin_client=warn,libp2p=info +export RUST_BACKTRACE=full diff --git a/.github/workflows/tests_and_checks.yml b/.github/workflows/tests_and_checks.yml index 776f6d6e..6a4585e4 100644 --- a/.github/workflows/tests_and_checks.yml +++ b/.github/workflows/tests_and_checks.yml @@ -79,7 +79,7 @@ jobs: if: ${{ matrix.rust-toolchain == 'stable' && github.event_name == 'push' }} run: cargo build --release - run-tests: + run-tests-all-features: runs-on: ubuntu-latest strategy: fail-fast: false @@ -112,3 +112,40 @@ jobs: - name: Run Tests run: cargo test --all-features + + - name: Run Tests + run: cargo test --no-default-features + + run-tests-no-default-features: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + rust-toolchain: + - stable + - nightly + steps: + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Install Environment Packages + run: | + sudo apt-get update -qqy + sudo apt-get install jq + + - name: Cache Project + uses: Swatinem/rust-cache@v2 + + - name: Install Rust Toolchain + uses: actions-rs/toolchain@v1 + with: + override: true + toolchain: ${{ matrix.rust-toolchain }} + + - name: Run Tests + run: cargo test --no-default-features diff --git a/.gitignore b/.gitignore index a17beb4a..ba1448f6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,12 @@ private *.temp *.db *.tmp +*.png +*.dot .history .DS_Store homestar-guest-wasm/out +homestar-wasm/out # locks homestar-wasm/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index 4a34834e..6ad22d99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -165,63 +165,87 @@ version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anstream" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", + "anstyle-query", "anstyle-wincon", - "concolor-override", - "concolor-query", + "colorchoice", "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "0.3.5" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" [[package]] name = "anstyle-parse" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" dependencies = [ "utf8parse", ] [[package]] -name = "anstyle-wincon" -version = "0.2.0" +name = "anstyle-query" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "anstyle", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] -name = "anyhow" -version = "1.0.70" +name = "anstyle-wincon" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] [[package]] -name = "arbitrary" -version = "1.3.0" +name = "anyhow" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +dependencies = [ + "backtrace", +] [[package]] name = "arc-swap" @@ -362,7 +386,7 @@ dependencies = [ "log", "parking", "polling", - "rustix 0.37.11", + "rustix 0.37.19", "slab", "socket2", "waker-fn", @@ -385,7 +409,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -433,7 +457,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -450,7 +474,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -474,9 +498,9 @@ checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" [[package]] name = "atomic_refcell" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "857253367827bd9d0fd973f0ef15506a96e79e41b0ad7aa691203a4e3214f6c8" +checksum = "79d6dc922a2792b006573f60b2648076355daeae5ce9cb59507e5908c9625d31" [[package]] name = "atty" @@ -495,6 +519,74 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +dependencies = [ + "async-trait", + "axum-core", + "base64 0.21.0", + "bitflags", + "bytes", + "futures-util", + "headers", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite 0.2.9", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1 0.10.5", + "sync_wrapper", + "tokio", + "tokio-tungstenite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide 0.6.2", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.11" @@ -561,6 +653,18 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake2" version = "0.10.6" @@ -707,15 +811,24 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6ed94e98ecff0c12dd1b04c15ec0d7d9458ca8fe806cea6f12954efe74c63b" + +[[package]] +name = "byte-unit" +version = "4.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c" +dependencies = [ + "utf8-width", +] [[package]] name = "bytecheck" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13fe11640a23eb24562225322cd3e452b93a3d4091d62fab69c70542fcd17d1f" +checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -724,9 +837,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5" +checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" dependencies = [ "proc-macro2", "quote", @@ -753,9 +866,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cap-fs-ext" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80754c33c036aa2b682c4c2f6d10f43c5a9b68527e89169706027ce285b0ea30" +checksum = "58bc48200a1a0fa6fba138b1802ad7def18ec1cdd92f7b2a04e21f1bd887f7b9" dependencies = [ "cap-primitives", "cap-std", @@ -763,39 +876,28 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "cap-net-ext" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12eb3ab6abfc0e1b81b2a0f4cd14e09f96e59459a47a1442bf144bdc63dfc50a" -dependencies = [ - "cap-primitives", - "cap-std", - "rustix 0.37.11", -] - [[package]] name = "cap-primitives" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1b2aadbde9f86e045e309df5d707600254d45eda76df251a7b840f81681d72" +checksum = "a4b6df5b295dca8d56f35560be8c391d59f0420f72e546997154e24e765e6451" dependencies = [ "ambient-authority", - "fs-set-times", + "fs-set-times 0.19.1", "io-extras", "io-lifetimes", "ipnet", "maybe-owned", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.48.0", "winx", ] [[package]] name = "cap-rand" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4213970e64bba7b90bd25158955f024221dd45c0751cfd0c42f2745e9b177c1" +checksum = "4d25555efacb0b5244cf1d35833d55d21abc916fff0eaad254b8e2453ea9b8ab" dependencies = [ "ambient-authority", "rand 0.8.5", @@ -803,25 +905,25 @@ dependencies = [ [[package]] name = "cap-std" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60db4439f80165589d00673a086e9e106e224944dd09cdf5553cedfbc90fe5c" +checksum = "3373a62accd150b4fcba056d4c5f3b552127f0ec86d3c8c102d60b978174a012" dependencies = [ "cap-primitives", "io-extras", "io-lifetimes", - "rustix 0.37.11", + "rustix 0.37.19", ] [[package]] name = "cap-time-ext" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afa389ffcd0c66daca4497d1a9992d18b985eff6b747ee8b4c86c2beae1f708" +checksum = "e95002993b7baee6b66c8950470e59e5226a23b3af39fc59c47fe416dd39821a" dependencies = [ "cap-primitives", "once_cell", - "rustix 0.37.11", + "rustix 0.37.19", "winx", ] @@ -891,11 +993,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "serde", + "winapi", +] + [[package]] name = "ciborium" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" dependencies = [ "ciborium-io", "ciborium-ll", @@ -904,15 +1019,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" [[package]] name = "ciborium-ll" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" dependencies = [ "ciborium-io", "half 1.8.2", @@ -932,20 +1047,6 @@ dependencies = [ "unsigned-varint", ] -[[package]] -name = "cid" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b68e3193982cd54187d71afdb2a271ad4cf8af157858e9cb911b91321de143" -dependencies = [ - "core2", - "multibase", - "multihash 0.17.0", - "serde", - "serde_bytes", - "unsigned-varint", -] - [[package]] name = "cid" version = "0.10.1" @@ -954,7 +1055,7 @@ checksum = "fd94671561e36e4e7de75f753f577edafb0e7c05d6e4547229fdf7938fbcd2c3" dependencies = [ "core2", "multibase", - "multihash 0.18.0", + "multihash 0.18.1", "serde", "serde_bytes", "unsigned-varint", @@ -990,9 +1091,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.23" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags", "clap_lex 0.2.4", @@ -1002,9 +1103,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.1" +version = "4.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" +checksum = "34d21f9bf1b425d2968943631ec91202fe5e837264063503708b83013f8fc938" dependencies = [ "clap_builder", "clap_derive", @@ -1013,9 +1114,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.1" +version = "4.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +checksum = "914c8c79fb560f238ef6429439a30023c862f7a28e688c58f7203f12b29970bd" dependencies = [ "anstream", "anstyle", @@ -1033,7 +1134,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -1057,6 +1158,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "common-multipart-rfc7578" version = "0.6.0" @@ -1074,27 +1181,74 @@ dependencies = [ ] [[package]] -name = "concolor-override" -version = "1.0.0" +name = "concat-in-place" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" +checksum = "c5b80dba65d26e0c4b692ad0312b837f1177e8175031af57fd1de4f3bc36b430" [[package]] -name = "concolor-query" -version = "0.3.3" +name = "concurrent-queue" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" dependencies = [ - "windows-sys 0.45.0", + "crossbeam-utils", ] [[package]] -name = "concurrent-queue" -version = "2.2.0" +name = "config" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" +dependencies = [ + "async-trait", + "json5", + "lazy_static", + "nom", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "yaml-rust", +] + +[[package]] +name = "console-api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ab2224a0311582eb03adba4caaf18644f7b1f10a760803a803b9b605187fc7" dependencies = [ + "console-api", + "crossbeam-channel", "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "parking_lot 0.12.1", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", ] [[package]] @@ -1145,31 +1299,32 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] [[package]] name = "cranelift-bforest" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1277fbfa94bc82c8ec4af2ded3e639d49ca5f7f3c7eeab2c66accd135ece4e70" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8c31ad3b2270e9aeec38723888fe1b0ace3bea2b06b3f749ccf46661d3220" dependencies = [ "bumpalo", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", - "cranelift-control", "cranelift-entity", "cranelift-isle", "gimli", @@ -1182,37 +1337,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ac5ac30d62b2d66f12651f6b606dbdfd9c2cfd0908de6b387560a277c5c9da" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" - -[[package]] -name = "cranelift-control" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" -dependencies = [ - "arbitrary", -] +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd82b8b376247834b59ed9bdc0ddeb50f517452827d4a11bccf5937b213748b8" [[package]] name = "cranelift-entity" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40099d38061b37e505e63f89bab52199037a72b931ad4868d9089ff7268660b0" dependencies = [ "serde", ] [[package]] name = "cranelift-frontend" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a25d9d0a0ae3079c463c34115ec59507b4707175454f0eee0891e83e30e82d" dependencies = [ "cranelift-codegen", "log", @@ -1222,13 +1373,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80de6a7d0486e4acbd5f9f87ec49912bf4c8fb6aea00087b989685460d4469ba" [[package]] name = "cranelift-native" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6b03e0e03801c4b3fd8ce0758a94750c07a44e7944cc0ffbf0d3f2e7c79b00" dependencies = [ "cranelift-codegen", "libc", @@ -1237,8 +1390,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.96.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "0.95.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff3220489a3d928ad91e59dd7aeaa8b3de18afb554a6211213673a71c90737ac" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -1284,7 +1438,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.23", + "clap 3.2.25", "criterion-plot", "itertools", "lazy_static", @@ -1310,6 +1464,20 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -1344,6 +1512,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -1420,17 +1598,6 @@ dependencies = [ "cipher 0.4.4", ] -[[package]] -name = "cuckoofilter" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b810a8449931679f64cd7eef1bbd0fa315801b6d5d9cdc1ace2804d6529eee18" -dependencies = [ - "byteorder", - "fnv", - "rand 0.7.3", -] - [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -1458,14 +1625,36 @@ dependencies = [ "zeroize", ] +[[package]] +name = "dagga" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee7c9e23428d020bc4597fa94fba2a84d71ddb6aec22797ff4dc1f9a6a0dd320" +dependencies = [ + "dot2", + "log", + "rustc-hash", + "snafu", +] + [[package]] name = "darling" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" +dependencies = [ + "darling_core 0.20.1", + "darling_macro 0.20.1", ] [[package]] @@ -1482,17 +1671,42 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.16", +] + [[package]] name = "darling_macro" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core", + "darling_core 0.14.4", "quote", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" +dependencies = [ + "darling_core 0.20.1", + "quote", + "syn 2.0.16", +] + [[package]] name = "data-encoding" version = "2.3.3" @@ -1573,7 +1787,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling", + "darling 0.14.4", "proc-macro2", "quote", "syn 1.0.109", @@ -1591,12 +1805,13 @@ dependencies = [ [[package]] name = "diesel" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4391a22b19c916e50bec4d6140f29bdda3e3bb187223fe6e3ea0b6e4d1021c04" +checksum = "72eb77396836a4505da85bae0712fa324b74acfe1876d7c2f7e694ef3d0ee373" dependencies = [ "diesel_derives", "libsqlite3-sys", + "r2d2", ] [[package]] @@ -1691,15 +1906,33 @@ checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" [[package]] name = "displaydoc" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "dot2" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "855423f2158bcc73798b3b9a666ec4204597a72370dc91dbdb8e7f9519de8cc3" + [[package]] name = "dotenvy" version = "0.15.7" @@ -1885,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ae6b3d9530211fb3b12a95374b8b0823be812f53d09e18c5675c0146b09642" dependencies = [ "cfg-if", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.48.0", ] @@ -1926,13 +2159,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide 0.6.2", + "miniz_oxide 0.7.1", ] [[package]] @@ -1954,6 +2187,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1963,6 +2211,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-set-times" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "857cf27edcb26c2a36d84b2954019573d335bb289876113aceacacdca47a4fd4" +dependencies = [ + "io-lifetimes", + "rustix 0.36.13", + "windows-sys 0.45.0", +] + [[package]] name = "fs-set-times" version = "0.19.1" @@ -1970,10 +2229,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7833d0f115a013d51c55950a3b09d30e4b057be9961b709acb9b5b17a1108861" dependencies = [ "io-lifetimes", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.48.0", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.28" @@ -2046,7 +2311,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -2205,9 +2470,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -2247,12 +2512,50 @@ dependencies = [ ] [[package]] -name = "hashbrown" -version = "0.13.2" +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.3", +] + +[[package]] +name = "hdrhistogram" +version = "7.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +dependencies = [ + "base64 0.13.1", + "byteorder", + "flate2", + "nom", + "num-traits", +] + +[[package]] +name = "headers" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" +dependencies = [ + "base64 0.13.1", + "bitflags", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1 0.10.5", +] + +[[package]] +name = "headers-core" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "ahash 0.8.3", + "http", ] [[package]] @@ -2338,9 +2641,9 @@ dependencies = [ "enum-as-inner", "enum-assoc", "generic-array", + "indexmap", "libipld", "proptest", - "semver 1.0.17", "serde", "signature 2.1.0", "thiserror", @@ -2362,27 +2665,48 @@ dependencies = [ name = "homestar-runtime" version = "0.1.0" dependencies = [ + "ansi_term", "anyhow", "async-trait", - "clap 4.2.1", + "axum", + "byte-unit", + "clap 4.2.7", + "concat-in-place", + "config", + "console-subscriber", "criterion", + "crossbeam", + "dagga", "diesel", "diesel_migrations", "dotenvy", - "env_logger", + "enum-assoc", + "futures", + "headers", "homestar-core", "homestar-wasm", + "http", + "http-serde", + "indexmap", "ipfs-api", "ipfs-api-backend-hyper", "itertools", + "json", "libipld", "libp2p", "libp2p-identity", "proptest", + "reqwest", + "semver 1.0.17", "serde", + "serde_with", + "thiserror", "tokio", + "tokio-tungstenite", "tracing", + "tracing-logfmt", "tracing-subscriber", + "tryhard", "url", ] @@ -2396,9 +2720,10 @@ dependencies = [ "enum-as-inner", "heck", "homestar-core", + "image", "itertools", "libipld", - "libipld-core 0.16.0", + "libipld-core", "rust_decimal", "serde_ipld_dagcbor", "stacker", @@ -2408,11 +2733,11 @@ dependencies = [ "tracing", "wasi-cap-std-sync", "wasi-common", - "wasmparser 0.101.1", + "wasmparser 0.104.0", "wasmtime", "wasmtime-component-util", "wat", - "wit-component", + "wit-component 0.8.2", ] [[package]] @@ -2448,6 +2773,16 @@ dependencies = [ "pin-project-lite 0.2.9", ] +[[package]] +name = "http-serde" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e272971f774ba29341db2f686255ff8a979365a26fb9e4277f6b6d9ec0cdd5e" +dependencies = [ + "http", + "serde", +] + [[package]] name = "httparse" version = "1.8.0" @@ -2468,9 +2803,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", @@ -2503,6 +2838,54 @@ dependencies = [ "hyper", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite 0.2.9", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows 0.48.0", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "id-arena" version = "2.2.1" @@ -2562,7 +2945,7 @@ dependencies = [ "rtnetlink", "system-configuration", "tokio", - "windows", + "windows 0.34.0", ] [[package]] @@ -2735,7 +3118,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.48.0", ] @@ -2794,18 +3177,35 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "keccak" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3afef3b6eff9ce9d8ff9b3601125eec7f0c8cbac7abd14f355d053fa56c98768" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" dependencies = [ "cpufeatures", ] @@ -2839,9 +3239,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libipld" @@ -2852,12 +3252,12 @@ dependencies = [ "fnv", "libipld-cbor", "libipld-cbor-derive", - "libipld-core 0.16.0", - "libipld-json 0.16.0", + "libipld-core", + "libipld-json", "libipld-macro", "libipld-pb", "log", - "multihash 0.18.0", + "multihash 0.18.1", "thiserror", ] @@ -2868,7 +3268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77d98c9d1747aa5eef1cf099cd648c3fd2d235249f5fed07522aaebc348e423b" dependencies = [ "byteorder", - "libipld-core 0.16.0", + "libipld-core", "thiserror", ] @@ -2885,21 +3285,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "libipld-core" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a704ba3b25dee9e7a2361fae2c7c19defae2a92e69ae96ffb203996705cd7c" -dependencies = [ - "anyhow", - "cid 0.9.0", - "core2", - "multibase", - "multihash 0.17.0", - "serde", - "thiserror", -] - [[package]] name = "libipld-core" version = "0.16.0" @@ -2910,31 +3295,19 @@ dependencies = [ "cid 0.10.1", "core2", "multibase", - "multihash 0.18.0", + "multihash 0.18.1", "serde", "thiserror", ] -[[package]] -name = "libipld-json" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc549d7c70f9a401031fcb6d3bf7eccfe91bcad938f7485f71ee8ba9f79c1e79" -dependencies = [ - "libipld-core 0.15.0", - "multihash 0.17.0", - "serde", - "serde_json", -] - [[package]] name = "libipld-json" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25856def940047b07b25c33d4e66d248597049ab0202085215dc4dca0487731c" dependencies = [ - "libipld-core 0.16.0", - "multihash 0.18.0", + "libipld-core", + "multihash 0.18.1", "serde", "serde_json", ] @@ -2945,7 +3318,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71171c54214f866ae6722f3027f81dff0931e600e5a61e6b1b6a49ca0b5ed4ae" dependencies = [ - "libipld-core 0.16.0", + "libipld-core", ] [[package]] @@ -2955,7 +3328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f2d0f866c4cd5dc9aa8068c429ba478d2882a3a4b70ab56f7e9a0eddf5d16f" dependencies = [ "bytes", - "libipld-core 0.16.0", + "libipld-core", "quick-protobuf", "thiserror", ] @@ -2968,9 +3341,9 @@ checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" [[package]] name = "libm" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libp2p" @@ -2987,13 +3360,13 @@ dependencies = [ "libp2p-connection-limits", "libp2p-core", "libp2p-dns", - "libp2p-floodsub", "libp2p-gossipsub", "libp2p-identify", "libp2p-identity", "libp2p-kad", "libp2p-mdns", "libp2p-metrics", + "libp2p-mplex", "libp2p-noise", "libp2p-quic", "libp2p-request-response", @@ -3008,9 +3381,9 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c54073648b20bb47335164544a4e5694434f44530f47a4f6618f5f585f3ff5" +checksum = "510daa05efbc25184458db837f6f9a5143888f1caa742426d92e1833ddd38a50" dependencies = [ "libp2p-core", "libp2p-identity", @@ -3032,9 +3405,9 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.39.1" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7f8b7d65c070a5a1b5f8f0510648189da08f787b8963f8e21219e0710733af" +checksum = "3c1df63c0b582aa434fb09b2d86897fa2b419ffeccf934b36f87fcedc8e835c2" dependencies = [ "either", "fnv", @@ -3072,37 +3445,17 @@ dependencies = [ "trust-dns-resolver", ] -[[package]] -name = "libp2p-floodsub" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "089336308101c0f5507e2aae8a693b0997bd3b31d88564530de1596d31a9b87a" -dependencies = [ - "asynchronous-codec", - "cuckoofilter", - "fnv", - "futures", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "log", - "quick-protobuf", - "quick-protobuf-codec", - "rand 0.8.5", - "smallvec", - "thiserror", -] - [[package]] name = "libp2p-gossipsub" -version = "0.44.3" +version = "0.44.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac213adad69bd9866fe87c37fbf241626715e5cd454fb6df9841aa2b02440ee" +checksum = "70b34b6da8165c0bde35c82db8efda39b824776537e73973549e76cadb3a77c5" dependencies = [ "asynchronous-codec", "base64 0.21.0", "byteorder", "bytes", + "either", "fnv", "futures", "hex_fmt", @@ -3120,14 +3473,15 @@ dependencies = [ "smallvec", "thiserror", "unsigned-varint", + "void", "wasm-timer", ] [[package]] name = "libp2p-identify" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40d1da1f75baf824cfdc80f6aced51f7cbf8dc14e32363e9443570a80d4ee337" +checksum = "5455f472243e63b9c497ff320ded0314254a9eb751799a39c283c6f20b793f3c" dependencies = [ "asynchronous-codec", "either", @@ -3147,27 +3501,27 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8ea433ae0cea7e3315354305237b9897afe45278b2118a7a57ca744e70fd27" +checksum = "9e2d584751cecb2aabaa56106be6be91338a60a0f4e420cf2af639204f596fc1" dependencies = [ "bs58", "ed25519-dalek", "log", "multiaddr", "multihash 0.17.0", - "prost", "quick-protobuf", "rand 0.8.5", + "sha2 0.10.6", "thiserror", "zeroize", ] [[package]] name = "libp2p-kad" -version = "0.43.2" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9647c76e63c4d0e9a10369cef9b929a2e5e8f03008b2097ab365fc4cb4e5318f" +checksum = "39d5ef876a2b2323d63c258e63c2f8e36f205fe5a11f0b3095d59635650790ff" dependencies = [ "arrayvec", "asynchronous-codec", @@ -3226,11 +3580,29 @@ dependencies = [ "prometheus-client", ] +[[package]] +name = "libp2p-mplex" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d34780b514b159e6f3fd70ba3e72664ec89da28dca2d1e7856ee55e2c7031ba" +dependencies = [ + "asynchronous-codec", + "bytes", + "futures", + "libp2p-core", + "log", + "nohash-hasher", + "parking_lot 0.12.1", + "rand 0.8.5", + "smallvec", + "unsigned-varint", +] + [[package]] name = "libp2p-noise" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c87c2803deffeae94108072a0387f8c9ff494af68a4908454c11c811e27b5e5" +checksum = "9c3673da89d29936bc6435bafc638e2f184180d554ce844db65915113f86ec5e" dependencies = [ "bytes", "curve25519-dalek 3.2.0", @@ -3273,27 +3645,25 @@ dependencies = [ [[package]] name = "libp2p-request-response" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872b9d63fed44d9f81110c30be6c7ca5593a093576d2a95c5d018051e294d2e9" +checksum = "7ffdb374267d42dc5ed5bc53f6e601d4a64ac5964779c6e40bb9e4f14c1e30d5" dependencies = [ "async-trait", - "bytes", "futures", "instant", "libp2p-core", + "libp2p-identity", "libp2p-swarm", - "log", "rand 0.8.5", "smallvec", - "unsigned-varint", ] [[package]] name = "libp2p-swarm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd1e223f02fcd7e3790f9b954e2e81791810e3512daeb27fa97df7652e946bc2" +checksum = "903b3d592d7694e56204d211f29d31bc004be99386644ba8731fc3e3ef27b296" dependencies = [ "either", "fnv", @@ -3408,23 +3778,22 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.43.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d048cd82f72c8800655aa1ee9b808ea8c9d523a649d3579b3ef0cfe23952d7fd" +checksum = "4dcd21d950662700a385d4c6d68e2f5f54d778e97068cdd718522222ef513bda" dependencies = [ "futures", "libp2p-core", "log", - "parking_lot 0.12.1", "thiserror", "yamux", ] [[package]] name = "libsqlite3-sys" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" dependencies = [ "pkg-config", "vcpkg", @@ -3432,9 +3801,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" dependencies = [ "cc", "pkg-config", @@ -3455,9 +3824,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" [[package]] name = "lock_api" @@ -3481,9 +3850,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e7d46de488603ffdd5f30afbc64fbba2378214a2c3a2fb83abf3d33126df17" +checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e" dependencies = [ "hashbrown 0.13.2", ] @@ -3512,12 +3881,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "maybe-owned" version = "0.3.4" @@ -3551,7 +3935,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffc89ccdc6e10d6907450f753537ebc5c5d3460d2e4e62ea74bd571db62c0f9e" dependencies = [ - "rustix 0.37.11", + "rustix 0.37.19", ] [[package]] @@ -3695,24 +4079,18 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" dependencies = [ - "blake2b_simd", - "blake2s_simd", - "blake3", "core2", "digest 0.10.6", "multihash-derive", - "serde", - "serde-big-array", "sha2 0.10.6", - "sha3", "unsigned-varint", ] [[package]] name = "multihash" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e5d911412e631e1de11eb313e4dd71f73fd964401102aab23d6c8327c431ba" +checksum = "cfd8a792c1694c6da4f68db0a9d707c72bd260994da179e6030a5dcee00bb815" dependencies = [ "blake2b_simd", "blake2s_simd", @@ -3764,6 +4142,24 @@ dependencies = [ "getrandom 0.2.9", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "netlink-packet-core" version = "0.4.2" @@ -3907,7 +4303,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", - "libm 0.2.6", + "libm 0.2.7", ] [[package]] @@ -3968,6 +4364,60 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -4072,6 +4522,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "pem" version = "1.1.1" @@ -4096,24 +4552,68 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "pest" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" +dependencies = [ + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "pest_meta" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.6", +] + [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] @@ -4146,9 +4646,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platforms" @@ -4199,9 +4699,9 @@ dependencies = [ [[package]] name = "polling" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be1c66a6add46bff50935c313dae30a5030cf8385c5206e8a95e9e9def974aa" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags", @@ -4299,9 +4799,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ "unicode-ident", ] @@ -4344,7 +4844,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax", + "regex-syntax 0.6.29", "rusty-fork", "tempfile", "unarray", @@ -4373,6 +4873,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + [[package]] name = "psm" version = "0.1.21" @@ -4487,13 +4996,30 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ "proc-macro2", ] +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot 0.12.1", + "scheduled-thread-pool", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -4664,13 +5190,22 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -4679,6 +5214,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + [[package]] name = "rend" version = "0.4.0" @@ -4688,6 +5229,43 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite 0.2.9", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.10.1", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -4726,29 +5304,43 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.41" +version = "0.7.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21499ed91807f07ae081880aabb2ccc0235e9d88011867d984525e9a4c3cfa3e" +checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" dependencies = [ + "bitvec", "bytecheck", "hashbrown 0.12.3", "ptr_meta", "rend", "rkyv_derive", "seahash", + "tinyvec", + "uuid", ] [[package]] name = "rkyv_derive" -version = "0.7.41" +version = "0.7.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1c672430eb41556291981f45ca900a0239ad007242d1cb4b4167af842db666" +checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" dependencies = [ "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "ron" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" +dependencies = [ + "base64 0.13.1", + "bitflags", + "serde", +] + [[package]] name = "rtcp" version = "0.7.2" @@ -4789,6 +5381,16 @@ dependencies = [ "webrtc-util", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rust_decimal" version = "1.29.1" @@ -4809,9 +5411,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a36c42d1873f9a77c53bde094f9664d9891bc604a45b4798fd2c389ed12e5b" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustc-hash" @@ -4848,9 +5450,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.12" +version = "0.36.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0af200a3324fa5bcd922e84e9b55a298ea9f431a489f01961acdebc6e908f25" +checksum = "3a38f9520be93aba504e8ca974197f46158de5dcaa9fa04b57c57cd6a679d658" dependencies = [ "bitflags", "errno", @@ -4862,16 +5464,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags", "errno", "io-lifetimes", "itoa", "libc", - "linux-raw-sys 0.3.1", + "linux-raw-sys 0.3.7", "once_cell", "windows-sys 0.48.0", ] @@ -4945,6 +5547,24 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot 0.12.1", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -5003,6 +5623,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.9.0" @@ -5026,9 +5669,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] @@ -5053,13 +5696,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -5085,6 +5728,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5097,6 +5749,34 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +dependencies = [ + "darling 0.20.1", + "proc-macro2", + "quote", + "syn 2.0.16", +] + [[package]] name = "sha-1" version = "0.9.8" @@ -5162,9 +5842,9 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c2bb1a323307527314a36bfb73f24febb08ce2b8a554bf4ffd6f51ad15198c" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ "digest 0.10.6", "keccak", @@ -5179,6 +5859,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -5227,9 +5916,9 @@ dependencies = [ [[package]] name = "slice-group-by" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" @@ -5237,6 +5926,28 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "snafu" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "snow" version = "0.9.2" @@ -5453,15 +6164,21 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" @@ -5489,9 +6206,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags", "core-foundation", @@ -5510,25 +6227,31 @@ dependencies = [ [[package]] name = "system-interface" -version = "0.25.6" +version = "0.25.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e1ab6a74e204b606bf397944fa991f3b01046113cc0a4ac269be3ef067cc24b" +checksum = "928ebd55ab758962e230f51ca63735c5b283f26292297c81404289cda5d78631" dependencies = [ "bitflags", "cap-fs-ext", "cap-std", "fd-lock", "io-lifetimes", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.48.0", "winx", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" -version = "0.12.6" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" [[package]] name = "tempfile" @@ -5539,7 +6262,7 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.45.0", ] @@ -5575,7 +6298,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -5601,9 +6324,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", "serde", @@ -5613,15 +6336,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -5653,9 +6376,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" dependencies = [ "autocfg", "bytes", @@ -5667,25 +6390,46 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "tracing", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite 0.2.9", + "tokio", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite 0.2.9", @@ -5705,11 +6449,23 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "tokio-tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -5729,6 +6485,60 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64 0.21.0", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite 0.2.9", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -5750,20 +6560,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -5780,16 +6590,33 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-logfmt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2fb2783ed7727b30a78ebecb49d59c98102b1f384105aa27d632487875a67b" +dependencies = [ + "time", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "parking_lot 0.12.1", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] @@ -5846,6 +6673,36 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "tryhard" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a30c54ab5d67a3aee4be0ffb0cc035426a06ffc3ef1ba979b3c658168688691" +dependencies = [ + "futures", + "pin-project-lite 0.2.9", + "tokio", +] + +[[package]] +name = "tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" +dependencies = [ + "base64 0.13.1", + "byteorder", + "bytes", + "http", + "httparse", + "log", + "rand 0.8.5", + "sha1 0.10.5", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "turn" version = "0.6.1" @@ -5884,22 +6741,22 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "ucan" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd5d89f3eb6d7e91ebcb733ceb95d186fa3479d264b4bacd971a25b3d5d336a" +checksum = "f45b08aa0818ab9be5d922d3931ea893a6c2e84d1a3bd6433ff2ca3e04a23425" dependencies = [ "anyhow", "async-recursion", "async-std", "async-trait", - "base64 0.13.1", + "base64 0.21.0", "bs58", - "cid 0.9.0", + "cid 0.10.1", "futures", "getrandom 0.2.9", "instant", - "libipld-core 0.15.0", - "libipld-json 0.15.0", + "libipld-core", + "libipld-json", "log", "rand 0.8.5", "serde", @@ -5910,6 +6767,12 @@ dependencies = [ "url", ] +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + [[package]] name = "uint" version = "0.9.5" @@ -6004,8 +6867,6 @@ checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" dependencies = [ "asynchronous-codec", "bytes", - "futures-io", - "futures-util", ] [[package]] @@ -6025,6 +6886,18 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8-width" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" + [[package]] name = "utf8parse" version = "0.2.1" @@ -6033,9 +6906,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ "getrandom 0.2.9", ] @@ -6132,55 +7005,53 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi-cap-std-sync" -version = "0.0.0" -source = "git+https://github.com/bytecodealliance/preview2-prototyping#d3bd2bc36a9caed047002e62561fc439c391df47" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612510e6c7b6681f7d29ce70ef26e18349c26acd39b7d89f1727d90b7f58b20e" dependencies = [ "anyhow", "async-trait", "cap-fs-ext", - "cap-net-ext", "cap-rand", "cap-std", "cap-time-ext", - "fs-set-times", + "fs-set-times 0.18.1", "io-extras", "io-lifetimes", - "ipnet", "is-terminal", "once_cell", - "rustix 0.37.11", + "rustix 0.36.13", "system-interface", "tracing", "wasi-common", - "windows-sys 0.48.0", + "windows-sys 0.45.0", ] [[package]] name = "wasi-common" -version = "0.0.0" -source = "git+https://github.com/bytecodealliance/preview2-prototyping#d3bd2bc36a9caed047002e62561fc439c391df47" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008136464e438c5049a614b6ea1bae9f6c4d354ce9ee2b4d9a1ac6e73f31aafc" dependencies = [ "anyhow", - "async-trait", "bitflags", - "cap-fs-ext", - "cap-net-ext", "cap-rand", "cap-std", "io-extras", - "ipnet", - "rustix 0.37.11", - "system-interface", + "log", + "rustix 0.36.13", "thiserror", "tracing", - "windows-sys 0.48.0", + "wasmtime", + "wiggle", + "windows-sys 0.45.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -6188,24 +7059,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if", "js-sys", @@ -6215,9 +7086,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6225,22 +7096,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "wasm-encoder" @@ -6251,6 +7122,24 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-encoder" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05d0b6fcd0aeb98adf16e7975331b3c17222aa815148f5b976370ce589d80ef" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77053dc709db790691d3732cfc458adc5acc881dec524965c608effdcd9c581" +dependencies = [ + "leb128", +] + [[package]] name = "wasm-metadata" version = "0.3.1" @@ -6260,10 +7149,23 @@ dependencies = [ "anyhow", "indexmap", "serde", - "wasm-encoder", + "wasm-encoder 0.25.0", "wasmparser 0.102.0", ] +[[package]] +name = "wasm-metadata" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbdef99fafff010c57fabb7bc703f0903ec16fcee49207a22dcc4f78ea44562f" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "wasm-encoder 0.26.0", + "wasmparser 0.104.0", +] + [[package]] name = "wasm-timer" version = "0.2.5" @@ -6281,9 +7183,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.101.1" +version = "0.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf2f22ef84ac5666544afa52f326f13e16f3d019d2e61e704fd8091c9358b130" +checksum = "48134de3d7598219ab9eaf6b91b15d8e50d31da76b8519fe4ecfcec2cf35104b" dependencies = [ "indexmap", "url", @@ -6291,9 +7193,19 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.102.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48134de3d7598219ab9eaf6b91b15d8e50d31da76b8519fe4ecfcec2cf35104b" +checksum = "6a396af81a7c56ad976131d6a35e4b693b78a1ea0357843bd436b4577e254a7d" +dependencies = [ + "indexmap", + "url", +] + +[[package]] +name = "wasmparser" +version = "0.105.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83be9e0b3f9570dc1979a33ae7b89d032c73211564232b99976553e5c155ec32" dependencies = [ "indexmap", "url", @@ -6301,18 +7213,19 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.2.54" +version = "0.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc17ae63836d010a2bf001c26a5fedbb9a05e5f71117fb63e0ab878bfbe1ca3" +checksum = "50b0e5ed7a74a065637f0d7798ce5f29cadb064980d24b0c82af5200122fa0d8" dependencies = [ "anyhow", - "wasmparser 0.102.0", + "wasmparser 0.105.0", ] [[package]] name = "wasmtime" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f907fdead3153cb9bfb7a93bbd5b62629472dc06dee83605358c64c52ed3dda9" dependencies = [ "anyhow", "async-trait", @@ -6338,23 +7251,24 @@ dependencies = [ "wasmtime-fiber", "wasmtime-jit", "wasmtime-runtime", - "wasmtime-winch", "wat", "windows-sys 0.45.0", ] [[package]] name = "wasmtime-asm-macros" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b9daa7c14cd4fa3edbf69de994408d5f4b7b0959ac13fa69d465f6597f810d" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-cache" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86437fa68626fe896e5afc69234bb2b5894949083586535f200385adfd71213" dependencies = [ "anyhow", "base64 0.21.0", @@ -6362,7 +7276,7 @@ dependencies = [ "directories-next", "file-per-thread-logger", "log", - "rustix 0.36.12", + "rustix 0.36.13", "serde", "sha2 0.10.6", "toml", @@ -6372,8 +7286,9 @@ dependencies = [ [[package]] name = "wasmtime-component-macro" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267096ed7cc93b4ab15d3daa4f195e04dbb7e71c7e5c6457ae7d52e9dd9c3607" dependencies = [ "anyhow", "proc-macro2", @@ -6381,22 +7296,23 @@ dependencies = [ "syn 1.0.109", "wasmtime-component-util", "wasmtime-wit-bindgen", - "wit-parser", + "wit-parser 0.6.4", ] [[package]] name = "wasmtime-component-util" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74e02ca7a4a3c69d72b88f26f0192e333958df6892415ac9ab84dcc42c9000c2" [[package]] name = "wasmtime-cranelift" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1cefde0cce8cb700b1b21b6298a3837dba46521affd7b8c38a9ee2c869eee04" dependencies = [ "anyhow", "cranelift-codegen", - "cranelift-control", "cranelift-entity", "cranelift-frontend", "cranelift-native", @@ -6413,12 +7329,12 @@ dependencies = [ [[package]] name = "wasmtime-cranelift-shared" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd041e382ef5aea1b9fc78442394f1a4f6d676ce457e7076ca4cb3f397882f8b" dependencies = [ "anyhow", "cranelift-codegen", - "cranelift-control", "cranelift-native", "gimli", "object", @@ -6428,8 +7344,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a990198cee4197423045235bf89d3359e69bd2ea031005f4c2d901125955c949" dependencies = [ "anyhow", "cranelift-entity", @@ -6440,7 +7357,7 @@ dependencies = [ "serde", "target-lexicon", "thiserror", - "wasm-encoder", + "wasm-encoder 0.25.0", "wasmparser 0.102.0", "wasmprinter", "wasmtime-component-util", @@ -6449,20 +7366,22 @@ dependencies = [ [[package]] name = "wasmtime-fiber" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab182d5ab6273a133ab88db94d8ca86dc3e57e43d70baaa4d98f94ddbd7d10a" dependencies = [ "cc", "cfg-if", - "rustix 0.36.12", + "rustix 0.36.13", "wasmtime-asm-macros", "windows-sys 0.45.0", ] [[package]] name = "wasmtime-jit" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de48df552cfca1c9b750002d3e07b45772dd033b0b206d5c0968496abf31244" dependencies = [ "addr2line", "anyhow", @@ -6485,18 +7404,20 @@ dependencies = [ [[package]] name = "wasmtime-jit-debug" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0554b84c15a27d76281d06838aed94e13a77d7bf604bbbaf548aa20eb93846" dependencies = [ "object", "once_cell", - "rustix 0.36.12", + "rustix 0.36.13", ] [[package]] name = "wasmtime-jit-icache-coherence" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecae978b13f7f67efb23bd827373ace4578f2137ec110bbf6a4a7cde4121bbd" dependencies = [ "cfg-if", "libc", @@ -6505,8 +7426,9 @@ dependencies = [ [[package]] name = "wasmtime-runtime" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658cf6f325232b6760e202e5255d823da5e348fdea827eff0a2a22319000b441" dependencies = [ "anyhow", "cc", @@ -6520,7 +7442,7 @@ dependencies = [ "memoffset 0.8.0", "paste", "rand 0.8.5", - "rustix 0.36.12", + "rustix 0.36.13", "wasmtime-asm-macros", "wasmtime-environ", "wasmtime-fiber", @@ -6530,8 +7452,9 @@ dependencies = [ [[package]] name = "wasmtime-types" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f6fffd2a1011887d57f07654dd112791e872e3ff4a2e626aee8059ee17f06f" dependencies = [ "cranelift-entity", "serde", @@ -6540,58 +7463,51 @@ dependencies = [ ] [[package]] -name = "wasmtime-winch" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +name = "wasmtime-wit-bindgen" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "983db9cc294d1adaa892a53ff6a0dc6605fc0ab1a4da5d8a2d2d4bde871ff7dd" dependencies = [ "anyhow", - "cranelift-codegen", - "gimli", - "object", - "target-lexicon", - "wasmparser 0.102.0", - "wasmtime-cranelift-shared", - "wasmtime-environ", - "winch-codegen", - "winch-environ", + "heck", + "wit-parser 0.6.4", ] [[package]] -name = "wasmtime-wit-bindgen" -version = "9.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" +name = "wast" +version = "35.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef140f1b49946586078353a453a1d28ba90adfc54dde75710bc1931de204d68" dependencies = [ - "anyhow", - "heck", - "wit-parser", + "leb128", ] [[package]] name = "wast" -version = "55.0.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4984d3e1406571f4930ba5cf79bd70f75f41d0e87e17506e0bd19b0e5d085f05" +checksum = "372eecae2d10a5091c2005b32377d7ecd6feecdf2c05838056d02d8b4f07c429" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder", + "wasm-encoder 0.27.0", ] [[package]] name = "wat" -version = "1.0.61" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af2b53f4da14db05d32e70e9c617abdf6620c575bd5dd972b7400037b4df2091" +checksum = "6d47446190e112ab1579ab40b3ad7e319d859d74e5134683f04e9f0747bf4173" dependencies = [ - "wast", + "wast 58.0.0", ] [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -6762,18 +7678,15 @@ dependencies = [ [[package]] name = "webrtc-media" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a3c157a040324e5049bcbd644ffc9079e6738fa2cfab2bcff64e5cc4c00d7" +checksum = "f72e1650a8ae006017d1a5280efb49e2610c19ccc3c0905b03b648aee9554991" dependencies = [ "byteorder", "bytes", - "derive_builder", - "displaydoc", "rand 0.8.5", "rtp", "thiserror", - "webrtc-util", ] [[package]] @@ -6850,6 +7763,48 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" +[[package]] +name = "wiggle" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b16a7462893c46c6d3dd2a1f99925953bdbb921080606e1a4c9344864492fa4" +dependencies = [ + "anyhow", + "async-trait", + "bitflags", + "thiserror", + "tracing", + "wasmtime", + "wiggle-macro", +] + +[[package]] +name = "wiggle-generate" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489499e186ab24c8ac6d89e9934c54ced6f19bd473730e6a74f533bd67ecd905" +dependencies = [ + "anyhow", + "heck", + "proc-macro2", + "quote", + "shellexpand", + "syn 1.0.109", + "witx", +] + +[[package]] +name = "wiggle-macro" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9142e7fce24a4344c85a43c8b719ef434fc6155223bade553e186cb4183b6cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wiggle-generate", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6881,30 +7836,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "winch-codegen" -version = "0.7.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" -dependencies = [ - "anyhow", - "cranelift-codegen", - "gimli", - "regalloc2", - "smallvec", - "target-lexicon", - "wasmparser 0.102.0", -] - -[[package]] -name = "winch-environ" -version = "0.7.0" -source = "git+https://github.com/bytecodealliance/wasmtime#91e36f3449055c19195552c668474e4a1c5e4cf4" -dependencies = [ - "wasmparser 0.102.0", - "wasmtime-environ", - "winch-codegen", -] - [[package]] name = "windows" version = "0.34.0" @@ -6918,6 +7849,30 @@ dependencies = [ "windows_x86_64_msvc 0.34.0", ] +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -7126,8 +8081,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef177b73007d86c720931d0e2ea7e30eb8c9776e58361717743fc1e83cfacfe5" dependencies = [ "anyhow", - "wit-component", - "wit-parser", + "wit-component 0.7.4", + "wit-parser 0.6.4", ] [[package]] @@ -7137,10 +8092,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efdf5b00935b7b52d0e56cae1960f8ac13019a285f5aa762ff6bd7139a5c28a2" dependencies = [ "heck", - "wasm-metadata", + "wasm-metadata 0.3.1", "wit-bindgen-core", "wit-bindgen-rust-lib", - "wit-component", + "wit-component 0.7.4", ] [[package]] @@ -7164,7 +8119,7 @@ dependencies = [ "syn 1.0.109", "wit-bindgen-core", "wit-bindgen-rust", - "wit-component", + "wit-component 0.7.4", ] [[package]] @@ -7178,10 +8133,27 @@ dependencies = [ "indexmap", "log", "url", - "wasm-encoder", - "wasm-metadata", + "wasm-encoder 0.25.0", + "wasm-metadata 0.3.1", "wasmparser 0.102.0", - "wit-parser", + "wit-parser 0.6.4", +] + +[[package]] +name = "wit-component" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e291ff83cb9c8e59963cc6922bdda77ed8f5517d6835f0c98070c4e7f1ae4996" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "url", + "wasm-encoder 0.26.0", + "wasm-metadata 0.5.0", + "wasmparser 0.104.0", + "wit-parser 0.7.1", ] [[package]] @@ -7199,6 +8171,42 @@ dependencies = [ "url", ] +[[package]] +name = "wit-parser" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca2581061573ef6d1754983d7a9b3ed5871ef859d52708ea9a0f5af32919172" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "pulldown-cmark", + "unicode-xid", + "url", +] + +[[package]] +name = "witx" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e366f27a5cabcddb2706a78296a40b8fcc451e1a6aba2fc1d94b4a01bdaaef4b" +dependencies = [ + "anyhow", + "log", + "thiserror", + "wast 35.0.2", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x25519-dalek" version = "1.1.1" @@ -7274,6 +8282,15 @@ dependencies = [ "winreg 0.8.0", ] +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yamux" version = "0.10.2" @@ -7314,7 +8331,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.16", ] [[package]] @@ -7349,9 +8366,9 @@ dependencies = [ [[package]] name = "zune-inflate" -version = "0.2.53" +version = "0.2.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "440a08fd59c6442e4b846ea9b10386c38307eae728b216e1ab2c305d1c9daaf8" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" dependencies = [ "simd-adler32", ] diff --git a/Cargo.toml b/Cargo.toml index 71ff2382..c08b520d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ "homestar-core", "homestar-guest-wasm", @@ -7,27 +8,38 @@ members = [ ] [workspace.package] -authors = ["Brooklyn Zelenka ", "Zeeshan Lakhani "] +authors = [ + "Zeeshan Lakhani ", + "Brooklyn Zelenka " +] edition = "2021" license = "Apache" rust-version = "1.66.0" +[workspace.dependencies] +anyhow = { version = "1.0", features = ["backtrace"] } +tracing = "0.1" + # Speedup build on macOS # See https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#splitting-debug-information [profile.dev] +debug-assertions = true split-debuginfo = "unpacked" [profile.release.package.homestar-core] # Tell `rustc` to optimize for small code size. opt-level = "s" +debug-assertions = false [profile.release.package.homestar-runtime] # Tell `rustc` to optimize for small code size. -opt-level = "s" +opt-level = "z" +debug-assertions = false [profile.release.package.homestar-wasm] # Tell `rustc` to optimize for small code size. opt-level = "s" +debug-assertions = false [profile.release.package.homestar-guest-wasm] # Perform optimizations on all codegen units. @@ -40,3 +52,4 @@ strip = "symbols" # Amount of debug information. # 0/false: no debug info at all; 1: line tables only; 2/true: full debug info debug = false +debug-assertions = false diff --git a/codecov.yml b/codecov.yml index 7f7e931b..e2fc69c4 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,5 +1,5 @@ ignore: - - "homestar/tests" + - "*/tests/*" - "homestar/src/test_utils" - "homestar-guest-wasm" - "homestar-wasm/src/test_utils" diff --git a/diesel.toml b/diesel.toml index 00b317bf..72072950 100644 --- a/diesel.toml +++ b/diesel.toml @@ -2,7 +2,7 @@ # see https://diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "homestar/src/schema.rs" +file = "homestar-runtime/src/db/schema.rs" [migrations_directory] -dir = "migrations" +dir = "homestar-runtime/migrations" diff --git a/flake.nix b/flake.nix index a0de0d60..a3cf75f4 100644 --- a/flake.nix +++ b/flake.nix @@ -80,16 +80,16 @@ packages.irust = pkgs.rustPlatform.buildRustPackage rec { pname = "irust"; - version = "1.66.0"; + version = "1.70.0"; src = pkgs.fetchFromGitHub { owner = "sigmaSd"; repo = "IRust"; rev = "v${version}"; - sha256 = "sha256-tqNTh8ojTT80kl9jDCMq289jSfPz82gQ37UFAmnfsOw="; + sha256 = "sha256-chZKesbmvGHXwhnJRZbXyX7B8OwJL9dJh0O1Axz/n2E="; }; doCheck = false; - cargoSha256 = "sha256-LXBQkiNU4vP6PIW+5iaEzonNGrpwnTMeVaosqLJoGCg="; + cargoSha256 = "sha256-FmsD3ajMqpPrTkXCX2anC+cmm0a2xuP+3FHqzj56Ma4="; }; } ); diff --git a/homestar-core/Cargo.toml b/homestar-core/Cargo.toml index aabf985a..f0d6e9d0 100644 --- a/homestar-core/Cargo.toml +++ b/homestar-core/Cargo.toml @@ -1,16 +1,15 @@ [package] name = "homestar-core" version = "0.1.0" -description = "" -keywords = [] -categories = [] - +description = "Homestar core library for working with tasks, instructions, etc" +keywords = ["ipld", "tasks", "receipts", "ipvm"] +categories = ["workflow-engines", "core", "libraries"] include = ["/src", "README.md", "LICENSE"] license = { workspace = true } readme = "README.md" edition = { workspace = true } rust-version = { workspace = true } -documentation = "https://docs.rs/homestar" +documentation = "https://docs.rs/homestar-core" repository = "https://github.com/ipvm-wg/homestar" authors = { workspace = true } @@ -21,18 +20,20 @@ doctest = true crate-type = ["cdylib", "rlib"] [dependencies] -anyhow = "1.0" +# return to version.workspace = true after the following issue is fixed: +# https://github.com/DevinR528/cargo-sort/issues/47 +anyhow = { workspace = true } diesel = { version = "2.0", features = ["sqlite"] } enum-as-inner = "0.5" enum-assoc = "0.4" generic-array = "0.14" +indexmap = "1.9" libipld = "0.16" proptest = { version = "1.1", optional = true } -semver = "1.0" serde = { version = "1.0", features = ["derive"] } signature = "2.0" thiserror = "1.0" -tracing = "0.1" +tracing = { workspace = true } ucan = "0.1" url = "2.3" xid = "1.0" @@ -43,3 +44,8 @@ criterion = "0.4" [features] default = [] test_utils = ["proptest"] + +[package.metadata.docs.rs] +all-features = true +# defines the configuration attribute `docsrs` +rustdoc-args = ["--cfg", "docsrs"] diff --git a/homestar-core/src/consts.rs b/homestar-core/src/consts.rs index 867da354..df94e5b3 100644 --- a/homestar-core/src/consts.rs +++ b/homestar-core/src/consts.rs @@ -1,4 +1,4 @@ //! Exported global constants. /// SemVer-formatted version of the UCAN Invocation Specification. -pub const VERSION: &str = "0.2.0"; +pub const INVOCATION_VERSION: &str = "0.2.0"; diff --git a/homestar-core/src/lib.rs b/homestar-core/src/lib.rs index 113af6fd..4596d257 100644 --- a/homestar-core/src/lib.rs +++ b/homestar-core/src/lib.rs @@ -2,13 +2,22 @@ #![warn(missing_debug_implementations, missing_docs, rust_2018_idioms)] #![deny(unreachable_pub, private_in_public)] -//! Homestar is a determistic Wasm runtime and effectful job system intended to -//! embed inside IPFS. -//! You can find a more complete description [here]. +//! `homestar-core` is the underlying foundation for all homestar +//! packages and implements much of the [Ucan invocation] and [IPVM] +//! specifications, among other useful library features. //! -//! [here]: https://github.com/ipvm-wg/spec. +//! +//! Related crates/packages: +//! +//! - [homestar-runtime] +//! - [homestar-wasm] +//! +//! [homestar-runtime]: +//! [homestar-wasm]: +//! [IPVM]: +//! [Ucan invocation]: -mod consts; +pub mod consts; #[cfg(any(test, feature = "test_utils"))] #[cfg_attr(docsrs, doc(cfg(feature = "test_utils")))] pub mod test_utils; diff --git a/homestar-core/src/test_utils/mod.rs b/homestar-core/src/test_utils/mod.rs index 3635b6eb..991f3bab 100644 --- a/homestar-core/src/test_utils/mod.rs +++ b/homestar-core/src/test_utils/mod.rs @@ -3,5 +3,8 @@ /// Random value generator for sampling data. #[cfg(feature = "test_utils")] mod rvg; +#[cfg(feature = "test_utils")] +pub mod workflow; + #[cfg(feature = "test_utils")] pub use rvg::*; diff --git a/homestar-core/src/test_utils/workflow.rs b/homestar-core/src/test_utils/workflow.rs new file mode 100644 index 00000000..be5a2ae3 --- /dev/null +++ b/homestar-core/src/test_utils/workflow.rs @@ -0,0 +1,159 @@ +//! Test utilities for working with [workflow] items. +//! +//! [workflow]: crate::workflow + +use crate::workflow::{ + pointer::{Await, AwaitResult}, + prf::UcanPrf, + Ability, Input, Instruction, InstructionResult, Nonce, Pointer, Receipt, +}; +use libipld::{ + cid::Cid, + multihash::{Code, MultihashDigest}, + Ipld, Link, +}; +use std::collections::BTreeMap; +use url::Url; + +const RAW: u64 = 0x55; + +type NonceBytes = Vec; + +/// Return a `mocked` `wasm/run` [Instruction]. +pub fn wasm_instruction<'a, T>() -> Instruction<'a, T> { + let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); + let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); + + Instruction::new( + resource, + Ability::from("wasm/run"), + Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ("args".into(), Ipld::List(vec![Ipld::Integer(1)])), + ]))), + ) +} + +/// Return `mocked` `wasm/run` [Instruction]'s, where the second is dependent on +/// the first. +pub fn related_wasm_instructions<'a, T>( +) -> (Instruction<'a, T>, Instruction<'a, T>, Instruction<'a, T>) +where + Ipld: From, + T: Clone, +{ + let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); + let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); + + let instr = Instruction::new( + resource.clone(), + Ability::from("wasm/run"), + Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ("args".into(), Ipld::List(vec![Ipld::Integer(1)])), + ]))), + ); + + let promise = Await::new( + Pointer::new(Cid::try_from(instr.clone()).unwrap()), + AwaitResult::Ok, + ); + + let dep_instr1 = Instruction::new( + resource.clone(), + Ability::from("wasm/run"), + Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ( + "args".into(), + Ipld::List(vec![Ipld::try_from(promise.clone()).unwrap()]), + ), + ]))), + ); + + let another_promise = Await::new( + Pointer::new(Cid::try_from(dep_instr1.clone()).unwrap()), + AwaitResult::Ok, + ); + + let dep_instr2 = Instruction::new( + resource, + Ability::from("wasm/run"), + Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_three".to_string())), + ( + "args".into(), + Ipld::List(vec![ + Ipld::try_from(another_promise).unwrap(), + Ipld::try_from(promise).unwrap(), + Ipld::Integer(42), + ]), + ), + ]))), + ); + + (instr, dep_instr1, dep_instr2) +} + +/// Return a `mocked` `wasm/run` [Instruction], along with it's [Nonce] as bytes. +pub fn wasm_instruction_with_nonce<'a, T>() -> (Instruction<'a, T>, NonceBytes) { + let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); + let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); + let nonce = Nonce::generate(); + + ( + Instruction::new_with_nonce( + resource, + Ability::from("wasm/run"), + Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ("args".into(), Ipld::List(vec![Ipld::Integer(1)])), + ]))), + nonce.clone(), + ), + nonce.as_nonce96().unwrap().to_vec(), + ) +} + +/// Return a `mocked` [Instruction]. +pub fn instruction<'a, T>() -> Instruction<'a, T> { + let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); + let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); + + Instruction::new( + resource, + Ability::from("ipld/fun"), + Input::Ipld(Ipld::List(vec![Ipld::Bool(true)])), + ) +} + +/// Return a `mocked` [Instruction], along with it's [Nonce] as bytes. +pub fn instruction_with_nonce<'a, T>() -> (Instruction<'a, T>, NonceBytes) { + let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); + let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); + let nonce = Nonce::generate(); + + ( + Instruction::new_with_nonce( + resource, + Ability::from("ipld/fun"), + Input::Ipld(Ipld::List(vec![Ipld::Bool(true)])), + nonce.clone(), + ), + nonce.as_nonce96().unwrap().to_vec(), + ) +} + +/// Return a `mocked` [Receipt] with an [Ipld] [InstructionResult]. +pub fn receipt() -> Receipt { + let h = Code::Blake3_256.digest(b"beep boop"); + let cid = Cid::new_v1(RAW, h); + let link: Link = Link::new(cid); + Receipt::new( + Pointer::new_from_link(link), + InstructionResult::Ok(Ipld::Bool(true)), + Ipld::Null, + None, + UcanPrf::default(), + ) +} diff --git a/homestar-core/src/workflow.rs b/homestar-core/src/workflow.rs new file mode 100644 index 00000000..68a34ca1 --- /dev/null +++ b/homestar-core/src/workflow.rs @@ -0,0 +1,34 @@ +//! Workflow and [Ucan invocation] componets for building Homestar pipelines. +//! +//! [Ucan invocation]: + +mod ability; +pub mod config; +pub mod input; +pub mod instruction; +mod instruction_result; +mod invocation; +mod issuer; +mod nonce; +pub mod pointer; +pub mod prf; +pub mod receipt; +pub mod task; + +pub use ability::*; +pub use input::Input; +pub use instruction::Instruction; +pub use instruction_result::*; +pub use invocation::*; +pub use issuer::Issuer; +pub use nonce::*; +pub use pointer::Pointer; +pub use receipt::Receipt; +pub use task::Task; + +/// Generic link, cid => T [IndexMap] for storing +/// invoked, raw values in-memory and using them to +/// resolve other steps within a runtime's workflow. +/// +/// [IndexMap]: indexmap::IndexMap +pub type LinkMap = indexmap::IndexMap; diff --git a/homestar-core/src/workflow/ability.rs b/homestar-core/src/workflow/ability.rs index 2385de76..421a6b41 100644 --- a/homestar-core/src/workflow/ability.rs +++ b/homestar-core/src/workflow/ability.rs @@ -1,7 +1,7 @@ //! [UCAN Ability] for a given [Resource]. //! //! [Resource]: url::Url -//! [UCAN Ability]: https://github.com/ucan-wg/spec/#23-ability +//! [UCAN Ability]: use libipld::{serde::from_ipld, Ipld}; use serde::{Deserialize, Serialize}; @@ -34,7 +34,7 @@ use std::{borrow::Cow, fmt}; /// assert_eq!(ability.to_string(), "example/test".to_string()); /// ``` /// -/// [UCAN Ability]: https://github.com/ucan-wg/spec/#23-ability +/// [UCAN Ability]: #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct Ability(String); diff --git a/homestar-core/src/workflow/input.rs b/homestar-core/src/workflow/input.rs index 12ebb4dd..174ce4b9 100644 --- a/homestar-core/src/workflow/input.rs +++ b/homestar-core/src/workflow/input.rs @@ -1,25 +1,17 @@ -//! Input paramters for [Task] execution and means to +//! Input paramters for [Instruction] execution and means to //! generally [parse] and [resolve] them. //! -//! [Task]: super::Task +//! [Instruction]: super::Instruction //! [parse]: Parse::parse //! [resolve]: Args::resolve use super::{ - pointer::{Await, AwaitResult, InvokedTaskPointer, ERR_BRANCH, OK_BRANCH, PTR_BRANCH}, - InvocationResult, + pointer::{Await, AwaitResult, ERR_BRANCH, OK_BRANCH, PTR_BRANCH}, + InstructionResult, Pointer, }; use anyhow::anyhow; use libipld::{serde::from_ipld, Cid, Ipld}; -use std::{ - collections::{btree_map::BTreeMap, HashMap}, - result::Result, -}; - -/// Generic link, cid => T [HashMap] for storing -/// invoked, raw values in-memory and using them to -/// resolve other steps within a runtime's workflow. -pub type LinkMap = HashMap; +use std::{collections::btree_map::BTreeMap, result::Result}; /// Parsed [Args] consisting of [Inputs] for execution flows, as well as an /// optional function name/definition. @@ -47,6 +39,21 @@ impl Parsed { fun: Some(fun), } } + + /// Parsed arguments. + pub fn args(&self) -> &Args { + &self.args + } + + /// Turn [Parsed] structure into owned [Args]. + pub fn into_args(self) -> Args { + self.args + } + + /// Parsed function named. + pub fn fun(&self) -> Option { + self.fun.as_ref().map(|f| f.to_string()) + } } impl From> for Args { @@ -55,15 +62,15 @@ impl From> for Args { } } -/// Interface for [Task] implementations, relying on `core` -/// to implement for custom parsing specifics. +/// Interface for [Instruction] implementations, relying on `homestore-core` +/// to implement custom parsing specifics. /// /// # Example /// /// ``` /// use homestar_core::{ /// workflow::{ -/// input::{Args, Parse}, Ability, Input, Task, +/// input::{Args, Parse}, Ability, Input, Instruction, /// }, /// Unit, /// }; @@ -73,19 +80,19 @@ impl From> for Args { /// let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); /// let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); /// -/// let task = Task::unique( +/// let inst = Instruction::unique( /// resource, /// Ability::from("wasm/run"), /// Input::::Ipld(Ipld::List(vec![Ipld::Bool(true)])) /// ); /// -/// let parsed = task.input().parse().unwrap(); +/// let parsed = inst.input().parse().unwrap(); /// /// // turn into Args for invocation: /// let args: Args = parsed.try_into().unwrap(); /// ``` /// -/// [Task]: super::Task +/// [Instruction]: super::Instruction pub trait Parse { /// Function returning [Parsed] structure for execution/invocation. /// @@ -98,10 +105,13 @@ pub trait Parse { #[derive(Clone, Debug, PartialEq)] pub struct Args(Vec>); -impl Args { +impl Args +where + T: std::fmt::Debug, +{ /// Create an [Args] [Vec]-type. pub fn new(args: Vec>) -> Self { - Args(args) + Self(args) } /// Return wrapped [Vec] of [inputs]. @@ -118,6 +128,18 @@ impl Args { &self.0 } + /// Return *only* deferred/awaited inputs. + pub fn deferreds(&self) -> Vec { + self.0.iter().fold(vec![], |mut acc, input| { + if let Input::Deferred(awaited_promise) = input { + acc.push(awaited_promise.instruction_cid()); + acc + } else { + acc + } + }) + } + /// Resolve [awaited promises] of [inputs] into task-specific [Input::Arg]'s, /// given a successful lookup function; otherwise, return [Input::Deferred] /// for unresolved promises, or just return [Input::Ipld], @@ -128,7 +150,7 @@ impl Args { /// [resolving Ipld links]: resolve_links pub fn resolve(self, lookup_fn: F) -> anyhow::Result where - F: Fn(Cid) -> anyhow::Result> + Clone, + F: Fn(Cid) -> anyhow::Result> + Clone, Ipld: From, T: Clone, { @@ -149,7 +171,7 @@ where impl TryFrom for Args where - InvocationResult: TryFrom, + InstructionResult: TryFrom, { type Error = anyhow::Error; @@ -158,7 +180,7 @@ where let args = vec .into_iter() .fold(Vec::>::new(), |mut acc, ipld| { - if let Ok(invocation_result) = InvocationResult::try_from(ipld.to_owned()) { + if let Ok(invocation_result) = InstructionResult::try_from(ipld.to_owned()) { acc.push(Input::Arg(invocation_result)); } else if let Ok(await_result) = Await::try_from(ipld.to_owned()) { acc.push(Input::Deferred(await_result)); @@ -185,15 +207,15 @@ where pub enum Input { /// [Ipld] Literals. Ipld(Ipld), - /// Promise-[links] awaiting the output of another [Task]'s invocation, - /// directly. + /// Promise-[links] awaiting the output of another [Instruction]'s + /// invocation, directly. /// - /// [links]: InvokedTaskPointer - /// [Task]: super::Task + /// [links]: Pointer + /// [Instruction]: super::Instruction Deferred(Await), - /// General argument, wrapping an [InvocationResult] over a task-specific + /// General argument, wrapping an [InstructionResult] over a task-specific /// implementation's own input type(s). - Arg(InvocationResult), + Arg(InstructionResult), } impl Input { @@ -208,13 +230,13 @@ impl Input { /// [resolving Ipld links]: resolve_links pub fn resolve(self, lookup_fn: F) -> Input where - F: Fn(Cid) -> anyhow::Result> + Clone, + F: Fn(Cid) -> anyhow::Result> + Clone, Ipld: From, { match self { Input::Ipld(ipld) => { if let Ok(await_promise) = Await::try_from(&ipld) { - if let Ok(func_ret) = lookup_fn(await_promise.task_cid()) { + if let Ok(func_ret) = lookup_fn(await_promise.instruction_cid()) { Input::Arg(func_ret) } else { Input::Deferred(await_promise) @@ -225,7 +247,7 @@ impl Input { } Input::Arg(ref _arg) => self, Input::Deferred(await_promise) => { - if let Ok(func_ret) = lookup_fn(await_promise.task_cid()) { + if let Ok(func_ret) = lookup_fn(await_promise.instruction_cid()) { Input::Arg(func_ret) } else { Input::Deferred(await_promise) @@ -273,15 +295,15 @@ where .or_else(|| map.get_key_value(ERR_BRANCH)) .or_else(|| map.get_key_value(PTR_BRANCH)) .map_or( - if let Ok(invocation_result) = InvocationResult::try_from(ipld.to_owned()) { + if let Ok(invocation_result) = InstructionResult::try_from(ipld.to_owned()) { Ok(Input::Arg(invocation_result)) } else { Ok(Input::Ipld(ipld)) }, |(branch, ipld)| { - let invoked_task = InvokedTaskPointer::try_from(ipld)?; + let instruction = Pointer::try_from(ipld)?; Ok(Input::Deferred(Await::new( - invoked_task, + instruction, AwaitResult::result(branch) .ok_or_else(|| anyhow!("wrong branch name: {branch}"))?, ))) @@ -292,7 +314,7 @@ where fn resolve_args(args: Vec>, lookup_fn: F) -> Vec> where - F: Fn(Cid) -> anyhow::Result> + Clone, + F: Fn(Cid) -> anyhow::Result> + Clone, Ipld: From, { let args = args.into_iter().map(|v| v.resolve(lookup_fn.clone())); @@ -304,7 +326,7 @@ where /// [awaited promises]: Await pub fn resolve_links(ipld: Ipld, lookup_fn: F) -> Ipld where - F: Fn(Cid) -> anyhow::Result> + Clone, + F: Fn(Cid) -> anyhow::Result> + Clone, Ipld: From, { match ipld { @@ -357,24 +379,7 @@ where #[cfg(test)] mod test { use super::*; - use crate::{ - workflow::{Ability, Nonce, Task}, - Unit, - }; - use url::Url; - - fn task<'a, T>() -> Task<'a, T> { - let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); - let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); - let nonce = Nonce::generate(); - - Task::new( - resource, - Ability::from("wasm/run"), - Input::Ipld(Ipld::List(vec![Ipld::Integer(88)])), - Some(nonce), - ) - } + use crate::{test_utils, Unit}; #[test] fn input_ipld_ipld_rountrip() { @@ -387,8 +392,8 @@ mod test { #[test] fn input_deferred_ipld_rountrip() { - let task: Task<'_, Unit> = task(); - let ptr: InvokedTaskPointer = task.try_into().unwrap(); + let instruction = test_utils::workflow::instruction::(); + let ptr: Pointer = instruction.try_into().unwrap(); let input: Input = Input::Deferred(Await::new(ptr.clone(), AwaitResult::Ptr)); let ipld = Ipld::from(input.clone()); @@ -401,7 +406,7 @@ mod test { #[test] fn input_arg_ipld_rountrip() { - let input: Input = Input::Arg(InvocationResult::Just(Ipld::Bool(false))); + let input: Input = Input::Arg(InstructionResult::Just(Ipld::Bool(false))); let ipld = Ipld::from(input.clone()); assert_eq!( diff --git a/homestar-core/src/workflow/instruction.rs b/homestar-core/src/workflow/instruction.rs new file mode 100644 index 00000000..efe66403 --- /dev/null +++ b/homestar-core/src/workflow/instruction.rs @@ -0,0 +1,363 @@ +//! An [Instruction] is the smallest unit of work that can be requested from a +//! UCAN, described via `resource`, `ability`. + +use super::{Ability, Input, Nonce, Pointer}; +use anyhow::anyhow; +use libipld::{ + cbor::DagCborCodec, + cid::{ + multibase::Base, + multihash::{Code, MultihashDigest}, + Cid, + }, + prelude::Codec, + serde::from_ipld, + Ipld, +}; +use std::{borrow::Cow, collections::BTreeMap, fmt}; +use url::Url; + +const DAG_CBOR: u64 = 0x71; +const RESOURCE_KEY: &str = "rsc"; +const OP_KEY: &str = "op"; +const INPUT_KEY: &str = "input"; +const NNC_KEY: &str = "nnc"; + +/// Enumerator for `either` an expanded [Instruction] structure or +/// an [Pointer] ([Cid] wrapper). +#[derive(Debug, Clone, PartialEq)] +pub enum RunInstruction<'a, T> { + /// [Instruction] as an expanded structure. + Expanded(Instruction<'a, T>), + /// [Instruction] as a pointer. + Ptr(Pointer), +} + +impl<'a, T> From> for RunInstruction<'a, T> { + fn from(instruction: Instruction<'a, T>) -> Self { + RunInstruction::Expanded(instruction) + } +} + +impl<'a, T> TryFrom> for Instruction<'a, T> +where + T: fmt::Debug, +{ + type Error = anyhow::Error; + + fn try_from(run: RunInstruction<'a, T>) -> Result { + match run { + RunInstruction::Expanded(instruction) => Ok(instruction), + e => Err(anyhow!("wrong discriminant: {e:?}")), + } + } +} + +impl From for RunInstruction<'_, T> { + fn from(ptr: Pointer) -> Self { + RunInstruction::Ptr(ptr) + } +} + +impl<'a, T> TryFrom> for Pointer +where + T: fmt::Debug, +{ + type Error = anyhow::Error; + + fn try_from(run: RunInstruction<'a, T>) -> Result { + match run { + RunInstruction::Ptr(ptr) => Ok(ptr), + e => Err(anyhow!("wrong discriminant: {e:?}")), + } + } +} + +impl<'a, 'b, T> TryFrom<&'b RunInstruction<'a, T>> for &'b Pointer +where + T: fmt::Debug, +{ + type Error = anyhow::Error; + + fn try_from(run: &'b RunInstruction<'a, T>) -> Result { + match run { + RunInstruction::Ptr(ptr) => Ok(ptr), + e => Err(anyhow!("wrong discriminant: {e:?}")), + } + } +} + +impl<'a, 'b, T> TryFrom<&'b RunInstruction<'a, T>> for Pointer +where + T: fmt::Debug, +{ + type Error = anyhow::Error; + + fn try_from(run: &'b RunInstruction<'a, T>) -> Result { + match run { + RunInstruction::Ptr(ptr) => Ok(ptr.to_owned()), + e => Err(anyhow!("wrong discriminant: {e:?}")), + } + } +} + +impl From> for Ipld +where + Ipld: From, +{ + fn from(run: RunInstruction<'_, T>) -> Self { + match run { + RunInstruction::Expanded(instruction) => instruction.into(), + RunInstruction::Ptr(instruction_ptr) => instruction_ptr.into(), + } + } +} + +impl TryFrom for RunInstruction<'_, T> +where + T: From, +{ + type Error = anyhow::Error; + + fn try_from<'a>(ipld: Ipld) -> Result { + match ipld { + Ipld::Map(_) => Ok(RunInstruction::Expanded(Instruction::try_from(ipld)?)), + Ipld::Link(_) => Ok(RunInstruction::Ptr(Pointer::try_from(ipld)?)), + _ => Err(anyhow!("unexpected conversion type")), + } + } +} + +/// +/// +/// # Example +/// +/// ``` +/// use homestar_core::{Unit, workflow::{Ability, Input, Instruction}}; +/// use libipld::Ipld; +/// use url::Url; +/// +/// let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); +/// let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); +/// +/// let instr = Instruction::unique( +/// resource, +/// Ability::from("wasm/run"), +/// Input::::Ipld(Ipld::List(vec![Ipld::Bool(true)])) +/// ); +/// ``` +/// +/// We can also set-up an [Instruction] with a Deferred input to await on: +/// ``` +/// use homestar_core::{ +/// workflow::{ +/// pointer::{Await, AwaitResult}, +/// Ability, Input, Instruction, Nonce, Pointer, +/// }, +/// Unit, +/// }; +/// use libipld::{cid::{multihash::{Code, MultihashDigest}, Cid}, Ipld, Link}; +/// use url::Url; + +/// let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); +/// let resource = Url::parse(format!("ipfs://{wasm}").as_str()).expect("IPFS URL"); +/// let h = Code::Blake3_256.digest(b"beep boop"); +/// let cid = Cid::new_v1(0x55, h); +/// let link: Link = Link::new(cid); +/// let awaited_instr = Pointer::new_from_link(link); +/// +/// let instr = Instruction::new_with_nonce( +/// resource, +/// Ability::from("wasm/run"), +/// Input::::Deferred(Await::new(awaited_instr, AwaitResult::Ok)), +/// Nonce::generate() +/// ); +/// +/// // And covert it to a pointer: +/// let ptr = Pointer::try_from(instr).unwrap(); +/// ``` +/// [deferred promise]: super::pointer::Await +#[derive(Clone, Debug, PartialEq)] +pub struct Instruction<'a, T> { + rsc: Url, + op: Cow<'a, Ability>, + input: Input, + nnc: Nonce, +} + +impl Instruction<'_, T> { + /// Create a new [Instruction] with an empty Nonce. + pub fn new(rsc: Url, ability: Ability, input: Input) -> Self { + Self { + rsc, + op: Cow::from(ability), + input, + nnc: Nonce::Empty, + } + } + + /// Create a new [Instruction] with a given [Nonce]. + pub fn new_with_nonce(rsc: Url, ability: Ability, input: Input, nnc: Nonce) -> Self { + Self { + rsc, + op: Cow::from(ability), + input, + nnc, + } + } + + /// Create a unique [Instruction], with a default [Nonce] generator. + pub fn unique(rsc: Url, ability: Ability, input: Input) -> Self { + Self { + rsc, + op: Cow::from(ability), + input, + nnc: Nonce::generate(), + } + } + + /// Return [Instruction] resource, i.e. [Url]. + pub fn resource(&self) -> &Url { + &self.rsc + } + + /// Return [Ability] associated with `op`. + pub fn op(&self) -> &Ability { + &self.op + } + + /// Return [Instruction] [Input]. + pub fn input(&self) -> &Input { + &self.input + } + + /// Return [Nonce] reference. + pub fn nonce(&self) -> &Nonce { + &self.nnc + } +} + +impl TryFrom> for Pointer +where + Ipld: From, +{ + type Error = anyhow::Error; + + fn try_from(instruction: Instruction<'_, T>) -> Result { + Ok(Pointer::new(Cid::try_from(instruction)?)) + } +} + +impl TryFrom> for Cid +where + Ipld: From, +{ + type Error = anyhow::Error; + + fn try_from(instruction: Instruction<'_, T>) -> Result { + let ipld: Ipld = instruction.into(); + let bytes = DagCborCodec.encode(&ipld)?; + let hash = Code::Sha3_256.digest(&bytes); + Ok(Cid::new_v1(DAG_CBOR, hash)) + } +} + +impl From> for Ipld +where + Ipld: From, +{ + fn from(instruction: Instruction<'_, T>) -> Self { + Ipld::Map(BTreeMap::from([ + (RESOURCE_KEY.into(), instruction.rsc.to_string().into()), + (OP_KEY.into(), instruction.op.to_string().into()), + (INPUT_KEY.into(), instruction.input.into()), + (NNC_KEY.into(), instruction.nnc.into()), + ])) + } +} + +impl TryFrom<&Ipld> for Instruction<'_, T> +where + T: From, +{ + type Error = anyhow::Error; + + fn try_from(ipld: &Ipld) -> Result { + TryFrom::try_from(ipld.to_owned()) + } +} + +impl TryFrom for Instruction<'_, T> +where + T: From, +{ + type Error = anyhow::Error; + + fn try_from(ipld: Ipld) -> Result { + let map = from_ipld::>(ipld)?; + + let rsc = match map.get(RESOURCE_KEY) { + Some(Ipld::Link(cid)) => cid + .to_string_of_base(Base::Base32Lower) + .map_err(|e| anyhow!("failed to encode CID into multibase string: {e}")) + .and_then(|txt| { + Url::parse(format!("{}{}", "ipfs://", txt).as_str()) + .map_err(|e| anyhow!("failed to parse URL: {e}")) + }), + Some(Ipld::String(txt)) => { + Url::parse(txt.as_str()).map_err(|e| anyhow!("failed to parse URL: {e}")) + } + _ => Err(anyhow!("no resource/with set.")), + }?; + + Ok(Self { + rsc, + op: from_ipld( + map.get(OP_KEY) + .ok_or_else(|| anyhow!("no `op` field set"))? + .to_owned(), + )?, + input: Input::try_from( + map.get(INPUT_KEY) + .ok_or_else(|| anyhow!("no `input` field set"))? + .to_owned(), + )?, + nnc: Nonce::try_from( + map.get(NNC_KEY) + .unwrap_or(&Ipld::String("".to_string())) + .to_owned(), + )?, + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{test_utils, Unit}; + + #[test] + fn ipld_roundtrip() { + let (instruction, bytes) = test_utils::workflow::instruction_with_nonce::(); + let ipld = Ipld::from(instruction.clone()); + + assert_eq!( + ipld, + Ipld::Map(BTreeMap::from([ + ( + RESOURCE_KEY.into(), + Ipld::String( + "ipfs://bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".into() + ) + ), + (OP_KEY.into(), Ipld::String("ipld/fun".to_string())), + (INPUT_KEY.into(), Ipld::List(vec![Ipld::Bool(true)])), + ( + NNC_KEY.into(), + Ipld::List(vec![Ipld::Integer(0), Ipld::Bytes(bytes)]) + ) + ])) + ); + assert_eq!(instruction, ipld.try_into().unwrap()) + } +} diff --git a/homestar-core/src/workflow/invocation_result.rs b/homestar-core/src/workflow/instruction_result.rs similarity index 63% rename from homestar-core/src/workflow/invocation_result.rs rename to homestar-core/src/workflow/instruction_result.rs index 25529f37..0823f72e 100644 --- a/homestar-core/src/workflow/invocation_result.rs +++ b/homestar-core/src/workflow/instruction_result.rs @@ -1,7 +1,7 @@ -//! The output `Result` of a [Task], as a `success` (`Ok`) / `failure` (`Error`) -//! state. +//! The output `Result` of an [Instruction], tagged as a `success` (`Ok`) or +//! `failure` (`Error`), or returned/inlined directly. //! -//! [Task]: super::Task +//! [Instruction]: super::Instruction use anyhow::anyhow; use diesel::{ @@ -19,30 +19,31 @@ const OK: &str = "ok"; const ERR: &str = "error"; const JUST: &str = "just"; -/// Resultant output of an executed [Task]. +/// Resultant output of an executed [Instruction]. /// -/// [Task]: super::Task +/// [Instruction]: super::Instruction #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, AsExpression, FromSqlRow)] #[diesel(sql_type = Binary)] -pub enum InvocationResult { +pub enum InstructionResult { /// `Ok` branch. Ok(T), /// `Error` branch. Error(T), /// `Just` branch, meaning `just the value`. Used for - /// not incorporating ok/error into arg/param. + /// not incorporating unwrapped ok/error into arg/param, where a + /// result may show up directly. Just(T), } -impl InvocationResult { +impl InstructionResult { /// Owned, inner result of a [Task] invocation. /// /// [Task]: super::Task pub fn into_inner(self) -> T { match self { - InvocationResult::Ok(inner) => inner, - InvocationResult::Error(inner) => inner, - InvocationResult::Just(inner) => inner, + InstructionResult::Ok(inner) => inner, + InstructionResult::Error(inner) => inner, + InstructionResult::Just(inner) => inner, } } @@ -51,27 +52,27 @@ impl InvocationResult { /// [Task]: super::Task pub fn inner(&self) -> &T { match self { - InvocationResult::Ok(inner) => inner, - InvocationResult::Error(inner) => inner, - InvocationResult::Just(inner) => inner, + InstructionResult::Ok(inner) => inner, + InstructionResult::Error(inner) => inner, + InstructionResult::Just(inner) => inner, } } } -impl From> for Ipld +impl From> for Ipld where Ipld: From, { - fn from(result: InvocationResult) -> Self { + fn from(result: InstructionResult) -> Self { match result { - InvocationResult::Ok(res) => Ipld::List(vec![OK.into(), res.into()]), - InvocationResult::Error(res) => Ipld::List(vec![ERR.into(), res.into()]), - InvocationResult::Just(res) => Ipld::List(vec![JUST.into(), res.into()]), + InstructionResult::Ok(res) => Ipld::List(vec![OK.into(), res.into()]), + InstructionResult::Error(res) => Ipld::List(vec![ERR.into(), res.into()]), + InstructionResult::Just(res) => Ipld::List(vec![JUST.into(), res.into()]), } } } -impl TryFrom for InvocationResult +impl TryFrom for InstructionResult where T: From, { @@ -81,13 +82,13 @@ where if let Ipld::List(v) = ipld { match &v[..] { [Ipld::String(result), res] if result == OK => { - Ok(InvocationResult::Ok(res.to_owned().try_into()?)) + Ok(InstructionResult::Ok(res.to_owned().try_into()?)) } [Ipld::String(result), res] if result == ERR => { - Ok(InvocationResult::Error(res.to_owned().try_into()?)) + Ok(InstructionResult::Error(res.to_owned().try_into()?)) } [Ipld::String(result), res] if result == JUST => { - Ok(InvocationResult::Just(res.to_owned().try_into()?)) + Ok(InstructionResult::Just(res.to_owned().try_into()?)) } _ => Err(anyhow!("unexpected conversion type")), } @@ -97,7 +98,7 @@ where } } -impl TryFrom<&Ipld> for InvocationResult +impl TryFrom<&Ipld> for InstructionResult where T: From, { @@ -109,7 +110,7 @@ where } /// Diesel, [Sqlite] [ToSql] implementation. -impl ToSql for InvocationResult +impl ToSql for InstructionResult where [u8]: ToSql, { @@ -121,12 +122,12 @@ where } /// Diesel, [Sqlite] [FromSql] implementation. -impl FromSql for InvocationResult { +impl FromSql for InstructionResult { fn from_sql(bytes: RawValue<'_, Sqlite>) -> deserialize::Result { let raw_bytes = <*const [u8] as FromSql>::from_sql(bytes)?; let raw_bytes: &[u8] = unsafe { &*raw_bytes }; let decoded: Ipld = DagCborCodec.decode(raw_bytes)?; - Ok(InvocationResult::try_from(decoded)?) + Ok(InstructionResult::try_from(decoded)?) } } @@ -136,9 +137,9 @@ mod test { #[test] fn ipld_roundtrip() { - let res1 = InvocationResult::Error(Ipld::String("bad stuff".to_string())); - let res2 = InvocationResult::Ok(Ipld::String("ok stuff".to_string())); - let res3 = InvocationResult::Just(Ipld::String("just the right stuff".to_string())); + let res1 = InstructionResult::Error(Ipld::String("bad stuff".to_string())); + let res2 = InstructionResult::Ok(Ipld::String("ok stuff".to_string())); + let res3 = InstructionResult::Just(Ipld::String("just the right stuff".to_string())); let ipld1 = Ipld::from(res1.clone()); let ipld2 = Ipld::from(res2.clone()); let ipld3 = Ipld::from(res3.clone()); diff --git a/homestar-core/src/workflow/invocation.rs b/homestar-core/src/workflow/invocation.rs index 2f599f0d..e3e75810 100644 --- a/homestar-core/src/workflow/invocation.rs +++ b/homestar-core/src/workflow/invocation.rs @@ -1,14 +1,8 @@ -//! [Invocation] container for running a [Task] or Task(s). +//! [Invocation] is a signed [Task]. //! //! [Task]: super::Task -use crate::{ - consts::VERSION, - workflow::{ - pointer::{InvocationPointer, InvokedTaskPointer}, - prf::UcanPrf, - task::RunTask, - }, -}; + +use super::{Pointer, Task}; use anyhow::anyhow; use libipld::{ cbor::DagCborCodec, @@ -20,87 +14,33 @@ use libipld::{ serde::from_ipld, Ipld, }; -use semver::Version; use std::collections::BTreeMap; -const VERSION_KEY: &str = "v"; -const RUN_KEY: &str = "run"; -const CAUSE_KEY: &str = "cause"; -const METADATA_KEY: &str = "meta"; -const PROOF_KEY: &str = "prf"; +const DAG_CBOR: u64 = 0x71; +const TASK_KEY: &str = "task"; -/// An Invocation is an instruction to the [Executor] to perform enclosed -/// [Task]. -/// -/// Invocations are not executable until they have been provided provable -/// authority (in form of UCANs in the [prf] field) and an Authorization. -/// -/// [Executor]: https://github.com/ucan-wg/invocation#212-executor -/// [Task]: super::Task -/// [prf]: super::prf +/// A signed [Task] wrapper/container. #[derive(Debug, Clone, PartialEq)] pub struct Invocation<'a, T> { - v: Version, - run: RunTask<'a, T>, - cause: Option, - meta: Ipld, - prf: UcanPrf, + task: Task<'a, T>, } -impl<'a, T> Invocation<'a, T> +impl<'a, T> From> for Invocation<'a, T> where Ipld: From, - T: Clone, { - /// Generate a new [Invocation] to run, with metadata, and `prf`. - pub fn new(run: RunTask<'a, T>, meta: Ipld, prf: UcanPrf) -> anyhow::Result { - let invok = Invocation { - v: Version::parse(VERSION)?, - run, - cause: None, - meta, - prf, - }; - - Ok(invok) - } - - /// Generate a new [Invocation] to run, with metadata, given a [cause], and - /// `prf`. - /// - /// [cause]: https://github.com/ucan-wg/invocation#523-cause - pub fn new_with_cause( - run: RunTask<'a, T>, - meta: Ipld, - prf: UcanPrf, - cause: Option, - ) -> anyhow::Result { - let invok = Invocation { - v: Version::parse(VERSION)?, - run, - cause, - meta, - prf, - }; - - Ok(invok) - } - - /// Return a reference pointer to given [Task] to run. - /// - /// [Task]: super::Task - pub fn run(&self) -> &RunTask<'_, T> { - &self.run + fn from(task: Task<'a, T>) -> Self { + Invocation::new(task) } +} - /// Return the [Cid] of the [Task] to run. - /// - /// [Task]: super::Task - pub fn task_cid(&self) -> anyhow::Result { - match &self.run { - RunTask::Expanded(task) => Ok(InvokedTaskPointer::try_from(task.to_owned())?.cid()), - RunTask::Ptr(taskptr) => Ok(taskptr.cid()), - } +impl<'a, T> Invocation<'a, T> +where + Ipld: From, +{ + /// Create a new [Invocation] container. + pub fn new(task: Task<'a, T>) -> Self { + Self { task } } } @@ -111,16 +51,10 @@ where type Error = anyhow::Error; fn try_from(invocation: Invocation<'_, T>) -> Result { - let map = Ipld::Map(BTreeMap::from([ - (VERSION_KEY.into(), invocation.v.to_string().into()), - (RUN_KEY.into(), invocation.run.try_into()?), - ( - CAUSE_KEY.into(), - invocation.cause.map_or(Ok(Ipld::Null), Ipld::try_from)?, - ), - (METADATA_KEY.into(), invocation.meta), - (PROOF_KEY.into(), invocation.prf.into()), - ])); + let map = Ipld::Map(BTreeMap::from([( + TASK_KEY.into(), + invocation.task.try_into()?, + )])); Ok(map) } @@ -135,58 +69,24 @@ where fn try_from(ipld: Ipld) -> Result { let map = from_ipld::>(ipld)?; - Ok(Invocation { - v: from_ipld::( - map.get(VERSION_KEY) - .ok_or_else(|| anyhow!("no `version` field set"))? - .to_owned(), - ) - .map(|s| Version::parse(&s))??, - run: RunTask::try_from( - map.get(RUN_KEY) - .ok_or_else(|| anyhow!("no `run` set"))? - .to_owned(), - )?, - cause: map - .get(CAUSE_KEY) - .and_then(|ipld| match ipld { - Ipld::Null => None, - ipld => Some(ipld), - }) - .and_then(|ipld| ipld.try_into().ok()), - meta: map - .get(METADATA_KEY) - .ok_or_else(|| anyhow!("no `metadata` field set"))? - .to_owned(), - prf: UcanPrf::try_from( - map.get(PROOF_KEY) - .ok_or_else(|| anyhow!("no proof field set"))? + Ok(Self { + task: Task::try_from( + map.get(TASK_KEY) + .ok_or_else(|| anyhow!("no `task` set"))? .to_owned(), )?, }) } } -impl TryFrom<&Ipld> for Invocation<'_, T> -where - T: From, -{ - type Error = anyhow::Error; - - fn try_from<'a>(ipld: &Ipld) -> Result { - TryFrom::try_from(ipld.to_owned()) - } -} - -impl TryFrom> for InvocationPointer +impl TryFrom> for Pointer where Ipld: From, - T: Clone, { type Error = anyhow::Error; fn try_from(invocation: Invocation<'_, T>) -> Result { - Ok(InvocationPointer::new(Cid::try_from(invocation)?)) + Ok(Pointer::new(Cid::try_from(invocation)?)) } } @@ -200,7 +100,7 @@ where let ipld: Ipld = invocation.try_into()?; let bytes = DagCborCodec.encode(&ipld)?; let hash = Code::Sha3_256.digest(&bytes); - Ok(Cid::new_v1(0x71, hash)) + Ok(Cid::new_v1(DAG_CBOR, hash)) } } @@ -208,96 +108,23 @@ where mod test { use super::*; use crate::{ - workflow::{config::Resources, Ability, Input, Task}, - Unit, VERSION, + test_utils, + workflow::{config::Resources, instruction::RunInstruction, prf::UcanPrf, Task}, + Unit, }; - use url::Url; - - fn task<'a, T>() -> Task<'a, T> { - let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); - let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); - - Task::new( - resource, - Ability::from("wasm/run"), - Input::Ipld(Ipld::List(vec![Ipld::Bool(true)])), - None, - ) - } #[test] fn ipld_roundtrip() { - let task: Task<'_, Unit> = task(); - let config = Resources::default(); - let invocation1 = Invocation::new( - RunTask::Expanded(task.clone()), - config.clone().into(), - UcanPrf::default(), - ) - .unwrap(); - - let ipld1 = Ipld::try_from(invocation1.clone()).unwrap(); - - let ipld_task = Ipld::Map(BTreeMap::from([ - ( - "on".into(), - Ipld::String( - "ipfs://bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".into(), - ), - ), - ("call".into(), Ipld::String("wasm/run".to_string())), - ("input".into(), Ipld::List(vec![Ipld::Bool(true)])), - ("nnc".into(), Ipld::Null), - ])); - - assert_eq!( - ipld1, - Ipld::Map(BTreeMap::from([ - (VERSION_KEY.into(), Ipld::String(VERSION.into())), - (RUN_KEY.into(), ipld_task), - (CAUSE_KEY.into(), Ipld::Null), - ( - METADATA_KEY.into(), - Ipld::Map(BTreeMap::from([ - ("fuel".into(), Ipld::Integer(u64::MAX.into())), - ("time".into(), Ipld::Integer(100_000)) - ])) - ), - (PROOF_KEY.into(), Ipld::List(vec![])) - ])) - ); - - assert_eq!(invocation1, ipld1.try_into().unwrap()); - - let invocation2 = Invocation::new_with_cause( - RunTask::Ptr::(task.try_into().unwrap()), + let instruction = test_utils::workflow::instruction::(); + let task = Task::new( + RunInstruction::Expanded(instruction.clone()), config.into(), UcanPrf::default(), - Some(InvocationPointer::try_from(invocation1.clone()).unwrap()), - ) - .unwrap(); - - let ipld2 = Ipld::try_from(invocation2.clone()).unwrap(); - let invocation1_ptr: InvocationPointer = invocation1.try_into().unwrap(); - - assert_eq!( - ipld2, - Ipld::Map(BTreeMap::from([ - (VERSION_KEY.into(), Ipld::String(VERSION.into())), - (RUN_KEY.into(), Ipld::Link(invocation2.task_cid().unwrap())), - (CAUSE_KEY.into(), Ipld::Link(invocation1_ptr.cid())), - ( - METADATA_KEY.into(), - Ipld::Map(BTreeMap::from([ - ("fuel".into(), Ipld::Integer(u64::MAX.into())), - ("time".into(), Ipld::Integer(100_000)) - ])) - ), - (PROOF_KEY.into(), Ipld::List(vec![])) - ])) ); - assert_eq!(invocation2, ipld2.try_into().unwrap()); + let invocation = Invocation::new(task); + let ipld = Ipld::try_from(invocation.clone()).unwrap(); + assert_eq!(invocation, Invocation::try_from(ipld).unwrap()); } } diff --git a/homestar-core/src/workflow/issuer.rs b/homestar-core/src/workflow/issuer.rs new file mode 100644 index 00000000..c8574e24 --- /dev/null +++ b/homestar-core/src/workflow/issuer.rs @@ -0,0 +1,69 @@ +//! Issuer referring to a principal (principal of least authority) that issues +//! a receipt. + +use diesel::{ + backend::RawValue, + deserialize::{self, FromSql}, + serialize::{self, IsNull, Output, ToSql}, + sql_types::Text, + sqlite::Sqlite, + AsExpression, FromSqlRow, +}; +use libipld::{serde::from_ipld, Ipld}; +use serde::{Deserialize, Serialize}; +use std::{fmt, str::FromStr}; +use ucan::ipld::Principle; + +/// [Principal] issuer of a receipt. If omitted issuer is +/// inferred from the [invocation] [task] audience. +/// +/// [invocation]: super::Invocation +/// [task]: super::Task +/// [Principal]: Principle +#[derive(Clone, Debug, Deserialize, Serialize, AsExpression, FromSqlRow, PartialEq)] +#[diesel(sql_type = Text)] +pub struct Issuer(Principle); + +impl Issuer { + /// Create a new [Issuer], wrapping a [Principle]. + pub fn new(principle: Principle) -> Self { + Issuer(principle) + } +} + +impl fmt::Display for Issuer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let did_as_string = self.0.to_string(); + write!(f, "{did_as_string}") + } +} + +impl From for Ipld { + fn from(issuer: Issuer) -> Self { + let principle = issuer.0.to_string(); + Ipld::String(principle) + } +} + +impl TryFrom for Issuer { + type Error = anyhow::Error; + + fn try_from(ipld: Ipld) -> Result { + let s = from_ipld::(ipld)?; + Ok(Issuer(Principle::from_str(&s)?)) + } +} + +impl ToSql for Issuer { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { + out.set_value(self.0.to_string()); + Ok(IsNull::No) + } +} + +impl FromSql for Issuer { + fn from_sql(bytes: RawValue<'_, Sqlite>) -> deserialize::Result { + let s = >::from_sql(bytes)?; + Ok(Issuer(Principle::from_str(&s)?)) + } +} diff --git a/homestar-core/src/workflow/mod.rs b/homestar-core/src/workflow/mod.rs deleted file mode 100644 index 9f3c97bc..00000000 --- a/homestar-core/src/workflow/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Workflow componets for building Homestar pipelines. - -mod ability; -pub mod config; -pub mod input; -mod invocation; -mod invocation_result; -mod nonce; -pub mod pointer; -pub mod prf; -pub mod receipt; -pub mod task; - -pub use ability::*; -pub use input::Input; -pub use invocation::*; -pub use invocation_result::*; -pub use nonce::*; -pub use task::Task; diff --git a/homestar-core/src/workflow/nonce.rs b/homestar-core/src/workflow/nonce.rs index 78c97d52..4603236b 100644 --- a/homestar-core/src/workflow/nonce.rs +++ b/homestar-core/src/workflow/nonce.rs @@ -1,6 +1,6 @@ -//! [Task] Nonce parameter. +//! [Instruction] nonce parameter. //! -//! [Task]: super::Task +//! [Instruction]: super::Instruction use anyhow::anyhow; use enum_as_inner::EnumAsInner; @@ -8,7 +8,8 @@ use generic_array::{ typenum::consts::{U12, U16}, GenericArray, }; -use libipld::Ipld; +use libipld::{multibase::Base::Base32HexLower, Ipld}; +use std::fmt; type Nonce96 = GenericArray; type Nonce128 = GenericArray; @@ -20,6 +21,8 @@ pub enum Nonce { Nonce96(Nonce96), /// 129-bit, 16-byte nonce. Nonce128(Nonce128), + /// No Nonce attributed. + Empty, } impl Nonce { @@ -29,6 +32,20 @@ impl Nonce { } } +impl fmt::Display for Nonce { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Nonce::Nonce96(nonce) => { + write!(f, "{}", Base32HexLower.encode(nonce.as_slice())) + } + Nonce::Nonce128(nonce) => { + write!(f, "{}", Base32HexLower.encode(nonce.as_slice())) + } + Nonce::Empty => write!(f, ""), + } + } +} + impl From for Ipld { fn from(nonce: Nonce) -> Self { match nonce { @@ -38,6 +55,7 @@ impl From for Ipld { Nonce::Nonce128(nonce) => { Ipld::List(vec![Ipld::Integer(1), Ipld::Bytes(nonce.to_vec())]) } + Nonce::Empty => Ipld::String("".to_string()), } } } @@ -57,7 +75,7 @@ impl TryFrom for Nonce { _ => Err(anyhow!("unexpected conversion type")), } } else { - Err(anyhow!("mismatched conversion type: {ipld:?}")) + Ok(Nonce::Empty) } } } diff --git a/homestar-core/src/workflow/pointer.rs b/homestar-core/src/workflow/pointer.rs index 301c1a49..f6935808 100644 --- a/homestar-core/src/workflow/pointer.rs +++ b/homestar-core/src/workflow/pointer.rs @@ -1,9 +1,12 @@ #![allow(missing_docs)] -//! Pointers and references to [Invocations] and [Tasks]. +//! Pointers and references to [Invocations], [Tasks], [Instructions], and/or +//! [Receipts], as well as handling for the [Await]'ed promises of pointers. //! //! [Invocations]: super::Invocation //! [Tasks]: super::Task +//! [Instructions]: super::Instruction +//! [Receipts]: super::Receipt use anyhow::ensure; use diesel::{ @@ -23,20 +26,13 @@ use libipld::{ use serde::{Deserialize, Serialize}; use std::{borrow::Cow, collections::btree_map::BTreeMap, fmt, str::FromStr}; -/// `await/ok` branch for a task invocation. +/// `await/ok` branch for instruction result. pub const OK_BRANCH: &str = "await/ok"; -/// `await/error` branch for a task invocation. +/// `await/error` branch for instruction result. pub const ERR_BRANCH: &str = "await/error"; -/// `await/*` branch for a task invocation. +/// `await/*` branch for instruction result. pub const PTR_BRANCH: &str = "await/*"; -/// Type alias around [InvocationPointer] for [Task] pointers. -/// -/// Essentially, reusing [InvocationPointer] as a [Cid] wrapper. -/// -/// [Task]: super::Task -pub type InvokedTaskPointer = InvocationPointer; - /// Enumerated wrapper around resulting branches of a promise /// that's being awaited on. /// @@ -63,7 +59,7 @@ pub enum AwaitResult { #[assoc(branch = ERR_BRANCH)] #[assoc(result = ERR_BRANCH)] Error, - /// + /// Direct resulting branch, without unwrapping of success or failure. #[assoc(branch = PTR_BRANCH)] #[assoc(result = PTR_BRANCH)] Ptr, @@ -79,29 +75,29 @@ impl fmt::Display for AwaitResult { } } -/// Describes the eventual output of the referenced [Task invocation], either -/// resolving to [OK_BRANCH], [ERR_BRANCH], or [PTR_BRANCH]. +/// Describes the eventual output of the referenced [Instruction] as a +/// [Pointer], either resolving to a tagged [OK_BRANCH], [ERR_BRANCH], or direct +/// result of a [PTR_BRANCH]. /// -/// [Task invocation]: InvokedTaskPointer +/// [Instruction]: super::Instruction #[derive(Clone, Debug, PartialEq, Eq)] pub struct Await { - invoked_task: InvokedTaskPointer, + instruction: Pointer, result: AwaitResult, } impl Await { - /// A new `Promise` [Await]'ed on, resulting in a [InvokedTaskPointer] + /// A new `Promise` [Await]'ed on, resulting in a [Pointer] /// and [AwaitResult]. - pub fn new(invoked: InvokedTaskPointer, result: AwaitResult) -> Self { - Await { - invoked_task: invoked, + pub fn new(instruction: Pointer, result: AwaitResult) -> Self { + Self { + instruction, result, } } - /// Return [Cid] for [InvokedTaskPointer]. - pub fn task_cid(&self) -> Cid { - self.invoked_task.cid() + pub fn instruction_cid(&self) -> Cid { + self.instruction.cid() } /// Return [AwaitResult] branch. @@ -114,7 +110,7 @@ impl From for Ipld { fn from(await_promise: Await) -> Self { Ipld::Map(BTreeMap::from([( await_promise.result.branch().to_string(), - await_promise.invoked_task.into(), + await_promise.instruction.into(), )])) } } @@ -133,7 +129,7 @@ impl TryFrom for Await { ensure!(map.len() == 1, "unexpected keys inside awaited promise"); let (key, value) = map.into_iter().next().unwrap(); - let invoked_task = InvokedTaskPointer::try_from(value)?; + let instruction = Pointer::try_from(value)?; let result = match key.as_str() { OK_BRANCH => AwaitResult::Ok, @@ -142,7 +138,7 @@ impl TryFrom for Await { }; Ok(Await { - invoked_task, + instruction, result, }) } @@ -156,14 +152,18 @@ impl TryFrom<&Ipld> for Await { } } -/// References a specific [Invocation], always by Cid. +/// References a specific [Invocation], [Task], [Instruction], and/or +/// [Receipt], always wrapping a [Cid]. /// /// [Invocation]: super::Invocation +/// [Task]: super::Task +/// [Instruction]: super::Instruction +/// [Receipt]: super::Receipt #[derive(Clone, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Serialize, Deserialize)] #[diesel(sql_type = Text)] -pub struct InvocationPointer(Cid); +pub struct Pointer(Cid); -impl fmt::Display for InvocationPointer { +impl fmt::Display for Pointer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let cid_as_string = self .0 @@ -174,39 +174,39 @@ impl fmt::Display for InvocationPointer { } } -impl InvocationPointer { - /// Return the `inner` [Cid] for the [InvocationPointer]. +impl Pointer { + /// Return the `inner` [Cid] for the [Pointer]. pub fn cid(&self) -> Cid { self.0 } - /// Wrap an [InvocationPointer] for a given [Cid]. + /// Wrap an [Pointer] for a given [Cid]. pub fn new(cid: Cid) -> Self { - InvocationPointer(cid) + Pointer(cid) } - /// Convert an [Ipld::Link] to an [InvocationPointer]. + /// Convert an [Ipld::Link] to an [Pointer]. pub fn new_from_link(link: Link) -> Self { - InvocationPointer(*link) + Pointer(*link) } } -impl From for Ipld { - fn from(ptr: InvocationPointer) -> Self { +impl From for Ipld { + fn from(ptr: Pointer) -> Self { Ipld::Link(ptr.cid()) } } -impl TryFrom for InvocationPointer { +impl TryFrom for Pointer { type Error = anyhow::Error; fn try_from(ipld: Ipld) -> Result { let s: Cid = from_ipld(ipld)?; - Ok(InvocationPointer(s)) + Ok(Pointer(s)) } } -impl TryFrom<&Ipld> for InvocationPointer { +impl TryFrom<&Ipld> for Pointer { type Error = anyhow::Error; fn try_from(ipld: &Ipld) -> Result { @@ -214,28 +214,28 @@ impl TryFrom<&Ipld> for InvocationPointer { } } -impl<'a> From for Cow<'a, InvocationPointer> { - fn from(ptr: InvocationPointer) -> Self { +impl<'a> From for Cow<'a, Pointer> { + fn from(ptr: Pointer) -> Self { Cow::Owned(ptr) } } -impl<'a> From<&'a InvocationPointer> for Cow<'a, InvocationPointer> { - fn from(ptr: &'a InvocationPointer) -> Self { +impl<'a> From<&'a Pointer> for Cow<'a, Pointer> { + fn from(ptr: &'a Pointer) -> Self { Cow::Borrowed(ptr) } } -impl ToSql for InvocationPointer { +impl ToSql for Pointer { fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { out.set_value(self.cid().to_string_of_base(Base::Base32Lower)?); Ok(IsNull::No) } } -impl FromSql for InvocationPointer { +impl FromSql for Pointer { fn from_sql(bytes: RawValue<'_, Sqlite>) -> deserialize::Result { let s = >::from_sql(bytes)?; - Ok(InvocationPointer::new(Cid::from_str(&s)?)) + Ok(Pointer::new(Cid::from_str(&s)?)) } } diff --git a/homestar-core/src/workflow/prf.rs b/homestar-core/src/workflow/prf.rs index a828520f..0be42167 100644 --- a/homestar-core/src/workflow/prf.rs +++ b/homestar-core/src/workflow/prf.rs @@ -14,9 +14,10 @@ use diesel::{ use libipld::{cbor::DagCborCodec, prelude::Codec, serde::from_ipld, Ipld, Link}; use ucan::ipld::UcanIpld; -/// Proof container, containing links to UCANs for a particular [Task]. +/// Proof container, containing links to UCANs for a particular [Task] or [Receipt]. /// /// [Task]: super::Task +/// [Receipt]: super::Receipt #[derive(Clone, Debug, Default, PartialEq, AsExpression, FromSqlRow)] #[diesel(sql_type = Binary)] pub struct UcanPrf(Vec>); diff --git a/homestar-core/src/workflow/receipt.rs b/homestar-core/src/workflow/receipt.rs index 302fb978..bd5d1550 100644 --- a/homestar-core/src/workflow/receipt.rs +++ b/homestar-core/src/workflow/receipt.rs @@ -1,14 +1,7 @@ //! Output of an invocation, referenced by its invocation pointer. -use super::{pointer::InvocationPointer, prf::UcanPrf, InvocationResult}; -use diesel::{ - backend::RawValue, - deserialize::{self, FromSql}, - serialize::{self, IsNull, Output, ToSql}, - sql_types::Text, - sqlite::Sqlite, - AsExpression, FromSqlRow, -}; +use super::{prf::UcanPrf, InstructionResult, Issuer, Pointer}; +use anyhow::anyhow; use libipld::{ cbor::DagCborCodec, cid::{ @@ -16,10 +9,10 @@ use libipld::{ Cid, }, prelude::Codec, + serde::from_ipld, Ipld, }; -use std::{borrow::Cow, collections::BTreeMap, fmt, str::FromStr}; -use ucan::ipld::Principle; +use std::collections::BTreeMap; const RAN_KEY: &str = "ran"; const OUT_KEY: &str = "out"; @@ -27,82 +20,51 @@ const ISSUER_KEY: &str = "iss"; const METADATA_KEY: &str = "meta"; const PROOF_KEY: &str = "prf"; -/// [Principal] that issued this receipt. If omitted issuer is -/// inferred from the [invocation] [task] audience. -/// -/// [invocation]: super::Invocation -/// [task]: suepr::Task -#[derive(Clone, Debug, AsExpression, FromSqlRow, PartialEq)] -#[diesel(sql_type = Text)] -pub struct Issuer(Principle); - -impl fmt::Display for Issuer { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let did_as_string = self.0.to_string(); - write!(f, "{did_as_string}") - } -} - -impl Issuer { - /// Create a new [Issuer], wrapping a [Principle]. - pub fn new(principle: Principle) -> Self { - Issuer(principle) - } -} - -/// A Receipt is an attestation of the [Result] and requested [Effects] by a -/// [Task Invocation]. -/// -/// A Receipt MUST be signed by the Executor or it's delegate. If signed by the -/// delegate, the proof of delegation from the [Executor] to the delegate -/// MUST be provided in prf. +/// A Receipt is a cryptographically signed description of the [Invocation] +/// and its [resulting output] and requested effects. /// /// TODO: Effects et al. /// -/// [Result]: InvocationResult -/// [Effects]: https://github.com/ucan-wg/invocation#7-effect -/// [Task Invocation]: super::Invocation -/// [Executor]: Issuer +/// [resulting output]: InstructionResult +/// [Invocation]: super::Invocation #[derive(Debug, Clone, PartialEq)] -pub struct Receipt<'a, T> { - ran: Cow<'a, InvocationPointer>, - out: InvocationResult, +pub struct Receipt { + ran: Pointer, + out: InstructionResult, meta: Ipld, - iss: Option, + issuer: Option, prf: UcanPrf, } -impl<'a, T> Receipt<'a, T> { +impl Receipt { /// pub fn new( - ran: InvocationPointer, - result: InvocationResult, + ran: Pointer, + result: InstructionResult, metadata: Ipld, issuer: Option, proof: UcanPrf, ) -> Self { Self { - ran: Cow::from(ran), + ran, out: result, meta: metadata, - iss: issuer, + issuer, prf: proof, } } } -impl Receipt<'_, T> { - /// [InvocationPointer] for [Task] ran. +impl Receipt { + /// [Pointer] for [Invocation] ran. /// - /// [Task]: super::Task - pub fn ran(&self) -> &InvocationPointer { + /// [Invocation]: super::Invocation + pub fn ran(&self) -> &Pointer { &self.ran } - /// [InvocationResult] output from [Task] invocation/execution. - /// - /// [Task]: super::Task - pub fn out(&self) -> &InvocationResult { + /// [InstructionResult] output from invocation/execution. + pub fn out(&self) -> &InstructionResult { &self.out } @@ -113,7 +75,7 @@ impl Receipt<'_, T> { /// Optional [Issuer] for [Receipt]. pub fn issuer(&self) -> &Option { - &self.iss + &self.issuer } /// [UcanPrf] delegation chain. @@ -122,27 +84,36 @@ impl Receipt<'_, T> { } } -impl TryFrom> for Vec { +impl TryFrom> for Vec { type Error = anyhow::Error; - fn try_from(receipt: Receipt<'_, Ipld>) -> Result { + fn try_from(receipt: Receipt) -> Result { let receipt_ipld = Ipld::from(&receipt); DagCborCodec.encode(&receipt_ipld) } } -impl TryFrom> for Cid { +impl TryFrom> for Receipt { type Error = anyhow::Error; - fn try_from(receipt: Receipt<'_, Ipld>) -> Result { + fn try_from(bytes: Vec) -> Result { + let ipld: Ipld = DagCborCodec.decode(&bytes)?; + ipld.try_into() + } +} + +impl TryFrom> for Cid { + type Error = anyhow::Error; + + fn try_from(receipt: Receipt) -> Result { TryFrom::try_from(&receipt) } } -impl TryFrom<&Receipt<'_, Ipld>> for Cid { +impl TryFrom<&Receipt> for Cid { type Error = anyhow::Error; - fn try_from(receipt: &Receipt<'_, Ipld>) -> Result { + fn try_from(receipt: &Receipt) -> Result { let ipld = Ipld::from(receipt); let bytes = DagCborCodec.encode(&ipld)?; let hash = Code::Sha3_256.digest(&bytes); @@ -150,24 +121,18 @@ impl TryFrom<&Receipt<'_, Ipld>> for Cid { } } -impl From> for Ipld { - fn from(receipt: Receipt<'_, Ipld>) -> Self { - From::from(&receipt) - } -} - -impl From<&Receipt<'_, Ipld>> for Ipld { - fn from(receipt: &Receipt<'_, Ipld>) -> Self { +impl From<&Receipt> for Ipld { + fn from(receipt: &Receipt) -> Self { Ipld::Map(BTreeMap::from([ - (RAN_KEY.into(), receipt.ran.as_ref().to_owned().into()), + (RAN_KEY.into(), receipt.ran.to_owned().into()), (OUT_KEY.into(), receipt.out.to_owned().into()), (METADATA_KEY.into(), receipt.meta.to_owned()), ( ISSUER_KEY.into(), receipt - .iss + .issuer .as_ref() - .map(|iss| iss.to_string().into()) + .map(|issuer| issuer.to_string().into()) .unwrap_or(Ipld::Null), ), (PROOF_KEY.into(), receipt.prf.to_owned().into()), @@ -175,16 +140,58 @@ impl From<&Receipt<'_, Ipld>> for Ipld { } } -impl ToSql for Issuer { - fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result { - out.set_value(self.0.to_string()); - Ok(IsNull::No) +impl From> for Ipld { + fn from(receipt: Receipt) -> Self { + From::from(&receipt) } } -impl FromSql for Issuer { - fn from_sql(bytes: RawValue<'_, Sqlite>) -> deserialize::Result { - let s = >::from_sql(bytes)?; - Ok(Issuer(Principle::from_str(&s)?)) +impl TryFrom for Receipt { + type Error = anyhow::Error; + + fn try_from(ipld: Ipld) -> Result { + let map = from_ipld::>(ipld)?; + + let ran = map + .get(RAN_KEY) + .ok_or_else(|| anyhow!("missing {RAN_KEY}"))? + .try_into()?; + + let out = map + .get(OUT_KEY) + .ok_or_else(|| anyhow!("missing {OUT_KEY}"))?; + + let meta = map + .get(METADATA_KEY) + .ok_or_else(|| anyhow!("missing {METADATA_KEY}"))?; + + let issuer = map + .get(ISSUER_KEY) + .and_then(|ipld| match ipld { + Ipld::Null => None, + ipld => Some(ipld), + }) + .and_then(|ipld| from_ipld(ipld.to_owned()).ok()) + .map(Issuer::new); + + let prf = map + .get(PROOF_KEY) + .ok_or_else(|| anyhow!("missing {PROOF_KEY}"))?; + + Ok(Receipt { + ran, + out: InstructionResult::try_from(out)?, + meta: meta.to_owned(), + issuer, + prf: UcanPrf::try_from(prf)?, + }) + } +} + +impl TryFrom> for Pointer { + type Error = anyhow::Error; + + fn try_from(receipt: Receipt) -> Result { + Ok(Pointer::new(Cid::try_from(receipt)?)) } } diff --git a/homestar-core/src/workflow/task.rs b/homestar-core/src/workflow/task.rs index 7313110c..4115c744 100644 --- a/homestar-core/src/workflow/task.rs +++ b/homestar-core/src/workflow/task.rs @@ -1,14 +1,10 @@ //! A [Task] is the smallest unit of work that can be requested from a UCAN. -use super::{ - pointer::{InvocationPointer, InvokedTaskPointer}, - Ability, Input, Nonce, -}; +use super::{instruction::RunInstruction, prf::UcanPrf, Pointer}; use anyhow::anyhow; use libipld::{ cbor::DagCborCodec, cid::{ - multibase::Base, multihash::{Code, MultihashDigest}, Cid, }, @@ -16,225 +12,157 @@ use libipld::{ serde::from_ipld, Ipld, }; -use std::{borrow::Cow, collections::BTreeMap, fmt}; -use url::Url; +use std::collections::BTreeMap; const DAG_CBOR: u64 = 0x71; -const ON_KEY: &str = "on"; -const CALL_KEY: &str = "call"; -const INPUT_KEY: &str = "input"; -const NNC_KEY: &str = "nnc"; - -/// Enumerator for `either` an expanded [Task] structure or -/// an [InvokedTaskPointer] ([Cid] wrapper). -#[derive(Debug, Clone, PartialEq)] -pub enum RunTask<'a, T> { - /// [Task] as an expanded structure. - Expanded(Task<'a, T>), - /// [Task] as a pointer. - Ptr(InvokedTaskPointer), -} +const RUN_KEY: &str = "run"; +const CAUSE_KEY: &str = "cause"; +const METADATA_KEY: &str = "meta"; +const PROOF_KEY: &str = "prf"; -impl<'a, T> From> for RunTask<'a, T> { - fn from(task: Task<'a, T>) -> Self { - RunTask::Expanded(task) - } +/// Contains the [Instruction], configuration, and a possible +/// [Receipt] of the invocation that caused this task to run. +/// +/// [Instruction]: super::Instruction +/// [Receipt]: super::Receipt +#[derive(Clone, Debug, PartialEq)] +pub struct Task<'a, T> { + run: RunInstruction<'a, T>, + cause: Option, + meta: Ipld, + prf: UcanPrf, } -impl<'a, T> TryFrom> for Task<'a, T> +impl<'a, T> Task<'a, T> where - T: fmt::Debug, + Ipld: From, + T: Clone, { - type Error = anyhow::Error; - - fn try_from(run: RunTask<'a, T>) -> Result { - match run { - RunTask::Expanded(task) => Ok(task), - e => Err(anyhow!("wrong discriminant: {e:?}")), + /// Generate a new [Task] to run, with metadata, and `prf`. + pub fn new(run: RunInstruction<'a, T>, meta: Ipld, prf: UcanPrf) -> Self { + Self { + run, + cause: None, + meta, + prf, } } -} - -impl From for RunTask<'_, T> { - fn from(ptr: InvokedTaskPointer) -> Self { - RunTask::Ptr(ptr) - } -} - -impl<'a, T> TryFrom> for InvokedTaskPointer -where - T: fmt::Debug, -{ - type Error = anyhow::Error; - fn try_from(run: RunTask<'a, T>) -> Result { - match run { - RunTask::Ptr(ptr) => Ok(ptr), - e => Err(anyhow!("wrong discriminant: {e:?}")), + /// Generate a new [Task] to execute, with metadata, given a `cause`, and + /// `prf`. + pub fn new_with_cause( + run: RunInstruction<'a, T>, + meta: Ipld, + prf: UcanPrf, + cause: Option, + ) -> Self { + Self { + run, + cause, + meta, + prf, } } -} -impl<'a, 'b, T> TryFrom<&'b RunTask<'a, T>> for &'b InvokedTaskPointer -where - T: fmt::Debug, -{ - type Error = anyhow::Error; + /// Return a reference pointer to given [Instruction] to run. + /// + /// [Instruction]: super::Instruction + pub fn run(&self) -> &RunInstruction<'_, T> { + &self.run + } - fn try_from(run: &'b RunTask<'a, T>) -> Result { - match run { - RunTask::Ptr(ptr) => Ok(ptr), - e => Err(anyhow!("wrong discriminant: {e:?}")), - } + /// Get [Task] metadata in [Ipld] form. + pub fn meta(&self) -> &Ipld { + &self.meta } -} -impl<'a, 'b, T> TryFrom<&'b RunTask<'a, T>> for InvokedTaskPointer -where - T: fmt::Debug, -{ - type Error = anyhow::Error; + /// Turn [Task] into owned [RunInstruction]. + pub fn into_instruction(self) -> RunInstruction<'a, T> { + self.run + } - fn try_from(run: &'b RunTask<'a, T>) -> Result { - match run { - RunTask::Ptr(ptr) => Ok(ptr.to_owned()), - e => Err(anyhow!("wrong discriminant: {e:?}")), + /// Return the [Cid] of the contained [Instruction]. + /// + /// [Instruction]: super::Instruction + pub fn instruction_cid(&self) -> anyhow::Result { + match &self.run { + RunInstruction::Expanded(instruction) => Ok(Cid::try_from(instruction.to_owned())?), + RunInstruction::Ptr(instruction_ptr) => Ok(instruction_ptr.cid()), } } } -impl From> for Ipld +impl From> for Ipld where Ipld: From, { - fn from(run: RunTask<'_, T>) -> Self { - match run { - RunTask::Expanded(task) => task.into(), - RunTask::Ptr(taskptr) => taskptr.into(), - } + fn from(task: Task<'_, T>) -> Self { + Ipld::Map(BTreeMap::from([ + (RUN_KEY.into(), task.run.into()), + ( + CAUSE_KEY.into(), + task.cause.map_or(Ipld::Null, |cause| cause.into()), + ), + (METADATA_KEY.into(), task.meta), + (PROOF_KEY.into(), task.prf.into()), + ])) } } -impl TryFrom for RunTask<'_, T> +impl TryFrom for Task<'_, T> where T: From, { type Error = anyhow::Error; - fn try_from<'a>(ipld: Ipld) -> Result { - match ipld { - Ipld::Map(_) => Ok(RunTask::Expanded(Task::try_from(ipld)?)), - Ipld::Link(_) => Ok(RunTask::Ptr(InvokedTaskPointer::try_from(ipld)?)), - _ => Err(anyhow!("unexpected conversion type")), - } - } -} - -/// A Task is the smallest unit of work that can be requested from a UCAN. -/// It describes one (resource, ability, input) triple. The [Input] field is -/// free-form, and depend on the specific resource and ability being interacted -/// with. Inputs can be expressed as [Ipld] or as a [deferred promise]. -/// -/// -/// # Example -/// -/// ``` -/// use homestar_core::{Unit, workflow::{Ability, Input, Task}}; -/// use libipld::Ipld; -/// use url::Url; -/// -/// let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); -/// let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); -/// -/// let task = Task::unique( -/// resource, -/// Ability::from("wasm/run"), -/// Input::::Ipld(Ipld::List(vec![Ipld::Bool(true)])) -/// ); -/// ``` -/// -/// We can also set-up a [Task] with a Deferred input to await on: -/// ``` -/// use homestar_core::{ -/// workflow::{Ability, Input, Nonce, Task, -/// pointer::{Await, AwaitResult, InvocationPointer, InvokedTaskPointer}, -/// }, -/// Unit, -/// }; -/// use libipld::{cid::{multihash::{Code, MultihashDigest}, Cid}, Ipld, Link}; -/// use url::Url; - -/// let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); -/// let resource = Url::parse(format!("ipfs://{wasm}").as_str()).expect("IPFS URL"); -/// let h = Code::Blake3_256.digest(b"beep boop"); -/// let cid = Cid::new_v1(0x55, h); -/// let link: Link = Link::new(cid); -/// let invoked_task = InvocationPointer::new_from_link(link); -/// -/// let task = Task::new( -/// resource, -/// Ability::from("wasm/run"), -/// Input::::Deferred(Await::new(invoked_task, AwaitResult::Ok)), -/// Some(Nonce::generate()) -/// ); -/// -/// // And covert it to a pointer: -/// let ptr = InvokedTaskPointer::try_from(task).unwrap(); -/// ``` -/// [deferred promise]: super::pointer::Await -#[derive(Clone, Debug, PartialEq)] -pub struct Task<'a, T> { - on: Url, - call: Cow<'a, Ability>, - input: Input, - nnc: Option, -} - -impl Task<'_, T> { - /// Create a new [Task]. - pub fn new(on: Url, ability: Ability, input: Input, nnc: Option) -> Self { - Task { - on, - call: Cow::from(ability), - input, - nnc, - } - } - - /// Create a unique [Task], with a default [Nonce] generator. - pub fn unique(on: Url, ability: Ability, input: Input) -> Self { - Task { - on, - call: Cow::from(ability), - input, - nnc: Some(Nonce::generate()), - } - } + fn try_from(ipld: Ipld) -> Result { + let map = from_ipld::>(ipld)?; - /// Return [Task] resource, i.e. [Url]. - pub fn resource(&self) -> &Url { - &self.on + Ok(Self { + run: RunInstruction::try_from( + map.get(RUN_KEY) + .ok_or_else(|| anyhow!("no `run` set"))? + .to_owned(), + )?, + cause: map + .get(CAUSE_KEY) + .and_then(|ipld| match ipld { + Ipld::Null => None, + ipld => Some(ipld), + }) + .and_then(|ipld| ipld.to_owned().try_into().ok()), + meta: map + .get(METADATA_KEY) + .ok_or_else(|| anyhow!("no `metadata` field set"))? + .to_owned(), + prf: UcanPrf::try_from( + map.get(PROOF_KEY) + .ok_or_else(|| anyhow!("no proof field set"))? + .to_owned(), + )?, + }) } +} - /// Return [Ability] associated with `call`. - pub fn call(&self) -> &Ability { - &self.call - } +impl TryFrom<&Ipld> for Task<'_, T> +where + T: From, +{ + type Error = anyhow::Error; - /// Return [Task] [Input]. - pub fn input(&self) -> &Input { - &self.input + fn try_from<'a>(ipld: &Ipld) -> Result { + TryFrom::try_from(ipld.to_owned()) } } -impl TryFrom> for InvocationPointer +impl TryFrom> for Pointer where Ipld: From, { type Error = anyhow::Error; fn try_from(task: Task<'_, T>) -> Result { - Ok(InvocationPointer::new(Cid::try_from(task)?)) + Ok(Pointer::new(Cid::try_from(task)?)) } } @@ -252,120 +180,83 @@ where } } -impl From> for Ipld -where - Ipld: From, -{ - fn from(task: Task<'_, T>) -> Self { - Ipld::Map(BTreeMap::from([ - (ON_KEY.into(), task.on.to_string().into()), - (CALL_KEY.into(), task.call.to_string().into()), - (INPUT_KEY.into(), task.input.into()), - ( - NNC_KEY.into(), - task.nnc.map(|nnc| nnc.into()).unwrap_or(Ipld::Null), - ), - ])) - } -} - -impl TryFrom<&Ipld> for Task<'_, T> -where - T: From, -{ - type Error = anyhow::Error; - - fn try_from(ipld: &Ipld) -> Result { - TryFrom::try_from(ipld.to_owned()) - } -} +#[cfg(test)] +mod test { + use super::*; + use crate::{test_utils, workflow::config::Resources, Unit}; -impl TryFrom for Task<'_, T> -where - T: From, -{ - type Error = anyhow::Error; + #[test] + fn ipld_roundtrip() { + let config = Resources::default(); + let instruction = test_utils::workflow::instruction::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction.clone()), + config.clone().into(), + UcanPrf::default(), + ); - fn try_from(ipld: Ipld) -> Result { - let map = from_ipld::>(ipld)?; + let ipld1 = Ipld::from(task1.clone()); - let on = match map.get(ON_KEY) { - Some(Ipld::Link(cid)) => cid - .to_string_of_base(Base::Base32Lower) - .map_err(|e| anyhow!("failed to encode CID into multibase string: {e}")) - .and_then(|txt| { - Url::parse(format!("{}{}", "ipfs://", txt).as_str()) - .map_err(|e| anyhow!("failed to parse URL: {e}")) - }), - Some(Ipld::String(txt)) => { - Url::parse(txt.as_str()).map_err(|e| anyhow!("failed to parse URL: {e}")) - } - _ => Err(anyhow!("no resource/with set.")), - }?; + let ipld_task = Ipld::Map(BTreeMap::from([ + ( + "rsc".into(), + Ipld::String( + "ipfs://bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".into(), + ), + ), + ("op".into(), Ipld::String("ipld/fun".to_string())), + ("input".into(), Ipld::List(vec![Ipld::Bool(true)])), + ("nnc".into(), Ipld::String("".to_string())), + ])); - Ok(Task { - on, - call: from_ipld( - map.get(CALL_KEY) - .ok_or_else(|| anyhow!("no `call` field set"))? - .to_owned(), - )?, - input: Input::try_from( - map.get(INPUT_KEY) - .ok_or_else(|| anyhow!("no `input` field set"))? - .to_owned(), - )?, - nnc: map.get(NNC_KEY).and_then(|ipld| match ipld { - Ipld::Null => None, - ipld => Nonce::try_from(ipld).ok(), - }), - }) - } -} + assert_eq!( + ipld1, + Ipld::Map(BTreeMap::from([ + (RUN_KEY.into(), ipld_task), + (CAUSE_KEY.into(), Ipld::Null), + ( + METADATA_KEY.into(), + Ipld::Map(BTreeMap::from([ + ("fuel".into(), Ipld::Integer(u64::MAX.into())), + ("time".into(), Ipld::Integer(100_000)) + ])) + ), + (PROOF_KEY.into(), Ipld::List(vec![])) + ])) + ); -#[cfg(test)] -mod test { - use super::*; - use crate::Unit; + assert_eq!(task1, ipld1.try_into().unwrap()); - fn task<'a, T>() -> (Task<'a, T>, Vec) { - let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string(); - let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap(); - let nonce = Nonce::generate(); + let receipt = test_utils::workflow::receipt(); - ( - Task::new( - resource, - Ability::from("wasm/run"), - Input::Ipld(Ipld::List(vec![Ipld::Bool(true)])), - Some(nonce.clone()), - ), - nonce.as_nonce96().unwrap().to_vec(), - ) - } + let task2 = Task::new_with_cause( + RunInstruction::Ptr::(instruction.try_into().unwrap()), + config.into(), + UcanPrf::default(), + Some(receipt.clone().try_into().unwrap()), + ); - #[test] - fn ipld_roundtrip() { - let (task, bytes) = task::(); - let ipld = Ipld::from(task.clone()); + let ipld2 = Ipld::from(task2.clone()); assert_eq!( - ipld, + ipld2, Ipld::Map(BTreeMap::from([ + (RUN_KEY.into(), Ipld::Link(task2.instruction_cid().unwrap())), ( - ON_KEY.into(), - Ipld::String( - "ipfs://bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".into() - ) + CAUSE_KEY.into(), + Ipld::Link(Cid::try_from(receipt).unwrap()) ), - (CALL_KEY.into(), Ipld::String("wasm/run".to_string())), - (INPUT_KEY.into(), Ipld::List(vec![Ipld::Bool(true)])), ( - NNC_KEY.into(), - Ipld::List(vec![Ipld::Integer(0), Ipld::Bytes(bytes)]) - ) + METADATA_KEY.into(), + Ipld::Map(BTreeMap::from([ + ("fuel".into(), Ipld::Integer(u64::MAX.into())), + ("time".into(), Ipld::Integer(100_000)) + ])) + ), + (PROOF_KEY.into(), Ipld::List(vec![])) ])) ); - assert_eq!(task, ipld.try_into().unwrap()) + + assert_eq!(task2, ipld2.try_into().unwrap()); } } diff --git a/homestar-guest-wasm/Cargo.toml b/homestar-guest-wasm/Cargo.toml index 8ff78700..5a70a01d 100644 --- a/homestar-guest-wasm/Cargo.toml +++ b/homestar-guest-wasm/Cargo.toml @@ -3,10 +3,16 @@ name = "homestar-guest-wasm" publish = false version = "0.1.0" edition = { workspace = true } +rust-version = { workspace = true } [dependencies] -image = "0.24" +image = { version = "0.24", default-features = false, features = ["png"] } wit-bindgen = "0.4" +[dev-dependencies] +image = "0.24" + [lib] +doc = false +bench = false crate-type = ["cdylib", "rlib"] diff --git a/homestar-guest-wasm/src/lib.rs b/homestar-guest-wasm/src/lib.rs index f8c255a5..0d86d22c 100644 --- a/homestar-guest-wasm/src/lib.rs +++ b/homestar-guest-wasm/src/lib.rs @@ -1,12 +1,11 @@ #![allow(clippy::too_many_arguments)] -use image::DynamicImage; - +use std::io::Cursor; wit_bindgen::generate!("test" in "./wits"); struct Component; -type Matrix = Vec>; +type Matrix = Vec>; impl Homestar for Component { fn add_one(a: i32) -> i32 { @@ -18,6 +17,10 @@ impl Homestar for Component { [a, b.to_string()].join("\n") } + fn join_strings(a: String, b: String) -> String { + [a, b].join("") + } + fn transpose(matrix: Matrix) -> Matrix { assert!(!matrix.is_empty()); let len = matrix[0].len(); @@ -26,41 +29,55 @@ impl Homestar for Component { .collect() } - fn blur(data: Vec, sigma: f32, width: u32, height: u32) -> Vec { - let img_buf = image::RgbImage::from_vec(width, height, data).unwrap(); + fn blur(data: Vec, sigma: f32) -> Vec { + let img = image::load_from_memory_with_format(&data, image::ImageFormat::Png).unwrap(); - let blurred = DynamicImage::ImageRgb8(img_buf).blur(sigma); - blurred.into_bytes() + let blurred = img.blur(sigma); + + let mut buffer: Vec = Vec::new(); + blurred + .write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); + + buffer } - fn crop( - data: Vec, - x: u32, - y: u32, - target_width: u32, - target_height: u32, - width: u32, - height: u32, - ) -> Vec { - let img_buf = image::RgbImage::from_vec(width, height, data).unwrap(); + fn crop(data: Vec, x: u32, y: u32, target_width: u32, target_height: u32) -> Vec { + let mut img = image::load_from_memory_with_format(&data, image::ImageFormat::Png).unwrap(); // Crop this image delimited by the bounding rectangle - let cropped = DynamicImage::ImageRgb8(img_buf).crop(x, y, target_width, target_height); - cropped.into_bytes() + let cropped = img.crop(x, y, target_width, target_height); + + let mut buffer: Vec = Vec::new(); + cropped + .write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); + + buffer } - fn grayscale(data: Vec, width: u32, height: u32) -> Vec { - let img_buf = image::RgbImage::from_vec(width, height, data).unwrap(); + fn grayscale(data: Vec) -> Vec { + let img = image::load_from_memory_with_format(&data, image::ImageFormat::Png).unwrap(); + let gray = img.grayscale(); + + let mut buffer: Vec = Vec::new(); + gray.write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); - let gray = DynamicImage::ImageRgb8(img_buf).grayscale(); - gray.to_rgb8().into_vec() + buffer } - fn rotate90(data: Vec, width: u32, height: u32) -> Vec { - let img_buf = image::RgbImage::from_vec(width, height, data).unwrap(); + fn rotate90(data: Vec) -> Vec { + let img = image::load_from_memory_with_format(&data, image::ImageFormat::Png).unwrap(); + + let rotated = img.rotate90(); - let rotated = DynamicImage::ImageRgb8(img_buf).rotate90(); - rotated.into_bytes() + let mut buffer: Vec = Vec::new(); + rotated + .write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); + + buffer } } @@ -95,64 +112,98 @@ mod test { #[test] fn blur() { let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap(); - let (width, height) = (img.width(), img.height()); - let img_vec = img.into_bytes(); + let mut buffer: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); // Call component to blur the image - let result = Component::blur(img_vec, 1.0, width, height); + let result = Component::blur(buffer, 0.3); + + let png_img = image::io::Reader::new(Cursor::new(&result)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); - let processed_buf = image::RgbImage::from_vec(width, height, result).unwrap(); - let processed = DynamicImage::ImageRgb8(processed_buf); - processed - .save("./out/blurred.jpg") - .expect("Failed to write cropped.jpg to filesystem"); + png_img + .save("./out/blurred.png") + .expect("Failed to write blurred.png to filesystem"); } #[test] fn crop() { let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap(); - let (width, height) = (img.width(), img.height()); - let img_vec = img.into_bytes(); - - // Call component to crop the image to a 200x200 square - let result = Component::crop(img_vec, 150, 350, 400, 400, width, height); - - let processed_buf = image::RgbImage::from_vec(400, 400, result).unwrap(); - let processed = DynamicImage::ImageRgb8(processed_buf); - processed - .save("./out/cropped.jpg") - .expect("Failed to write cropped.jpg to filesystem"); + let mut buffer: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); + + // Call component to crop the image to a 400x400 square + let result = Component::crop(buffer, 150, 350, 400, 400); + + let png_img = image::io::Reader::new(Cursor::new(&result)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + + png_img + .save("./out/cropped.png") + .expect("Failed to write cropped.png to filesystem"); } #[test] fn grayscale() { let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap(); - let (width, height) = (img.width(), img.height()); - let img_vec = img.into_bytes(); + let mut buffer: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); // Call component to grayscale the image - let result = Component::grayscale(img_vec, width, height); + let result = Component::grayscale(buffer); + + let png_img = image::io::Reader::new(Cursor::new(&result)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); - let processed_buf = image::RgbImage::from_vec(width, height, result).unwrap(); - let processed = DynamicImage::ImageRgb8(processed_buf); - processed - .save("./out/graycat.jpg") + png_img + .save("./out/graycat.png") .expect("Failed to write graycat.jpg to filesystem"); } #[test] fn rotate() { let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap(); - let (width, height) = (img.width(), img.height()); - let img_vec = img.into_bytes(); + let mut buffer: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); // Call component to rotate the image 90 deg clockwise - let result = Component::rotate90(img_vec, width, height); + let result = Component::rotate90(buffer); - let processed_buf = image::RgbImage::from_vec(width, height, result).unwrap(); - let processed = DynamicImage::ImageRgb8(processed_buf); - processed - .save("./out/rotated.jpg") + let png_img = image::io::Reader::new(Cursor::new(&result)) + .with_guessed_format() + .unwrap() + .decode() + .unwrap(); + + png_img + .save("./out/rotated.png") .expect("Failed to write graycat.jpg to filesystem"); } + + #[test] + fn mixed() { + let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap(); + let mut buffer: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut buffer), image::ImageOutputFormat::Png) + .unwrap(); + + // Call component to rotate the image 90 deg clockwise + let rotated = Component::rotate90(buffer); + let gray = Component::grayscale(rotated); + let cropped = Component::crop(gray, 150, 350, 400, 400); + Component::blur(cropped, 0.1); + } } diff --git a/homestar-guest-wasm/wits/test.wit b/homestar-guest-wasm/wits/test.wit index f21e556b..d12296a9 100644 --- a/homestar-guest-wasm/wits/test.wit +++ b/homestar-guest-wasm/wits/test.wit @@ -1,9 +1,10 @@ default world homestar { export add-one: func(a: s32) -> s32 export append-string: func(a: string) -> string - export transpose: func(matrix: list>) -> list> - export blur: func(data: list, sigma: float32, width: u32, height: u32) -> list - export crop: func(data: list, x: u32, y: u32, target-width: u32, target-height: u32, width: u32, height: u32) -> list - export grayscale: func(data: list, width: u32, height: u32) -> list - export rotate90: func(data: list, width: u32, height: u32) -> list + export join-strings: func(a: string, b: string) -> string + export transpose: func(matrix: list>) -> list> + export blur: func(data: list, sigma: float32) -> list + export crop: func(data: list, x: u32, y: u32, target-width: u32, target-height: u32) -> list + export grayscale: func(data: list) -> list + export rotate90: func(data: list) -> list } diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml index 35362306..86d2c85d 100644 --- a/homestar-runtime/Cargo.toml +++ b/homestar-runtime/Cargo.toml @@ -1,10 +1,9 @@ [package] name = "homestar-runtime" version = "0.1.0" -description = "" -keywords = [] -categories = [] - +description = "Homestar runtime implementation" +keywords = ["ipfs", "workflows", "ipld", "ipvm"] +categories = ["workflow-engines", "distributed-systems", "runtimes", "networking"] include = ["/src", "README.md", "LICENSE"] license = { workspace = true } readme = "README.md" @@ -27,31 +26,64 @@ doc = false bench = false [dependencies] -anyhow = "1.0" +ansi_term = { version = "0.12", optional = true, default-features = false } +# return to version.workspace = true after the following issue is fixed: +# https://github.com/DevinR528/cargo-sort/issues/47 +anyhow = { workspace = true } async-trait = "0.1" +axum = { version = "0.6", features = ["ws", "headers"] } +byte-unit = { version = "4.0", default-features = false } clap = { version = "4.1", features = ["derive"] } -diesel = { version = "2.0", features = ["sqlite"] } +concat-in-place = "1.1" +config = "0.13" +console-subscriber = { version = "0.1", default-features = false, features = [ "parking_lot" ], optional = true } +crossbeam = "0.8" +dagga = "0.2" +diesel = { version = "2.0", features = ["sqlite", "r2d2", "returning_clauses_for_sqlite_3_35"] } diesel_migrations = "2.0" dotenvy = "0.15" -env_logger = "0.10" +enum-assoc = "0.4" +futures = "0.3" +headers = "0.3" homestar-core = { version = "0.1", path = "../homestar-core" } homestar-wasm = { version = "0.1", path = "../homestar-wasm" } -ipfs-api = "0.17" -ipfs-api-backend-hyper = { version = "0.6", features = ["with-builder"] } +http = "0.2" +http-serde = "1.1" +indexmap = "1.9" +ipfs-api = { version = "0.17", optional = true } +ipfs-api-backend-hyper = { version = "0.6", features = ["with-builder", "with-send-sync"], optional = true } itertools = "0.10" +json = "0.12" libipld = "0.16" -libp2p = { version = "0.51", features = ["kad", "request-response", "macros", "identify", "mdns", "floodsub", "gossipsub", "tokio", "dns", "tcp", "noise", "yamux", "websocket"] } +libp2p = { version = "0.51", features = ["kad", "request-response", "macros", "identify", "mdns", "gossipsub", "tokio", "dns", "mplex", "tcp", "noise", "yamux", "websocket"] } libp2p-identity = "0.1" proptest = { version = "1.1", optional = true } +reqwest = { version = "0.11", features = ["json"] } +semver = "1.0" serde = { version = "1.0", features = ["derive"] } -tokio = { version = "1.26", features = ["io-util", "io-std", "macros", "rt", "rt-multi-thread"] } -tracing = "0.1" -tracing-subscriber = "0.3" +serde_with = "2.3" +thiserror = "1.0" +tokio = { version = "1.26", features = ["fs", "io-util", "io-std", "macros", "rt", "rt-multi-thread"] } +tracing = { workspace = true } +tracing-logfmt = { version = "0.3", optional = true } +tracing-subscriber = { version = "0.3", features = ["env-filter", "parking_lot", "registry"] } +tryhard = "0.5" url = "2.3" [dev-dependencies] criterion = "0.4" +homestar-core = { version = "0.1", path = "../homestar-core", features = [ "test_utils" ] } +tokio-tungstenite = "0.18" [features] -default = [] +default = ["console", "ipfs", "logfmt"] +ansi-logs = ["ansi_term"] +console = ["console-subscriber"] +ipfs = ["ipfs-api", "ipfs-api-backend-hyper"] +logfmt = ["tracing-logfmt"] test_utils = ["proptest"] + +[package.metadata.docs.rs] +all-features = true +# defines the configuration attribute `docsrs` +rustdoc-args = ["--cfg", "docsrs"] diff --git a/homestar-runtime/config/settings.toml b/homestar-runtime/config/settings.toml new file mode 100644 index 00000000..528e63c6 --- /dev/null +++ b/homestar-runtime/config/settings.toml @@ -0,0 +1,4 @@ +[monitoring] +process_collector_interval = 10 + +[node] diff --git a/migrations/.keep b/homestar-runtime/migrations/.keep similarity index 100% rename from migrations/.keep rename to homestar-runtime/migrations/.keep diff --git a/homestar-runtime/migrations/2022-12-11-183928_create_receipts/down.sql b/homestar-runtime/migrations/2022-12-11-183928_create_receipts/down.sql new file mode 100644 index 00000000..518332ca --- /dev/null +++ b/homestar-runtime/migrations/2022-12-11-183928_create_receipts/down.sql @@ -0,0 +1,2 @@ +DROP TABLE receipts; +DROP INDEX instruction_index; diff --git a/homestar-runtime/migrations/2022-12-11-183928_create_receipts/up.sql b/homestar-runtime/migrations/2022-12-11-183928_create_receipts/up.sql new file mode 100644 index 00000000..e5d0dd40 --- /dev/null +++ b/homestar-runtime/migrations/2022-12-11-183928_create_receipts/up.sql @@ -0,0 +1,12 @@ +CREATE TABLE receipts ( + cid TEXT NOT NULL PRIMARY KEY, + ran TEXT NOT NULL, + instruction TEXT NOT NULL, + out BLOB NOT NULL, + meta BLOB NOT NULL, + issuer TEXT, + prf BLOB NOT NULL, + version TEXT NOT NULL +); + +CREATE INDEX instruction_index ON receipts (instruction); diff --git a/homestar-runtime/src/cli.rs b/homestar-runtime/src/cli.rs index 12e35eb2..ef9c308c 100644 --- a/homestar-runtime/src/cli.rs +++ b/homestar-runtime/src/cli.rs @@ -1,50 +1,25 @@ //! CLI commands/arguments. use clap::Parser; -use libp2p::core::Multiaddr; /// CLI arguments. #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] pub struct Args { - /// Peer address. - #[arg(long)] - pub peer: Option, - - /// Listen address. - #[arg(long)] - pub listen: Option, - /// Ipvm-specific [Argument]. #[clap(subcommand)] pub argument: Argument, } -/// An Ipvm-specific CLI argument. +/// CLI Argument types. #[derive(Debug, Parser)] pub enum Argument { - /// Provider arguments. - Provide { - /// Wasm or WAT [Cid]. - /// - /// [Cid]: libipld::cid::Cid - #[arg(short, long)] - wasm: String, - - /// Function name within Wasm module. + /// TODO: Run [Workflow] given a file. + /// + /// [Workflow]: crate::Workflow + Run { + /// Configuration file for *homestar* node settings. #[arg(short, long)] - fun: String, - - /// Parameters / arguments to Wasm function. - #[arg(short, long, num_args(0..))] - args: Vec, - }, - /// GET/read arguments. - Get { - #[clap(long)] - /// [Cid] name/pointer to content. - /// - /// [Cid]: libipld::cid::Cid - name: String, + runtime_config: Option, }, } diff --git a/homestar-runtime/src/db.rs b/homestar-runtime/src/db.rs index b0b0f889..d5f2c128 100644 --- a/homestar-runtime/src/db.rs +++ b/homestar-runtime/src/db.rs @@ -3,14 +3,124 @@ #[allow(missing_docs, unused_imports)] pub mod schema; -use diesel::prelude::*; +use crate::{settings, Receipt}; +use anyhow::Result; +use byte_unit::{AdjustedByte, Byte, ByteUnit}; +use diesel::{prelude::*, r2d2}; use dotenvy::dotenv; -use std::env; - -/// Establish connection to Sqlite database. -pub fn establish_connection() -> SqliteConnection { - dotenv().ok(); - let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - SqliteConnection::establish(&database_url) - .unwrap_or_else(|_| panic!("Error connecting to {database_url}")) +use homestar_core::workflow::Pointer; +use std::{env, sync::Arc, time::Duration}; +use tokio::fs; + +/// A Sqlite connection [pool]. +/// +/// [pool]: r2d2::Pool +pub type Pool = r2d2::Pool>; +/// A [connection] from the Sqlite connection [pool]. +/// +/// [connection]: r2d2::PooledConnection +/// [pool]: r2d2::Pool +pub type Connection = r2d2::PooledConnection>; + +/// The database object, which wraps an inner [Arc] to the connection pool. +#[derive(Debug)] +pub struct Db(Arc); + +impl Clone for Db { + fn clone(&self) -> Self { + Db(Arc::clone(&self.0)) + } +} + +impl Db { + fn url() -> String { + dotenv().ok(); + env::var("DATABASE_URL").expect("DATABASE_URL must be set") + } + + /// Get size of SQlite file in megabytes (via async call). + pub async fn size() -> Result { + let len = fs::metadata(Db::url()).await?.len(); + let byte = Byte::from_bytes(len); + let byte_unit = byte.get_adjusted_unit(ByteUnit::MB); + Ok(byte_unit) + } +} + +/// Database trait for working with different Sqlite [pool] and [connection] +/// configurations. +/// +/// [pool]: Pool +/// [connection]: Connection +pub trait Database { + /// Establish a pooled connection to Sqlite database. + fn setup_connection_pool(settings: &settings::Node) -> Self; + /// Get a [pooled connection] for the database. + /// + /// [pooled connection]: Connection + fn conn(&self) -> Result; + /// Store receipt given a [Connection] to the DB [Pool]. + /// + /// On conflicts, do nothing. + fn store_receipt(receipt: Receipt, conn: &mut Connection) -> Result { + diesel::insert_into(schema::receipts::table) + .values(&receipt) + .on_conflict(schema::receipts::cid) + .do_nothing() + .get_result(conn) + .map_err(Into::into) + } + + /// Store receipts given a [Connection] to the DB [Pool]. + fn store_receipts(receipts: Vec, conn: &mut Connection) -> Result { + diesel::insert_into(schema::receipts::table) + .values(&receipts) + .execute(conn) + .map_err(Into::into) + } + + /// Find receipt for a given [Instruction] [Pointer], which is indexed. + /// + /// This *should* always return one receipt, but sometimes it's nicer to + /// work across vecs/arrays. + /// + /// [Instruction]: homestar_core::workflow::Instruction + fn find_instructions(pointers: Vec, conn: &mut Connection) -> Result> { + let found_receipts = schema::receipts::dsl::receipts + .filter(schema::receipts::instruction.eq_any(pointers)) + .load(conn)?; + Ok(found_receipts) + } + + /// Find receipt for a given [Instruction] [Pointer], which is indexed. + /// + /// [Instruction]: homestar_core::workflow::Instruction + fn find_instruction(pointer: Pointer, conn: &mut Connection) -> Result { + let found_receipt = schema::receipts::dsl::receipts + .filter(schema::receipts::instruction.eq(pointer)) + .first(conn)?; + Ok(found_receipt) + } +} + +impl Database for Db { + fn setup_connection_pool(settings: &settings::Node) -> Self { + let manager = r2d2::ConnectionManager::::new(Db::url()); + + let pool = r2d2::Pool::builder() + // Max number of conns. + .max_size(settings.db.max_pool_size) + // Never maintain idle connections + .min_idle(Some(0)) + // Close connections after 30 seconds of idle time + .idle_timeout(Some(Duration::from_secs(30))) + .build(manager) + .expect("DATABASE_URL must be set to an SQLite DB file"); + Db(Arc::new(pool)) + } + + fn conn(&self) -> Result { + let conn = self.0.get()?; + Ok(conn) + } } diff --git a/homestar-runtime/src/db/schema.rs b/homestar-runtime/src/db/schema.rs index 92648aba..c892efdb 100644 --- a/homestar-runtime/src/db/schema.rs +++ b/homestar-runtime/src/db/schema.rs @@ -4,9 +4,11 @@ diesel::table! { receipts (cid) { cid -> Text, ran -> Text, + instruction -> Text, out -> Binary, meta -> Binary, - iss -> Nullable, + issuer -> Nullable, prf -> Binary, + version -> Text, } } diff --git a/homestar-runtime/src/lib.rs b/homestar-runtime/src/lib.rs index 2855f366..158c1336 100644 --- a/homestar-runtime/src/lib.rs +++ b/homestar-runtime/src/lib.rs @@ -2,18 +2,40 @@ #![warn(missing_debug_implementations, missing_docs, rust_2018_idioms)] #![deny(unreachable_pub, private_in_public)] -//! Homestar is a determistic Wasm runtime and effectful job system intended to -//! embed inside IPFS. +//! homestar-runtime is a determistic Wasm runtime and effectful workflow/job +//! system intended to be embedded inside or run alongside IPFS. +//! //! You can find a more complete description [here]. //! -//! [here]: https://github.com/ipvm-wg/spec. +//! +//! Related crates/packages: +//! +//! - [homestar-core] +//! - [homestar-wasm] +//! +//! [here]: +//! [homestar-core]: homestar_core +//! [homestar-wasm]: homestar_wasm pub mod cli; pub mod db; +pub mod logger; pub mod network; mod receipt; +mod runtime; +pub mod scheduler; +pub mod settings; +pub mod tasks; +mod worker; +pub mod workflow; -pub use receipt::*; +pub use db::Db; +#[cfg(feature = "ipfs")] +pub use network::ipfs::IpfsCli; +pub use receipt::Receipt; +pub use runtime::*; +pub use worker::Worker; +pub use workflow::Workflow; /// Test utilities. #[cfg(any(test, feature = "test_utils"))] diff --git a/homestar-runtime/src/logger.rs b/homestar-runtime/src/logger.rs new file mode 100644 index 00000000..9c4501e2 --- /dev/null +++ b/homestar-runtime/src/logger.rs @@ -0,0 +1,59 @@ +//! Logger initialization. + +use anyhow::Result; +#[cfg(not(feature = "logfmt"))] +use tracing_subscriber::prelude::*; +#[cfg(feature = "logfmt")] +use tracing_subscriber::{layer::SubscriberExt as _, prelude::*, EnvFilter}; + +/// Initialize a [tracing_subscriber::Registry] with a [logfmt] layer. +/// +/// [logfmt]: +#[cfg(feature = "logfmt")] +pub fn init() -> Result<()> { + let registry = tracing_subscriber::Registry::default() + .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .with(tracing_logfmt::layer()); + + #[cfg(all(feature = "console", tokio_unstable))] + #[cfg_attr(docsrs, doc(cfg(feature = "console")))] + { + let console_layer = console_subscriber::ConsoleLayer::builder() + .retention(Duration::from_secs(60)) + .spawn(); + + registry.with(console_layer).init(); + } + + #[cfg(any(not(feature = "console"), not(tokio_unstable)))] + { + registry.init(); + } + + Ok(()) +} + +/// Initialize a default [tracing_subscriber::FmtSubscriber]. +#[cfg(not(feature = "logfmt"))] +pub fn init() -> Result<()> { + let registry = tracing_subscriber::FmtSubscriber::builder() + .with_target(false) + .finish(); + + #[cfg(all(feature = "console", tokio_unstable))] + #[cfg_attr(docsrs, doc(cfg(feature = "console")))] + { + let console_layer = console_subscriber::ConsoleLayer::builder() + .retention(Duration::from_secs(60)) + .spawn(); + + registry.with(console_layer).init(); + } + + #[cfg(any(not(feature = "console"), not(tokio_unstable)))] + { + registry.init(); + } + + Ok(()) +} diff --git a/homestar-runtime/src/main.rs b/homestar-runtime/src/main.rs index 6711e546..f35593c7 100644 --- a/homestar-runtime/src/main.rs +++ b/homestar-runtime/src/main.rs @@ -1,235 +1,58 @@ -use anyhow::{anyhow, bail, Result}; +use anyhow::Result; use clap::Parser; -use diesel::RunQueryDsl; -use homestar_core::workflow::{ - config::Resources, input::Parse, prf::UcanPrf, receipt::Receipt as LocalReceipt, Ability, - Input, Invocation, InvocationResult, Task, -}; +#[cfg(feature = "ipfs")] +use homestar_runtime::network::ipfs::IpfsCli; use homestar_runtime::{ cli::{Args, Argument}, - db::{self, schema}, + db::{Database, Db}, + logger, network::{ - client::Client, - eventloop::{Event, RECEIPTS_TOPIC}, - swarm::{self, Topic, TopicMessage}, + eventloop::{EventLoop, RECEIPTS_TOPIC}, + swarm, + ws::WebSocket, }, - Receipt, -}; -use homestar_wasm::wasmtime; -use ipfs_api::{ - request::{DagCodec, DagPut}, - response::DagPutResponse, - IpfsApi, IpfsClient, -}; -use itertools::Itertools; -use libipld::{ - cid::{multibase::Base, Cid}, - Ipld, -}; -use libp2p::{ - futures::{future, TryStreamExt}, - identity::Keypair, - multiaddr::Protocol, + settings::Settings, }; -use libp2p_identity::PeerId; -use std::{ - collections::BTreeMap, - io::{stdout, Cursor, Write}, - str::{self, FromStr}, -}; -use url::Url; +use std::sync::Arc; -#[tokio::main] +#[tokio::main(flavor = "multi_thread")] async fn main() -> Result<()> { - env_logger::init(); + logger::init()?; let opts = Args::parse(); - let keypair = Keypair::generate_ed25519(); - - let mut swarm = swarm::new(keypair).await?; - - // subscribe to `receipts` topic - swarm.behaviour_mut().gossip_subscribe(RECEIPTS_TOPIC)?; - - let (mut client, mut events, event_loop) = Client::new(swarm).await?; - tokio::spawn(event_loop.run()); + #[cfg(feature = "ipfs")] + let ipfs = IpfsCli::default(); - if let Some(addr) = opts.peer { - let peer_id = match addr.iter().last() { - Some(Protocol::P2p(hash)) => PeerId::from_multihash(hash).expect("Valid hash."), - _ => bail!("Expect peer multiaddr to contain peer ID."), - }; - client.dial(peer_id, addr).await.expect("Dial to succeed."); - } - - match opts.listen { - Some(addr) => client - .start_listening(addr) - .await - .expect("Listening not to fail."), - - None => client - .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) - .await - .expect("Listening not to fail."), - }; - - // TODO: abstraction for this and redo inner parts, around ownership, etc. - // TODO: cleanup-up use, clones, etc. match opts.argument { - Argument::Get { name } => { - let cid_name = Cid::from_str(&name)?; - let cid_string = cid_name.to_string_of_base(Base::Base32Lower)?; - let providers = client.get_providers(cid_string.clone()).await?; - - if providers.is_empty() { - Err(anyhow!("could not find provider for file {name}"))?; - } - - let requests = providers.into_iter().map(|p| { - let mut client = client.clone(); - let name = cid_string.clone(); - #[allow(unknown_lints, clippy::redundant_async_block)] - Box::pin(async move { client.request_file(p, name).await }) - }); - - let file_content = future::select_ok(requests) - .await - .map_err(|_| anyhow!("none of the providers returned file"))? - .0; - - stdout().write_all(&file_content)? - } - - Argument::Provide { wasm, fun, args } => { - let ipfs = IpfsClient::default(); - - // Pull Wasm (module) *out* of IPFS - let wasm_bytes = ipfs - .cat(wasm.as_str()) - .map_ok(|chunk| chunk.to_vec()) - .try_concat() - .await?; - - let wasm_args = - // Pull arg *out* of IPFS - future::try_join_all(args.iter().map(|arg| - ipfs - .cat(arg.as_str()) - .map_ok(|chunk| { - chunk.to_vec() - }) - .try_concat() - )).await?; - - // TODO: Don't read randomly from file. - // The interior of this is test specific code, - // unil we use a format for params, like Json. - let ipld_args = wasm_args - .iter() - .map(|a| { - if let Ok(arg) = str::from_utf8(a) { - match i32::from_str(arg) { - Ok(num) => Ok::(Ipld::from(num)), - Err(_e) => Ok::(Ipld::from(arg)), - } - } else { - Err(anyhow!("Unreadable input bytes: {a:?}")) - } - }) - .fold_ok(vec![], |mut acc, elem| { - acc.push(elem); - acc - })?; - - // TODO: Only works off happy path, but need to work with traps to - // capture error. - // TODO: State will derive from resources, other configuration. - let resource = Url::parse(format!("ipfs://{wasm}").as_str()).expect("IPFS URL"); - - let task = Task::new( - resource, - Ability::from("wasm/run"), - Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(ipld_args), - )]))), - None, - ); - let config = Resources::default(); - let invocation = Invocation::new( - task.clone().into(), - config.clone().into(), - UcanPrf::default(), - )?; - - let mut env = - wasmtime::World::instantiate(wasm_bytes, fun, wasmtime::State::default()).await?; - let res = env.execute(task.input().parse()?.try_into()?).await?; - - let local_receipt = LocalReceipt::new( - invocation.try_into()?, - InvocationResult::Ok(res.try_into()?), - Ipld::Null, - None, - UcanPrf::default(), - ); - let receipt = Receipt::try_from(&local_receipt)?; - - let receipt_bytes: Vec = local_receipt.try_into()?; - let dag_builder = DagPut::builder() - .input_codec(DagCodec::Cbor) - .hash("sha3-256") // sadly no support for blake3-256 - .build(); - let DagPutResponse { cid } = ipfs - .dag_put_with_options(Cursor::new(receipt_bytes.clone()), dag_builder) - .await - .expect("a CID"); - - // //Test for now - assert_eq!(cid.cid_string, receipt.cid()); - - let mut conn = db::establish_connection(); - // TODO: insert (or upsert via event handling when subscribed) - diesel::insert_into(schema::receipts::table) - .values(&receipt) - .on_conflict(schema::receipts::cid) - .do_nothing() - .execute(&mut conn) - .expect("Error saving new receipt"); - println!("stored: {receipt}"); - - let invoked_cid = receipt.ran(); - let output = receipt.output().clone(); - let async_client = client.clone(); - // We delay messages to make sure peers are within the mesh. - tokio::spawn(async move { - // TODO: make this configurable, but currently matching heartbeat. - tokio::time::sleep(std::time::Duration::from_secs(10)).await; - let _ = async_client - .publish_message( - Topic::new(RECEIPTS_TOPIC.to_string()), - TopicMessage::Receipt(receipt), - ) - .await; - }); - - let _ = client.start_providing(invoked_cid.clone()).await; - - loop { - match events.recv().await { - Some(Event::InboundRequest { request, channel }) => { - if request.eq(&invoked_cid) { - let output = format!("{output:?}"); - client.respond_file(output.into_bytes(), channel).await?; - } - } - e => todo!("{:?}", e), - } - } + Argument::Run { runtime_config } => { + let settings = if let Some(file) = runtime_config { + Settings::load_from_file(file) + } else { + Settings::load() + }?; + + let db = Db::setup_connection_pool(settings.node()); + let mut swarm = swarm::new(settings.node()).await?; + + // subscribe to `receipts` topic + swarm.behaviour_mut().gossip_subscribe(RECEIPTS_TOPIC)?; + + let (_tx, rx) = EventLoop::setup_channel(settings.node()); + // instantiate and start event-loop for events + let eventloop = EventLoop::new(swarm, rx, settings.node()); + + #[cfg(not(feature = "ipfs"))] + tokio::spawn(eventloop.run(db)); + + #[cfg(feature = "ipfs")] + tokio::spawn(eventloop.run(db, ipfs)); + + let (ws_tx, ws_rx) = WebSocket::setup_channel(settings.node()); + let ws_sender = Arc::new(ws_tx); + let ws_receiver = Arc::new(ws_rx); + WebSocket::start_server(ws_sender, ws_receiver, settings.node()).await?; + Ok(()) } } - - Ok(()) } diff --git a/homestar-runtime/src/network/client.rs b/homestar-runtime/src/network/client.rs deleted file mode 100644 index d7c2d179..00000000 --- a/homestar-runtime/src/network/client.rs +++ /dev/null @@ -1,179 +0,0 @@ -//! - -use crate::network::{ - eventloop::{Event, EventLoop}, - swarm::{ComposedBehaviour, Topic, TopicMessage}, -}; -use anyhow::Result; -use libp2p::{request_response::ResponseChannel, Multiaddr, PeerId, Swarm}; -use std::collections::HashSet; -use tokio::sync::{mpsc, oneshot}; - -/// A client for interacting with the [libp2p] networking layer. -#[derive(Clone, Debug)] -pub struct Client { - sender: mpsc::Sender, -} - -impl Client { - /// Initialize a client with an event [mpsc::Receiver] and [EventLoop]. - pub async fn new( - swarm: Swarm, - ) -> Result<(Self, mpsc::Receiver, EventLoop)> { - let (command_sender, command_receiver) = mpsc::channel(1); - let (event_sender, event_receiver) = mpsc::channel(1); - - Ok(( - Client { - sender: command_sender, - }, - event_receiver, - EventLoop::new(swarm, command_receiver, event_sender), - )) - } - - /// Publish a [message] to a topic on a running pubsub protocol. - /// - /// [message]: TopicMessage - pub async fn publish_message(&self, topic: Topic, msg: TopicMessage) -> Result<()> { - let (sender, receiver) = oneshot::channel(); - self.sender - .send(Command::PublishMessage { msg, sender, topic }) - .await?; - receiver.await? - } - - /// Listen for incoming connections on the given address. - pub async fn start_listening(&mut self, addr: Multiaddr) -> Result<()> { - let (sender, receiver) = oneshot::channel(); - self.sender - .send(Command::StartListening { addr, sender }) - .await?; - receiver.await? - } - - /// Dial the given peer at the given address. - pub async fn dial(&mut self, peer_id: PeerId, peer_addr: Multiaddr) -> Result<()> { - let (sender, receiver) = oneshot::channel(); - self.sender - .send(Command::Dial { - peer_id, - peer_addr, - sender, - }) - .await?; - receiver.await? - } - - /// Advertise the local node as the provider of the given file on the DHT. - pub async fn start_providing(&mut self, file_name: String) -> Result<()> { - let (sender, receiver) = oneshot::channel(); - self.sender - .send(Command::StartProviding { file_name, sender }) - .await?; - receiver.await? - } - - /// Find the providers for the given file on the DHT. - pub async fn get_providers(&mut self, file_name: String) -> Result> { - let (sender, receiver) = oneshot::channel(); - self.sender - .send(Command::GetProviders { file_name, sender }) - .await?; - receiver.await? - } - - /// Request the content of the given file from the given peer. - pub async fn request_file(&mut self, peer: PeerId, file_name: String) -> Result> { - let (sender, receiver) = oneshot::channel(); - self.sender - .send(Command::RequestFile { - file_name, - peer, - sender, - }) - .await?; - receiver.await? - } - - /// Respond with the provided file content to the given request. - pub async fn respond_file( - &mut self, - file: Vec, - channel: ResponseChannel, - ) -> Result<()> { - self.sender - .send(Command::RespondFile { file, channel }) - .await?; - Ok(()) - } -} - -/// Wrapper-type for file request name. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileRequest(pub(crate) String); - -/// Wrapper-type for file response content/bytes. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileResponse(pub(crate) Vec); - -#[derive(Debug)] -/// [Client] commands. -pub enum Command { - /// Start listening on an address. - StartListening { - /// Address to listen on. - addr: Multiaddr, - /// Channel to send on. - sender: oneshot::Sender>, - }, - /// Dial a peer in the cluster. - Dial { - /// Peer identifier. - peer_id: PeerId, - /// Peer address. - peer_addr: Multiaddr, - /// Channel to send on. - sender: oneshot::Sender>, - }, - /// Start providing content over channel. - StartProviding { - /// File. - file_name: String, - /// Channel to send on. - sender: oneshot::Sender>, - }, - /// Lookup providers for given key (file). - GetProviders { - /// File. - file_name: String, - /// Channel to send on. - sender: oneshot::Sender>>, - }, - /// Request file from peer. - /// TODO: File type(s)? - RequestFile { - /// File. - file_name: String, - /// Peer identifier. - peer: PeerId, - /// Channel to send on. - sender: oneshot::Sender>>, - }, - /// Respond with file content. - RespondFile { - /// File content. - file: Vec, - /// Channel to send on. - channel: ResponseChannel, - }, - /// Publish message on the topic name. - PublishMessage { - /// Pubsub (i.e. [libp2p::gossipsub]) topic. - topic: Topic, - /// [TopicMessage]. - msg: TopicMessage, - /// Channel to send on. - sender: oneshot::Sender>, - }, -} diff --git a/homestar-runtime/src/network/eventloop.rs b/homestar-runtime/src/network/eventloop.rs index 5572ffc0..ba62bea2 100644 --- a/homestar-runtime/src/network/eventloop.rs +++ b/homestar-runtime/src/network/eventloop.rs @@ -1,155 +1,328 @@ //! [EventLoop] implementation for handling network events and messages, as well //! as commands for the running [libp2p] node. +use super::swarm::TopicMessage; +#[cfg(feature = "ipfs")] +use crate::IpfsCli; use crate::{ - network::{ - client::{Command, FileRequest, FileResponse}, - swarm::{ComposedBehaviour, ComposedEvent}, - }, + db::{Database, Db}, + network::swarm::{ComposedBehaviour, ComposedEvent}, + settings, + workflow::WorkflowInfo, Receipt, }; use anyhow::{anyhow, Result}; +use concat_in_place::veccat; +use crossbeam::channel; +use homestar_core::{ + consts, + workflow::{Pointer, Receipt as InvocationReceipt}, +}; +use libipld::Cid; use libp2p::{ - floodsub::FloodsubEvent, futures::StreamExt, gossipsub, - kad::{GetProvidersOk, KademliaEvent, QueryId, QueryResult}, + kad::{ + record::Key, AddProviderOk, BootstrapOk, GetProvidersOk, GetRecordOk, KademliaEvent, + PeerRecord, PutRecordOk, QueryId, QueryResult, Quorum, Record, + }, mdns, multiaddr::Protocol, - request_response::{self, RequestId, ResponseChannel}, swarm::{Swarm, SwarmEvent}, }; -use libp2p_identity::PeerId; -use std::collections::{hash_map, HashMap, HashSet}; -use tokio::sync::{mpsc, oneshot}; +use std::{collections::HashMap, fmt, num::NonZeroUsize, str}; +use tokio::sync::mpsc; /// [Receipt]-related topic for pub(gossip)sub. /// /// [Receipt]: homestar_core::workflow::receipt pub const RECEIPTS_TOPIC: &str = "receipts"; +type WorkerSender = channel::Sender<(Cid, Receipt)>; + /// Event loop handler for [libp2p] network events and commands. #[allow(missing_debug_implementations)] pub struct EventLoop { + receiver: mpsc::Receiver, + receipt_quorum: usize, + worker_senders: HashMap, swarm: Swarm, - command_receiver: mpsc::Receiver, - event_sender: mpsc::Sender, - pending_dial: HashMap>>, - pending_start_providing: HashMap>>, - pending_get_providers: HashMap>>>, - pending_request_file: HashMap>>>, } impl EventLoop { + /// Setup bounded, MPSC channel for runtime to send and receive internal + /// events with workers. + pub fn setup_channel( + settings: &settings::Node, + ) -> (mpsc::Sender, mpsc::Receiver) { + mpsc::channel(settings.network.events_buffer_len) + } + /// Create an [EventLoop] with channel sender/receiver defaults. pub fn new( swarm: Swarm, - command_receiver: mpsc::Receiver, - event_sender: mpsc::Sender, + receiver: mpsc::Receiver, + settings: &settings::Node, ) -> Self { Self { + receiver, + receipt_quorum: settings.network.receipt_quorum, + worker_senders: HashMap::new(), swarm, - command_receiver, - event_sender, - pending_dial: Default::default(), - pending_start_providing: Default::default(), - pending_get_providers: Default::default(), - pending_request_file: Default::default(), } } /// Loop and select over swarm and pubsub [events] and client [commands]. /// /// [events]: SwarmEvent - /// [commands]: Command - pub async fn run(mut self) -> Result<()> { + #[cfg(not(feature = "ipfs"))] + pub async fn run(mut self, db: Db) -> Result<()> { loop { tokio::select! { - event = self.swarm.select_next_some() => self.handle_event(event).await, - command = self.command_receiver.recv() => if let Some(c) = command {self.handle_command(c).await} + swarm_event = self.swarm.select_next_some() => self.handle_event(swarm_event, db.clone()).await, + runtime_event = self.receiver.recv() => if let Some(ev) = runtime_event { self.handle_runtime_event(ev).await }, } } } - async fn handle_event( - &mut self, - event: SwarmEvent, - ) { + /// Loop and select over swarm and pubsub [events]. + /// + /// [events]: SwarmEvent + #[cfg(feature = "ipfs")] + pub async fn run(mut self, db: Db, ipfs: IpfsCli) -> Result<()> { + loop { + tokio::select! { + swarm_event = self.swarm.select_next_some() => self.handle_event(swarm_event, db.clone()).await, + runtime_event = self.receiver.recv() => if let Some(ev) = runtime_event { self.handle_runtime_event(ev, ipfs.clone()).await }, + } + } + } + + #[cfg(not(feature = "ipfs"))] + async fn handle_runtime_event(&mut self, event: Event) { match event { - SwarmEvent::Behaviour(ComposedEvent::Floodsub(FloodsubEvent::Message(message))) => { - match Receipt::try_from(message.data) { - Ok(receipt) => println!("got message: {receipt}"), + Event::CapturedReceipt(receipt, workflow_info) => { + match self.on_capture(receipt, workflow_info) { + Ok((cid, _bytes)) => { + tracing::debug!( + cid = cid, + "record replicated with quorum {}", + self.receipt_quorum + ) + } Err(err) => { - println!("cannot handle_message: {err}") + tracing::error!(error=?err, "error putting record on DHT with quorum {}", self.receipt_quorum) } } } - SwarmEvent::Behaviour(ComposedEvent::Floodsub(FloodsubEvent::Subscribed { - peer_id, - topic, - })) => { - println!("{peer_id} subscribed to topic {} over pubsub", topic.id()) + Event::FindReceipt(cid, sender) => self.on_find_receipt(cid, sender), + } + } + + #[cfg(feature = "ipfs")] + async fn handle_runtime_event(&mut self, event: Event, ipfs: IpfsCli) { + match event { + Event::CapturedReceipt(receipt, workflow_info) => { + match self.on_capture(receipt, workflow_info) { + Ok((cid, bytes)) => { + tracing::debug!( + cid = cid, + "record replicated with quorum {}", + self.receipt_quorum + ); + + // Spawn client call in background, without awaiting. + tokio::spawn(async move { + match ipfs.put_receipt_bytes(bytes.to_vec()).await { + Ok(put_cid) => { + tracing::info!(cid = put_cid, "IPLD DAG node stored"); + + #[cfg(debug_assertions)] + debug_assert_eq!(put_cid, cid); + } + Err(err) => { + tracing::info!(error=?err, cid=cid, "Failed to store IPLD DAG node") + } + } + }); + } + Err(err) => { + tracing::error!(error=?err, "error putting record on DHT with quorum {}", self.receipt_quorum) + } + } } - SwarmEvent::Behaviour(ComposedEvent::Floodsub(_)) => {} + Event::FindReceipt(cid, sender) => self.on_find_receipt(cid, sender), + } + } + + fn on_capture( + &mut self, + receipt: Receipt, + mut workflow_info: WorkflowInfo, + ) -> Result<(String, Vec)> { + let receipt_cid = receipt.cid(); + let invocation_receipt = InvocationReceipt::from(&receipt); + let instruction_bytes = receipt.instruction_cid_as_bytes(); + match self.swarm.behaviour_mut() + .gossip_publish(RECEIPTS_TOPIC, TopicMessage::CapturedReceipt(receipt)) { + Ok(msg_id) => + tracing::info!("message {msg_id} published on {RECEIPTS_TOPIC} for receipt with cid: {receipt_cid}"), + Err(err) => tracing::error!(error=?err, "message not published on {RECEIPTS_TOPIC} for receipt with cid: {receipt_cid}") + } + + let quorum = if self.receipt_quorum > 0 { + unsafe { Quorum::N(NonZeroUsize::new_unchecked(self.receipt_quorum)) } + } else { + Quorum::One + }; + + if let Ok(receipt_bytes) = Vec::try_from(invocation_receipt) { + let ref_bytes = &receipt_bytes; + let value = veccat!(consts::INVOCATION_VERSION.as_bytes() ref_bytes); + let _id = self + .swarm + .behaviour_mut() + .kademlia + .put_record(Record::new(instruction_bytes, value.to_vec()), quorum) + .map_err(anyhow::Error::msg)?; + + // increment progress with capture + workflow_info.increment_progress(); + let _id = self + .swarm + .behaviour_mut() + .kademlia + .put_record( + Record::new(workflow_info.cid.to_bytes(), Vec::try_from(workflow_info)?), + quorum, + ) + .map_err(anyhow::Error::msg)?; + + Ok((receipt_cid, receipt_bytes)) + } else { + Err(anyhow!("cannot convert receipt {receipt_cid} to bytes")) + } + } + + fn on_find_receipt(&mut self, instruction_cid: Cid, sender: WorkerSender) { + let id = self + .swarm + .behaviour_mut() + .kademlia + .get_record(Key::new(&instruction_cid.to_bytes())); + self.worker_senders.insert(id, sender); + } + + fn on_found_record(key_cid: Cid, value: Vec) -> Result { + if value.starts_with(consts::INVOCATION_VERSION.as_bytes()) { + let receipt_bytes = &value[consts::INVOCATION_VERSION.as_bytes().len()..]; + let invocation_receipt = InvocationReceipt::try_from(receipt_bytes.to_vec())?; + Receipt::try_with(Pointer::new(key_cid), &invocation_receipt) + } else { + Err(anyhow!( + "receipt version mismatch, current version: {}", + consts::INVOCATION_VERSION + )) + } + } + + async fn handle_event( + &mut self, + event: SwarmEvent, + db: Db, + ) { + match event { SwarmEvent::Behaviour(ComposedEvent::Gossipsub(gossipsub::Event::Message { message, propagation_source, message_id, - })) => - match Receipt::try_from(message.data) { - Ok(receipt) => println!( + })) => match Receipt::try_from(message.data) { + Ok(receipt) => { + tracing::info!( "got message: {receipt} from {propagation_source} with message id: {message_id}" - ), + ); - Err(err) => println!( - "cannot handle_message: {err}" - ) - } + // Store gossiped receipt. + let _ = db + .conn() + .as_mut() + .map(|conn| Db::store_receipt(receipt, conn)); + } + Err(err) => tracing::info!(err=?err, "cannot handle incoming event message"), + }, SwarmEvent::Behaviour(ComposedEvent::Gossipsub(gossipsub::Event::Subscribed { peer_id, topic, })) => { - println!("{peer_id} subscribed to topic {topic} over gossipsub") + tracing::debug!("{peer_id} subscribed to topic {topic} over gossipsub") } SwarmEvent::Behaviour(ComposedEvent::Gossipsub(_)) => {} SwarmEvent::Behaviour(ComposedEvent::Kademlia( - KademliaEvent::OutboundQueryProgressed { - id, - result: QueryResult::StartProviding(_), + KademliaEvent::OutboundQueryProgressed { id, result, .. }, + )) => match result { + QueryResult::Bootstrap(Ok(BootstrapOk { peer, .. })) => { + tracing::debug!("successfully bootstrapped peer: {peer}") + } + QueryResult::GetProviders(Ok(GetProvidersOk::FoundProviders { + key, + providers, .. - }, - )) => { - let sender = self - .pending_start_providing - .remove(&id) - .expect("Completed query to be previously pending"); - let _ = sender.send(Ok(())); - } - SwarmEvent::Behaviour(ComposedEvent::Kademlia( - KademliaEvent::OutboundQueryProgressed { - id, - result: - QueryResult::GetProviders(Ok(GetProvidersOk::FoundProviders { - providers, .. - })), + })) => { + for peer in providers { + tracing::debug!("peer {peer} provides key: {key:#?}"); + } + } + QueryResult::GetProviders(Err(err)) => { + tracing::error!("error retrieving outbound query providers: {err}") + } + QueryResult::GetRecord(Ok(GetRecordOk::FoundRecord(PeerRecord { + record: + Record { + key, + value, + publisher, + .. + }, .. - }, - )) => { - let _ = self - .pending_get_providers - .remove(&id) - .expect("Completed query to be previously pending") - .send(Ok(providers)); - } + }))) => { + tracing::debug!("found record {key:#?}, published by {publisher:?}"); + if let Ok(cid) = Cid::try_from(key.as_ref()) { + match Self::on_found_record(cid, value) { + Ok(receipt) => { + tracing::info!("found receipt: {receipt}"); + if let Some(sender) = self.worker_senders.remove(&id) { + let _ = sender.send((cid, receipt)); + } else { + tracing::error!("error converting key {key:#?} to cid") + } + } + Err(err) => tracing::error!(err=?err, "error retrieving receipt"), + } + } + } + QueryResult::GetRecord(Ok(_)) => {} + QueryResult::GetRecord(Err(err)) => { + tracing::error!("error retrieving record: {err}"); + } + QueryResult::PutRecord(Ok(PutRecordOk { key })) => { + tracing::debug!("successfully put record {key:#?}"); + } + QueryResult::PutRecord(Err(err)) => { + tracing::error!("error putting record: {err}") + } + QueryResult::StartProviding(Ok(AddProviderOk { key })) => { + tracing::debug!("successfully put provider record {key:#?}"); + } + QueryResult::StartProviding(Err(err)) => { + tracing::error!("error putting provider record: {err}"); + } + _ => {} + }, SwarmEvent::Behaviour(ComposedEvent::Kademlia(_)) => {} SwarmEvent::Behaviour(ComposedEvent::Mdns(mdns::Event::Discovered(list))) => { for (peer_id, _multiaddr) in list { - println!("mDNS discovered a new peer: {peer_id}"); - self.swarm - .behaviour_mut() - .floodsub - .add_node_to_partial_view(peer_id); + tracing::info!("mDNS discovered a new peer: {peer_id}"); self.swarm .behaviour_mut() @@ -159,12 +332,7 @@ impl EventLoop { } SwarmEvent::Behaviour(ComposedEvent::Mdns(mdns::Event::Expired(list))) => { for (peer_id, _multiaddr) in list { - println!("mDNS discover peer has expired: {peer_id}"); - - self.swarm - .behaviour_mut() - .floodsub - .remove_node_from_partial_view(&peer_id); + tracing::info!("mDNS discover peer has expired: {peer_id}"); self.swarm .behaviour_mut() @@ -172,174 +340,48 @@ impl EventLoop { .remove_explicit_peer(&peer_id); } } - SwarmEvent::Behaviour(ComposedEvent::RequestResponse( - request_response::Event::Message { message, .. }, - )) => match message { - request_response::Message::Request { - request, channel, .. - } => { - self.event_sender - .send(Event::InboundRequest { - request: request.0, - channel, - }) - .await - .expect("Event receiver not to be dropped"); - } - request_response::Message::Response { - request_id, - response, - } => { - let _ = self - .pending_request_file - .remove(&request_id) - .expect("Request to still be pending") - .send(Ok(response.0)); - } - }, - SwarmEvent::Behaviour(ComposedEvent::RequestResponse( - request_response::Event::InboundFailure { - request_id: _, - error: _, - .. - }, - )) => {} - SwarmEvent::Behaviour(ComposedEvent::RequestResponse( - request_response::Event::OutboundFailure { - request_id, error, .. - }, - )) => { - let _ = self - .pending_request_file - .remove(&request_id) - .expect("Request to still be pending") - .send(Err(anyhow!(error))); - } - SwarmEvent::Behaviour(ComposedEvent::RequestResponse( - request_response::Event::ResponseSent { .. }, - )) => {} SwarmEvent::NewListenAddr { address, .. } => { let local_peer_id = *self.swarm.local_peer_id(); - println!( + tracing::info!( "local node is listening on {:?}", address.with(Protocol::P2p(local_peer_id.into())) ); } SwarmEvent::IncomingConnection { .. } => {} - SwarmEvent::ConnectionEstablished { - peer_id, endpoint, .. - } => { - if endpoint.is_dialer() { - if let Some(sender) = self.pending_dial.remove(&peer_id) { - let _ = sender.send(Ok(())); - } - } - } - SwarmEvent::ConnectionClosed { .. } => {} - SwarmEvent::OutgoingConnectionError { peer_id, error, .. } => { - if let Some(peer_id) = peer_id { - if let Some(sender) = self.pending_dial.remove(&peer_id) { - let _ = sender.send(Err(anyhow!(error))); - } - } - } - SwarmEvent::IncomingConnectionError { .. } => {} - SwarmEvent::Dialing(peer_id) => println!("dialing {peer_id}"), - e => panic!("{e:?}"), - } - } - - async fn handle_command(&mut self, command: Command) { - match command { - Command::PublishMessage { topic, msg, sender } => { - let _ = match self - .swarm - .behaviour_mut() - .gossip_publish(&topic.to_string(), msg) - { - Ok(_) => sender.send(Ok(())), - Err(e) => sender.send(Err(anyhow!(e))), - }; - } - Command::StartListening { addr, sender } => { - let _ = match self.swarm.listen_on(addr) { - Ok(_) => sender.send(Ok(())), - Err(e) => sender.send(Err(anyhow!(e))), - }; - } - Command::Dial { - peer_id, - peer_addr, - sender, - } => { - if let hash_map::Entry::Vacant(e) = self.pending_dial.entry(peer_id) { - self.swarm - .behaviour_mut() - .kademlia - .add_address(&peer_id, peer_addr.clone()); - match self - .swarm - .dial(peer_addr.with(Protocol::P2p(peer_id.into()))) - { - Ok(()) => { - e.insert(sender); - } - Err(e) => { - let _ = sender.send(Err(anyhow!(e))); - } - } - } else { - todo!("Already dialing peer."); - } - } - Command::StartProviding { file_name, sender } => { - let query_id = self - .swarm - .behaviour_mut() - .kademlia - .start_providing(file_name.into_bytes().into()) - .expect("No store error"); - self.pending_start_providing.insert(query_id, sender); - } - Command::GetProviders { file_name, sender } => { - let query_id = self - .swarm - .behaviour_mut() - .kademlia - .get_providers(file_name.into_bytes().into()); - self.pending_get_providers.insert(query_id, sender); - } - Command::RequestFile { - file_name, - peer, - sender, - } => { - let request_id = self - .swarm - .behaviour_mut() - .request_response - .send_request(&peer, FileRequest(file_name)); - self.pending_request_file.insert(request_id, sender); - } - Command::RespondFile { file, channel } => { - self.swarm - .behaviour_mut() - .request_response - .send_response(channel, FileResponse(file)) - .expect("Connection to peer to be still open"); - } + _ => {} } } } /// Internal events to capture. -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum Event { - /// Information about a received and handled inbound request. - InboundRequest { - /// Request name or [libipld::cid::Cid]. - request: String, - /// Response channel. - channel: ResponseChannel, - }, + /// [Receipt] stored and captured event. + CapturedReceipt(Receipt, WorkflowInfo), + /// Find a [Receipt] stored in the DHT. + /// + /// [Receipt]: InvocationReceipt + FindReceipt(Cid, WorkerSender), +} + +#[cfg(test)] +mod test { + use super::*; + use crate::test_utils; + + #[test] + fn found_record() { + let (invocation_receipt, receipt) = test_utils::receipt::receipts(); + let instruction_bytes = receipt.instruction_cid_as_bytes(); + let bytes = Vec::try_from(invocation_receipt).unwrap(); + let ref_bytes = &bytes; + let value = veccat!(consts::INVOCATION_VERSION.as_bytes() ref_bytes); + let record = Record::new(instruction_bytes, value.to_vec()); + let record_value = record.value; + let found_receipt = + EventLoop::on_found_record(Cid::try_from(receipt.instruction()).unwrap(), record_value) + .unwrap(); + + assert_eq!(found_receipt, receipt); + } } diff --git a/homestar-runtime/src/network/ipfs.rs b/homestar-runtime/src/network/ipfs.rs new file mode 100644 index 00000000..1db3f458 --- /dev/null +++ b/homestar-runtime/src/network/ipfs.rs @@ -0,0 +1,72 @@ +//! Ipfs Client container for an [Arc]'ed [IpfsClient]. + +use anyhow::Result; +use futures::TryStreamExt; +use homestar_core::workflow::Receipt; +use ipfs_api::{ + request::{DagCodec, DagPut}, + response::DagPutResponse, + IpfsApi, IpfsClient, +}; +use libipld::{Cid, Ipld}; +use std::{io::Cursor, sync::Arc}; +use url::Url; + +const SHA3_256: &str = "sha3-256"; + +/// [IpfsClient]-wrapper. +#[allow(missing_debug_implementations)] +pub struct IpfsCli(Arc); + +impl Clone for IpfsCli { + fn clone(&self) -> Self { + IpfsCli(Arc::clone(&self.0)) + } +} + +impl Default for IpfsCli { + fn default() -> Self { + Self(Arc::new(IpfsClient::default())) + } +} + +impl IpfsCli { + /// Retrieve content from a [Url]. + pub async fn get_resource(&self, url: &Url) -> Result> { + let cid = Cid::try_from(url.to_string())?; + self.get_cid(cid).await + } + + /// Retrieve content from a [Cid]. + pub async fn get_cid(&self, cid: Cid) -> Result> { + self.0 + .cat(&cid.to_string()) + .map_ok(|chunk| chunk.to_vec()) + .try_concat() + .await + .map_err(Into::into) + } + + /// Put/Write [Receipt] into IPFS. + pub async fn put_receipt(&self, receipt: Receipt) -> Result { + let receipt_bytes: Vec = receipt.try_into()?; + self.put_receipt_bytes(receipt_bytes).await + } + + /// Put/Write [Receipt], as bytes, into IPFS. + pub async fn put_receipt_bytes(&self, receipt_bytes: Vec) -> Result { + let dag_builder = DagPut::builder() + .store_codec(DagCodec::Cbor) + .input_codec(DagCodec::Cbor) + .hash(SHA3_256) // sadly no support for blake3-256 + .build(); + + let DagPutResponse { cid } = self + .0 + .dag_put_with_options(Cursor::new(receipt_bytes.clone()), dag_builder) + .await + .expect("a CID"); + + Ok(cid.cid_string) + } +} diff --git a/homestar-runtime/src/network/mod.rs b/homestar-runtime/src/network/mod.rs index 6eb9f348..27d0bbb4 100644 --- a/homestar-runtime/src/network/mod.rs +++ b/homestar-runtime/src/network/mod.rs @@ -1,8 +1,12 @@ -//! [libp2p] networking interface. +//! [libp2p], [websocket], and [ipfs] networking interfaces. //! //! [libp2p]: +//! [websocket]: ws +//! [ipfs]: ipfs -pub mod client; pub mod eventloop; +#[cfg(feature = "ipfs")] +pub mod ipfs; pub mod pubsub; pub mod swarm; +pub mod ws; diff --git a/homestar-runtime/src/network/pubsub.rs b/homestar-runtime/src/network/pubsub.rs index bfa399e4..876334aa 100644 --- a/homestar-runtime/src/network/pubsub.rs +++ b/homestar-runtime/src/network/pubsub.rs @@ -1,11 +1,9 @@ -//! [gossipsub] and [Floodsub] initializers for PubSub across connected peers. +//! [gossipsub] initializer for PubSub across connected peers. use anyhow::Result; use libp2p::{ - floodsub::Floodsub, gossipsub::{self, ConfigBuilder, Message, MessageAuthenticity, MessageId, ValidationMode}, identity::Keypair, - PeerId, }; use std::{ collections::hash_map::DefaultHasher, @@ -13,13 +11,10 @@ use std::{ time::Duration, }; -/// Setup direct [Floodsub] protocol with a given [PeerId]. -pub fn new_floodsub(peer_id: PeerId) -> Floodsub { - Floodsub::new(peer_id) -} +use crate::settings; /// Setup [gossipsub] mesh protocol with default configuration. -pub fn new_gossipsub(keypair: Keypair) -> Result { +pub fn new(keypair: Keypair, settings: &settings::Node) -> Result { // To content-address message, we can take the hash of message and use it as an ID. let message_id_fn = |message: &Message| { let mut s = DefaultHasher::new(); @@ -27,9 +22,8 @@ pub fn new_gossipsub(keypair: Keypair) -> Result { MessageId::from(s.finish().to_string()) }; - // TODO: Make configurable let gossipsub_config = ConfigBuilder::default() - .heartbeat_interval(Duration::from_secs(10)) + .heartbeat_interval(Duration::from_secs(settings.network.pubsub_heartbeat_secs)) // This sets the kind of message validation. The default is Strict (enforce message signing). .validation_mode(ValidationMode::Strict) .mesh_n_low(1) diff --git a/homestar-runtime/src/network/swarm.rs b/homestar-runtime/src/network/swarm.rs index acebd9d6..0961e801 100644 --- a/homestar-runtime/src/network/swarm.rs +++ b/homestar-runtime/src/network/swarm.rs @@ -1,60 +1,49 @@ //! Sets up a [libp2p] [Swarm], containing the state of the network and the way //! it should behave. -use crate::{ - network::{ - client::{FileRequest, FileResponse}, - pubsub, - }, - Receipt, -}; +use crate::{network::pubsub, settings, Receipt}; use anyhow::{anyhow, Result}; -use async_trait::async_trait; use libp2p::{ - core::upgrade::{self, read_length_prefixed, write_length_prefixed, ProtocolName}, - floodsub::{self, Floodsub, FloodsubEvent}, - futures::{AsyncRead, AsyncWrite, AsyncWriteExt}, + core::upgrade, gossipsub::{self, MessageId, SubscriptionError, TopicHash}, identity::Keypair, kad::{record::store::MemoryStore, Kademlia, KademliaEvent}, mdns, noise, - request_response::{self, ProtocolSupport}, swarm::{NetworkBehaviour, Swarm, SwarmBuilder}, - tcp, - yamux::YamuxConfig, - Transport, + tcp, yamux, Transport, }; -use std::{fmt, io, iter}; +use std::fmt; /// Build a new [Swarm] with a given transport and a tokio executor. -pub async fn new(keypair: Keypair) -> Result> { +pub async fn new(settings: &settings::Node) -> Result> { + let keypair = Keypair::generate_ed25519(); let peer_id = keypair.public().to_peer_id(); let transport = tcp::tokio::Transport::new(tcp::Config::default().nodelay(true)) - .upgrade(upgrade::Version::V1) + .upgrade(upgrade::Version::V1Lazy) .authenticate( - noise::NoiseAuthenticated::xx(&keypair) - .expect("Signing libp2p-noise static DH keypair failed"), + noise::Config::new(&keypair).expect("Signing libp2p-noise static DH keypair failed"), ) - .multiplex(YamuxConfig::default()) + .multiplex(yamux::Config::default()) + // TODO: configure + //.timeout(Duration::from_secs(5)) .boxed(); - Ok(SwarmBuilder::with_tokio_executor( + let mut swarm = SwarmBuilder::with_tokio_executor( transport, ComposedBehaviour { - floodsub: pubsub::new_floodsub(peer_id), - gossipsub: pubsub::new_gossipsub(keypair)?, + gossipsub: pubsub::new(keypair, settings)?, kademlia: Kademlia::new(peer_id, MemoryStore::new(peer_id)), mdns: mdns::Behaviour::new(mdns::Config::default(), peer_id)?, - request_response: request_response::Behaviour::new( - FileExchangeCodec(), - iter::once((FileExchangeProtocol(), ProtocolSupport::Full)), - Default::default(), - ), }, peer_id, ) - .build()) + .build(); + + // Listen-on given address + swarm.listen_on(settings.network.listen_address.to_string().parse()?)?; + + Ok(swarm) } /// Custom event types to listen for and respond to. @@ -62,14 +51,10 @@ pub async fn new(keypair: Keypair) -> Result> { pub enum ComposedEvent { /// [gossipsub::Event] event. Gossipsub(gossipsub::Event), - /// [floodsub::FloodsubEvent] event. - Floodsub(FloodsubEvent), /// [KademliaEvent] event. Kademlia(KademliaEvent), /// [mdns::Event] event. Mdns(mdns::Event), - /// [request_response::Event] event. - RequestResponse(request_response::Event), } /// Message topic. @@ -93,7 +78,7 @@ impl Topic { #[derive(Debug)] pub enum TopicMessage { /// Receipt topic, wrapping [Receipt]. - Receipt(Receipt), + CapturedReceipt(Receipt), } /// Custom behaviours for [Swarm]. @@ -103,33 +88,13 @@ pub enum TopicMessage { pub struct ComposedBehaviour { /// [gossipsub::Behaviour] behaviour. pub gossipsub: gossipsub::Behaviour, - /// [floodsub::Floodsub] behaviour. - pub floodsub: Floodsub, /// In-memory [kademlia: Kademlia] behaviour. pub kademlia: Kademlia, /// [mdns::tokio::Behaviour] behaviour. pub mdns: mdns::tokio::Behaviour, - /// [request_response::Behaviour] behaviour. - pub request_response: request_response::Behaviour, } impl ComposedBehaviour { - /// Subscribe to [Floodsub] topic. - pub fn subscribe(&mut self, topic: &str) -> bool { - let topic = floodsub::Topic::new(topic); - self.floodsub.subscribe(topic) - } - - /// Serialize [TopicMessage] and publish to [Floodsub] topic. - pub fn publish(&mut self, topic: &str, msg: TopicMessage) -> Result<()> { - let id_topic = floodsub::Topic::new(topic); - // Make this an or msg to match on other topics. - let TopicMessage::Receipt(receipt) = msg; - let msg_bytes: Vec = receipt.try_into()?; - self.floodsub.publish(id_topic, msg_bytes); - Ok(()) - } - /// Subscribe to [gossipsub] topic. pub fn gossip_subscribe(&mut self, topic: &str) -> Result { let topic = gossipsub::IdentTopic::new(topic); @@ -139,8 +104,8 @@ impl ComposedBehaviour { /// Serialize [TopicMessage] and publish to [gossipsub] topic. pub fn gossip_publish(&mut self, topic: &str, msg: TopicMessage) -> Result { let id_topic = gossipsub::IdentTopic::new(topic); - // Make this an or msg to match on other topics. - let TopicMessage::Receipt(receipt) = msg; + // Make this a match once we have other topics. + let TopicMessage::CapturedReceipt(receipt) = msg; let msg_bytes: Vec = receipt.try_into()?; if self .gossipsub @@ -165,12 +130,6 @@ impl From for ComposedEvent { } } -impl From for ComposedEvent { - fn from(event: FloodsubEvent) -> Self { - ComposedEvent::Floodsub(event) - } -} - impl From for ComposedEvent { fn from(event: KademliaEvent) -> Self { ComposedEvent::Kademlia(event) @@ -182,94 +141,3 @@ impl From for ComposedEvent { ComposedEvent::Mdns(event) } } - -impl From> for ComposedEvent { - fn from(event: request_response::Event) -> Self { - ComposedEvent::RequestResponse(event) - } -} - -/// Simple file-exchange protocol. -#[derive(Debug, Clone)] -pub struct FileExchangeProtocol(); - -/// File-exchange codec. -#[derive(Debug, Clone)] -pub struct FileExchangeCodec(); - -impl ProtocolName for FileExchangeProtocol { - fn protocol_name(&self) -> &[u8] { - "/file-exchange/1".as_bytes() - } -} - -#[async_trait] -impl request_response::codec::Codec for FileExchangeCodec { - type Protocol = FileExchangeProtocol; - type Request = FileRequest; - type Response = FileResponse; - - async fn read_request( - &mut self, - _: &FileExchangeProtocol, - io: &mut T, - ) -> io::Result - where - T: AsyncRead + Unpin + Send, - { - let vec = read_length_prefixed(io, 1_000_000).await?; - - if vec.is_empty() { - return Err(io::ErrorKind::UnexpectedEof.into()); - } - - Ok(FileRequest(String::from_utf8(vec).unwrap())) - } - - async fn read_response( - &mut self, - _: &FileExchangeProtocol, - io: &mut T, - ) -> io::Result - where - T: AsyncRead + Unpin + Send, - { - let vec = read_length_prefixed(io, 500_000_000).await?; // update transfer maximum - - if vec.is_empty() { - return Err(io::ErrorKind::UnexpectedEof.into()); - } - - Ok(FileResponse(vec)) - } - - async fn write_request( - &mut self, - _: &FileExchangeProtocol, - io: &mut T, - FileRequest(data): FileRequest, - ) -> io::Result<()> - where - T: AsyncWrite + Unpin + Send, - { - write_length_prefixed(io, data).await?; - io.close().await?; - - Ok(()) - } - - async fn write_response( - &mut self, - _: &FileExchangeProtocol, - io: &mut T, - FileResponse(data): FileResponse, - ) -> io::Result<()> - where - T: AsyncWrite + Unpin + Send, - { - write_length_prefixed(io, data).await?; - io.close().await?; - - Ok(()) - } -} diff --git a/homestar-runtime/src/network/ws.rs b/homestar-runtime/src/network/ws.rs new file mode 100644 index 00000000..9a233370 --- /dev/null +++ b/homestar-runtime/src/network/ws.rs @@ -0,0 +1,226 @@ +//! Sets up a websocket server for sending and receiving messages from browser +//! clients. + +use crate::settings; +use anyhow::{anyhow, Result}; +use axum::{ + extract::{ + ws::{self, Message, WebSocketUpgrade}, + ConnectInfo, State, TypedHeader, + }, + response::IntoResponse, + routing::get, + Router, +}; +use futures::{stream::StreamExt, SinkExt}; +use std::{ + net::{IpAddr, SocketAddr, TcpListener}, + ops::ControlFlow, + str::FromStr, + sync::Arc, +}; +use tokio::sync::broadcast; + +/// WebSocket state information. +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub struct WebSocket { + addr: SocketAddr, + sender: Arc>, + receiver: Arc>, +} + +impl WebSocket { + /// Setup bounded, MPMC channel for runtime to send and received messages + /// through the websocket connection(s). + pub fn setup_channel( + settings: &settings::Node, + ) -> (broadcast::Sender, broadcast::Receiver) { + broadcast::channel(settings.network.websocket_capacity) + } + + /// Start the websocket server given settings. + pub async fn start_server( + sender: Arc>, + receiver: Arc>, + settings: &settings::Node, + ) -> Result<()> { + let host = IpAddr::from_str(&settings.network.websocket_host.to_string())?; + let addr = if port_available(host, settings.network.websocket_port) { + SocketAddr::from((host, settings.network.websocket_port)) + } else { + let port = (settings.network.websocket_port..settings.network.websocket_port + 1000) + .find(|port| port_available(host, *port)) + .ok_or_else(|| anyhow!("no free TCP ports available"))?; + SocketAddr::from((host, port)) + }; + + let ws_state = Self { + addr, + sender, + receiver, + }; + let app = Router::new().route("/", get(ws_handler).with_state(ws_state)); + + tokio::spawn(async move { + axum::Server::bind(&addr) + .serve(app.into_make_service_with_connect_info::()) + .await + .expect("Websocket server to start"); + }); + + tracing::info!("websocket server starting on {addr}"); + + Ok(()) + } +} + +async fn ws_handler( + ws: WebSocketUpgrade, + user_agent: Option>, + State(state): State, + ConnectInfo(addr): ConnectInfo, +) -> impl IntoResponse { + let user_agent = if let Some(TypedHeader(user_agent)) = user_agent { + user_agent.to_string() + } else { + String::from("Unknown browser") + }; + tracing::info!("`{user_agent}` at {addr} connected."); + + // Finalize the upgrade process by returning upgrade callback. + // We can customize the callback by sending additional info such as address. + ws.on_upgrade(move |socket| handle_socket(socket, state)) +} + +async fn handle_socket(mut socket: ws::WebSocket, state: WebSocket) { + let addr = state.addr; + + // Send a ping (unsupported by some browsers) just to kick things off and + // get a response. + if socket.send(Message::Ping(vec![1, 2, 3])).await.is_ok() { + tracing::debug!("Pinged {}...", addr); + } else { + tracing::info!("Could not send ping {}!", addr); + // no Error here since the only thing we can do is to close the connection. + // If we can not send messages, there is no way to salvage the statemachine anyway. + return; + } + + // Receive single message from a client (we can either receive or send with + // the socket). This will likely be the Pong for our Ping or a processed + // message from client. + // Waiting for message from a client will block this task, but will not + // block other client's connections. + if let Some(msg) = socket.recv().await { + if let Ok(msg) = msg { + if process_message(msg, addr).await.is_break() { + return; + } + } else { + tracing::info!("client {} abruptly disconnected", state.addr); + return; + } + } + + // By splitting socket we can send and receive at the same time. + let (mut socket_sender, mut socket_receiver) = socket.split(); + let mut subscribed_rx = state.sender.subscribe(); + + let mut send_task = tokio::spawn(async move { + while let Ok(msg) = subscribed_rx.recv().await { + // In any websocket error, break loop. + if socket_sender + .send(Message::Binary(msg.into())) + .await + .is_err() + { + break; + } + } + }); + + let mut recv_task = tokio::spawn(async move { + let mut cnt = 0; + while let Some(Ok(msg)) = socket_receiver.next().await { + cnt += 1; + if process_message(msg, addr).await.is_break() { + break; + } + } + cnt + }); + + // If any one of the tasks exit, abort the other. + tokio::select! { + _ = (&mut send_task) => recv_task.abort(), + _ = (&mut recv_task) => send_task.abort(), + }; + + tracing::info!("Websocket context {} destroyed", addr); +} + +/// Process [messages]. +/// +/// [messages]: Message +async fn process_message(msg: Message, addr: SocketAddr) -> ControlFlow<(), ()> { + match msg { + Message::Text(t) => { + tracing::info!(">>> {} sent str: {:?}", addr, t); + } + Message::Binary(d) => { + tracing::info!(">>> {} sent {} bytes: {:?}", addr, d.len(), d); + } + Message::Close(c) => { + if let Some(cf) = c { + tracing::info!( + ">>> {} sent close with code {} and reason `{}`", + addr, + cf.code, + cf.reason + ); + } else { + tracing::info!(">>> {} somehow sent close message without CloseFrame", addr); + } + return ControlFlow::Break(()); + } + + Message::Pong(v) => { + tracing::info!(">>> {} sent pong with {:?}", addr, v); + } + // You should never need to manually handle Message::Ping, as axum's websocket library + // will do so for you automagically by replying with Pong and copying the v according to + // spec. But if you need the contents of the pings you can see them here. + Message::Ping(v) => { + tracing::info!(">>> {} sent ping with {:?}", addr, v); + } + } + ControlFlow::Continue(()) +} + +fn port_available(host: IpAddr, port: u16) -> bool { + TcpListener::bind((host.to_string(), port)).is_ok() +} + +#[cfg(test)] +mod test { + use crate::settings::Settings; + + use super::*; + + #[tokio::test] + async fn ws_connect() { + let (tx, rx) = broadcast::channel(1); + let sender = Arc::new(tx); + let receiver = Arc::new(rx); + let settings = Settings::load().unwrap(); + + WebSocket::start_server(Arc::clone(&sender), Arc::clone(&receiver), settings.node()) + .await + .unwrap(); + + tokio_tungstenite::connect_async("ws://localhost:1337".to_string()) + .await + .unwrap(); + } +} diff --git a/homestar-runtime/src/receipt.rs b/homestar-runtime/src/receipt.rs index 6a4d0d8b..8a1490c7 100644 --- a/homestar-runtime/src/receipt.rs +++ b/homestar-runtime/src/receipt.rs @@ -10,81 +10,157 @@ use diesel::{ sqlite::Sqlite, AsExpression, FromSqlRow, Insertable, Queryable, }; -use homestar_core::workflow::{ - pointer::InvocationPointer, - prf::UcanPrf, - receipt::{Issuer, Receipt as LocalReceipt}, - InvocationResult, +use homestar_core::{ + consts, + workflow::{prf::UcanPrf, InstructionResult, Issuer, Pointer, Receipt as InvocationReceipt}, }; -use libipld::{cbor::DagCborCodec, cid::Cid, prelude::Codec, serde::from_ipld, Ipld}; +use homestar_wasm::io::Arg; +use libipld::{ + cbor::DagCborCodec, cid::Cid, json::DagJsonCodec, prelude::Codec, serde::from_ipld, Ipld, +}; +use semver::Version; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fmt}; +const CID_KEY: &str = "cid"; +const INSTRUCTION_KEY: &str = "instruction"; const RAN_KEY: &str = "ran"; const OUT_KEY: &str = "out"; const ISSUER_KEY: &str = "iss"; const METADATA_KEY: &str = "meta"; const PROOF_KEY: &str = "prf"; -const CID_KEY: &str = "cid"; +const VERSION_KEY: &str = "version"; -/// Receipt for [Invocation], including it's own [Cid]. +/// Receipt for [Invocation], including it's own [Cid] and a [Cid] for an [Instruction]. /// -/// `@See` [LocalReceipt] for more info on the internal fields. +/// `@See` [homestar_core::workflow::Receipt] for more info on some internal +/// fields. /// /// [Invocation]: homestar_core::workflow::Invocation +/// [Instruction]: homestar_core::workflow::Instruction #[derive(Debug, Clone, PartialEq, Queryable, Insertable)] pub struct Receipt { - cid: InvocationPointer, - ran: InvocationPointer, - out: InvocationResult, + cid: Pointer, + ran: Pointer, + instruction: Pointer, + out: InstructionResult, meta: LocalIpld, - iss: Option, + issuer: Option, prf: UcanPrf, + version: String, } impl fmt::Display for Receipt { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "Receipt: [cid: {}, ran: {}, output: {:?}, metadata: {:?}, issuer: {:?}]", - self.cid, self.ran, self.out, self.meta.0, self.iss + "Receipt: [cid: {}, instruction: {}, ran: {}, output: {:?}, metadata: {:?}, issuer: {:?}]", + self.cid, self.instruction, self.ran, self.out, self.meta.0, self.issuer ) } } impl Receipt { /// Generate a receipt. - pub fn new(cid: Cid, local: &LocalReceipt<'_, Ipld>) -> Self { + pub fn new( + cid: Cid, + instruction: Pointer, + invocation_receipt: &InvocationReceipt, + ) -> Self { Self { - ran: local.ran().to_owned(), - out: local.out().to_owned(), - meta: LocalIpld(local.meta().to_owned()), - iss: local.issuer().to_owned(), - prf: local.prf().to_owned(), - cid: InvocationPointer::new(cid), + cid: Pointer::new(cid), + ran: invocation_receipt.ran().to_owned(), + instruction, + out: invocation_receipt.out().to_owned(), + meta: LocalIpld(invocation_receipt.meta().to_owned()), + issuer: invocation_receipt.issuer().to_owned(), + prf: invocation_receipt.prf().to_owned(), + version: consts::INVOCATION_VERSION.to_string(), } } + /// Return a runtime [Receipt] given an [Instruction] [Pointer] and + /// [UCAN Invocation Receipt]. + /// + /// [Instruction]: homestar_core::workflow::Instruction + /// [UCAN Invocation Receipt]: homestar_core::workflow::Receipt + pub fn try_with( + instruction: Pointer, + invocation_receipt: &InvocationReceipt, + ) -> anyhow::Result { + let cid = Cid::try_from(invocation_receipt)?; + Ok(Receipt::new(cid, instruction, invocation_receipt)) + } + + /// Get [Ipld] metadata on a [Receipt]. + pub fn meta(&self) -> &Ipld { + self.meta.inner() + } + + /// Set [Ipld] metadata on a [Receipt]. + pub fn set_meta(&mut self, meta: Ipld) { + self.meta = LocalIpld(meta) + } + /// Get unique identifier of receipt. pub fn cid(&self) -> String { self.cid.to_string() } + /// Get inner [Cid] as bytes. + pub fn cid_as_bytes(&self) -> Vec { + self.cid.cid().to_bytes() + } + + /// Return the [Cid] of the [Receipt]'s associated [Instruction]. + /// + /// [Instruction]: homestar_core::workflow::Instruction + pub fn instruction(&self) -> String { + self.instruction.to_string() + } + + /// Get instruction [Pointer] inner [Cid] as bytes. + pub fn instruction_cid_as_bytes(&self) -> Vec { + self.instruction.cid().to_bytes() + } + /// Get [Cid] in [Receipt] as a [String]. pub fn ran(&self) -> String { self.ran.to_string() } /// Get executed result/value in [Receipt] as [Ipld]. - pub fn output(&self) -> &InvocationResult { + pub fn output(&self) -> &InstructionResult { &self.out } + /// Return [InstructionResult] output as [Arg] for execution. + pub fn output_as_arg(&self) -> InstructionResult { + match self.out.to_owned() { + InstructionResult::Ok(res) => InstructionResult::Ok(res.into()), + InstructionResult::Error(res) => InstructionResult::Error(res.into()), + InstructionResult::Just(res) => InstructionResult::Just(res.into()), + } + } + /// Get executed result/value in [Receipt] as encoded Cbor. pub fn output_encoded(&self) -> anyhow::Result> { let ipld = Ipld::from(self.out.to_owned()); DagCborCodec.encode(&ipld) } + + /// Return semver [Version] of [Receipt]. + pub fn version(&self) -> Result { + Version::parse(&self.version) + } + + /// Return runtime receipt as stringified Json. + pub fn to_json(&self) -> anyhow::Result { + let encoded = DagJsonCodec.encode(&Ipld::from(self.to_owned()))?; + let s = std::str::from_utf8(&encoded) + .map_err(|e| anyhow!("cannot stringify encoded value: {e}"))?; + Ok(s.to_string()) + } } impl TryFrom for Vec { @@ -101,37 +177,51 @@ impl TryFrom> for Receipt { fn try_from(bytes: Vec) -> Result { let ipld: Ipld = DagCborCodec.decode(&bytes)?; - Receipt::try_from(ipld) + ipld.try_into() } } -impl From for LocalReceipt<'_, Ipld> { +impl From for InvocationReceipt { fn from(receipt: Receipt) -> Self { - LocalReceipt::new( + InvocationReceipt::new( receipt.ran, receipt.out, receipt.meta.0, - receipt.iss, + receipt.issuer, receipt.prf, ) } } +impl From<&Receipt> for InvocationReceipt { + fn from(receipt: &Receipt) -> Self { + InvocationReceipt::new( + receipt.ran.clone(), + receipt.out.clone(), + receipt.meta.0.clone(), + receipt.issuer.clone(), + receipt.prf.clone(), + ) + } +} + impl From for Ipld { fn from(receipt: Receipt) -> Self { Ipld::Map(BTreeMap::from([ + (CID_KEY.into(), receipt.cid.into()), (RAN_KEY.into(), receipt.ran.into()), + (INSTRUCTION_KEY.into(), receipt.instruction.into()), (OUT_KEY.into(), receipt.out.into()), (METADATA_KEY.into(), receipt.meta.0), ( ISSUER_KEY.into(), receipt - .iss - .map(|iss| iss.to_string().into()) + .issuer + .map(|issuer| issuer.to_string().into()) .unwrap_or(Ipld::Null), ), (PROOF_KEY.into(), receipt.prf.into()), - (CID_KEY.into(), receipt.cid.into()), + (VERSION_KEY.into(), receipt.version.into()), ])) } } @@ -142,11 +232,22 @@ impl TryFrom for Receipt { fn try_from(ipld: Ipld) -> Result { let map = from_ipld::>(ipld)?; + let cid = from_ipld( + map.get(CID_KEY) + .ok_or_else(|| anyhow!("missing {CID_KEY}"))? + .to_owned(), + )?; + let ran = map .get(RAN_KEY) .ok_or_else(|| anyhow!("missing {RAN_KEY}"))? .try_into()?; + let instruction = map + .get(INSTRUCTION_KEY) + .ok_or_else(|| anyhow!("missing {INSTRUCTION_KEY}"))? + .try_into()?; + let out = map .get(OUT_KEY) .ok_or_else(|| anyhow!("missing {OUT_KEY}"))?; @@ -155,7 +256,7 @@ impl TryFrom for Receipt { .get(METADATA_KEY) .ok_or_else(|| anyhow!("missing {METADATA_KEY}"))?; - let iss = map + let issuer = map .get(ISSUER_KEY) .and_then(|ipld| match ipld { Ipld::Null => None, @@ -168,45 +269,42 @@ impl TryFrom for Receipt { .get(PROOF_KEY) .ok_or_else(|| anyhow!("missing {PROOF_KEY}"))?; - let cid = from_ipld( - map.get(CID_KEY) - .ok_or_else(|| anyhow!("missing {CID_KEY}"))? + let version = from_ipld( + map.get(VERSION_KEY) + .ok_or_else(|| anyhow!("missing {VERSION_KEY}"))? .to_owned(), )?; Ok(Receipt { + cid: Pointer::new(cid), ran, - out: InvocationResult::try_from(out)?, + instruction, + out: InstructionResult::try_from(out)?, meta: LocalIpld(meta.to_owned()), - iss, + issuer, prf: UcanPrf::try_from(prf)?, - cid: InvocationPointer::new(cid), + version, }) } } -impl TryFrom> for Receipt { - type Error = anyhow::Error; +/// Wrapper-type for [Ipld] in order integrate to/from for local storage/db. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, AsExpression, FromSqlRow)] +#[diesel(sql_type = Binary)] +pub struct LocalIpld(Ipld); - fn try_from(receipt: LocalReceipt<'_, Ipld>) -> Result { - TryFrom::try_from(&receipt) +impl LocalIpld { + /// Convert into owned, inner [Ipld]. + pub fn into_inner(self) -> Ipld { + self.0 } -} -impl TryFrom<&LocalReceipt<'_, Ipld>> for Receipt { - type Error = anyhow::Error; - - fn try_from(receipt: &LocalReceipt<'_, Ipld>) -> Result { - let cid = Cid::try_from(receipt)?; - Ok(Receipt::new(cid, receipt)) + /// Convert into referenced, inner [Ipld]. + pub fn inner(&self) -> &Ipld { + &self.0 } } -/// Wrapper-type for [Ipld] in order integrate to/from for local storage/db. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, AsExpression, FromSqlRow)] -#[diesel(sql_type = Binary)] -pub struct LocalIpld(pub Ipld); - impl ToSql for LocalIpld where [u8]: ToSql, @@ -229,49 +327,36 @@ impl FromSql for LocalIpld { #[cfg(test)] mod test { use super::*; - use crate::{db::schema, receipt::receipts, test_utils::db}; - use diesel::prelude::*; - use libipld::{ - multihash::{Code, MultihashDigest}, - Link, + use crate::{ + db::{schema, Database}, + receipt::receipts, + settings::Settings, + test_utils, }; - const RAW: u64 = 0x55; - - fn receipt<'a>() -> (LocalReceipt<'a, Ipld>, Receipt) { - let h = Code::Blake3_256.digest(b"beep boop"); - let cid = Cid::new_v1(RAW, h); - let link: Link = Link::new(cid); - let local = LocalReceipt::new( - InvocationPointer::new_from_link(link), - InvocationResult::Ok(Ipld::Bool(true)), - Ipld::Null, - None, - UcanPrf::default(), - ); - let receipt = Receipt::try_from(&local).unwrap(); - (local, receipt) - } + use diesel::prelude::*; #[test] - fn local_into_receipt() { - let (local, receipt) = receipt(); - assert_eq!(local.ran().to_string(), receipt.ran()); - assert_eq!(local.out(), receipt.output()); - assert_eq!(local.meta(), &receipt.meta.0); - assert_eq!(local.issuer(), &receipt.iss); - assert_eq!(local.prf(), &receipt.prf); + fn invocation_into_receipt() { + let (invocation, receipt) = test_utils::receipt::receipts(); + assert_eq!(invocation.ran().to_string(), receipt.ran()); + assert_eq!(invocation.out(), receipt.output()); + assert_eq!(invocation.meta(), &receipt.meta.0); + assert_eq!(invocation.issuer(), &receipt.issuer); + assert_eq!(invocation.prf(), &receipt.prf); let output_bytes = DagCborCodec - .encode::(&local.out().clone().into()) + .encode::(&invocation.out().clone().into()) .unwrap(); assert_eq!(output_bytes, receipt.output_encoded().unwrap()); } #[test] fn receipt_sql_roundtrip() { - let mut conn = db::setup().unwrap(); - - let (_, receipt) = receipt(); + let mut conn = + test_utils::db::MemoryDb::setup_connection_pool(Settings::load().unwrap().node()) + .conn() + .unwrap(); + let (_, receipt) = test_utils::receipt::receipts(); let rows_inserted = diesel::insert_into(schema::receipts::table) .values(&receipt) @@ -282,12 +367,27 @@ mod test { let inserted_receipt = receipts::table.load::(&mut conn).unwrap(); - assert_eq!(vec![receipt], inserted_receipt); + assert_eq!(vec![receipt.clone()], inserted_receipt); + } + + #[test] + fn receipt_to_json() { + let (_, receipt) = test_utils::receipt::receipts(); + assert_eq!( + receipt.to_json().unwrap(), + format!( + r#"{{"cid":{{"/":"{}"}},"instruction":{{"/":"{}"}},"iss":null,"meta":null,"out":["ok",true],"prf":[],"ran":{{"/":"{}"}},"version":"{}"}}"#, + receipt.cid(), + receipt.instruction(), + receipt.ran(), + consts::INVOCATION_VERSION + ) + ); } #[test] fn receipt_bytes_roundtrip() { - let (_, receipt) = receipt(); + let (_, receipt) = test_utils::receipt::receipts(); let bytes: Vec = receipt.clone().try_into().unwrap(); let from_bytes = Receipt::try_from(bytes).unwrap(); diff --git a/homestar-runtime/src/runtime.rs b/homestar-runtime/src/runtime.rs new file mode 100644 index 00000000..11bdad2d --- /dev/null +++ b/homestar-runtime/src/runtime.rs @@ -0,0 +1,18 @@ +//! General [Runtime] for working across multiple workers +//! and workflows. +//! +//! TODO: Fill this out. + +use homestar_wasm::io::Arg; +use tokio::task::JoinSet; + +/// Runtime for starting workers on workflows. +#[allow(dead_code)] +#[derive(Debug)] +pub struct Runtime { + /// The set of [workers] for [workflows] + /// + /// [workers]: crate::Worker + /// [workflows]: crate::Workflow + pub(crate) workers: JoinSet>, +} diff --git a/homestar-runtime/src/scheduler.rs b/homestar-runtime/src/scheduler.rs new file mode 100644 index 00000000..dc547922 --- /dev/null +++ b/homestar-runtime/src/scheduler.rs @@ -0,0 +1,311 @@ +//! [Scheduler] module for executing a series of tasks for a given +//! [Workflow]. +//! +//! [Scheduler]: TaskScheduler + +use crate::{ + db::{Connection, Database}, + workflow::{Resource, Vertex}, + Db, Workflow, +}; +use anyhow::Result; +use dagga::Node; +use futures::future::BoxFuture; +use homestar_core::workflow::{InstructionResult, LinkMap, Pointer}; +use homestar_wasm::io::Arg; +use indexmap::IndexMap; +use libipld::Cid; +use std::{ops::ControlFlow, str::FromStr}; + +type Schedule<'a> = Vec, usize>>>; + +/// Type for [instruction]-based, batched, execution graph and set of task +/// resources. +/// +/// [instruction]: homestar_core::workflow::Instruction +#[allow(missing_debug_implementations)] +pub struct ExecutionGraph<'a> { + /// A built-up [Dag] [Schedule] of batches. + /// + /// [Dag]: dagga::Dag + pub(crate) schedule: Schedule<'a>, + /// Vector of [resources] to fetch for executing functions in [Workflow]. + /// + /// [resources]: Resource + pub(crate) resources: Vec, +} + +/// Scheduler for a series of tasks, including what's run, +/// what's left to run, and data structures to track resources +/// and what's been executed in memory. +#[allow(dead_code)] +#[derive(Debug, Default)] +pub struct TaskScheduler<'a> { + /// In-memory map of task/instruction results. + pub(crate) linkmap: LinkMap>, + /// [ExecutionGraph] of what's been run so far for a [Workflow] of [Tasks]. + /// + /// [Tasks]: homestar_core::workflow::Task + pub(crate) ran: Option>, + + /// [ExecutionGraph] of what's left to run for a [Workflow] of [Tasks]. + /// + /// [Tasks]: homestar_core::workflow::Task + pub(crate) run: Schedule<'a>, + + /// Step/batch to resume from. + pub(crate) resume_step: Option, + + /// Resources that tasks within a [Workflow] rely on, retrieved + /// through IPFS Client or the DHT directly ahead-of-time. + /// + /// This is transferred from the [ExecutionGraph] for actually executing the + /// schedule. + pub(crate) resources: IndexMap>, +} + +impl<'a> TaskScheduler<'a> { + /// Initialize Task Scheduler, given [Receipt] cache. + /// + /// [Receipt]: crate::Receipt + pub async fn init( + workflow: Workflow<'a, Arg>, + mut conn: Connection, + fetch_fn: F, + ) -> Result> + where + F: FnOnce(Vec) -> BoxFuture<'a, Result>>>, + { + let graph = workflow.graph()?; + let mut schedule = graph.schedule; + let schedule_length = schedule.len(); + let fetched = fetch_fn(graph.resources).await?; + + let resume = schedule + .iter() + .enumerate() + .rev() + .try_for_each(|(idx, vec)| { + let folded_pointers = vec.iter().try_fold(vec![], |mut ptrs, node| { + ptrs.push(Pointer::new(Cid::from_str(node.name())?)); + Ok::<_, anyhow::Error>(ptrs) + }); + + if let Ok(pointers) = folded_pointers { + match Db::find_instructions(pointers, &mut conn) { + Ok(found) => { + let linkmap = found.iter().fold( + LinkMap::>::new(), + |mut map, receipt| { + if let Ok(cid) = receipt.instruction().try_into() { + let _ = map.insert(cid, receipt.output_as_arg()); + } + map + }, + ); + + if found.len() == vec.len() { + ControlFlow::Break((idx + 1, linkmap)) + } else if !found.is_empty() && found.len() < vec.len() { + ControlFlow::Break((idx, linkmap)) + } else { + ControlFlow::Continue(()) + } + } + _ => ControlFlow::Continue(()), + } + } else { + ControlFlow::Continue(()) + } + }); + + match resume { + ControlFlow::Break((idx, linkmap)) => { + let pivot = schedule.split_off(idx); + let step = if idx >= schedule_length || idx == 0 { + None + } else { + Some(idx) + }; + + Ok(Self { + linkmap, + ran: Some(schedule), + run: pivot, + resume_step: step, + resources: fetched, + }) + } + _ => Ok(Self { + linkmap: LinkMap::>::new(), + ran: None, + run: schedule, + resume_step: None, + resources: fetched, + }), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{db::Database, settings::Settings, test_utils, Receipt}; + use futures::FutureExt; + use homestar_core::{ + test_utils::workflow as workflow_test_utils, + workflow::{ + config::Resources, instruction::RunInstruction, prf::UcanPrf, Invocation, + Receipt as InvocationReceipt, Task, + }, + }; + use libipld::Ipld; + + #[tokio::test] + async fn initialize_task_scheduler() { + let config = Resources::default(); + let (instruction1, instruction2, _) = + workflow_test_utils::related_wasm_instructions::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let db = test_utils::db::MemoryDb::setup_connection_pool(Settings::load().unwrap().node()); + let conn = db.conn().unwrap(); + let workflow = Workflow::new(vec![task1.clone(), task2.clone()]); + let fetch_fn = |_rscs: Vec| { async { Ok(IndexMap::default()) } }.boxed(); + let scheduler = TaskScheduler::init(workflow, conn, fetch_fn).await.unwrap(); + + assert!(scheduler.linkmap.is_empty()); + assert!(scheduler.ran.is_none()); + assert_eq!(scheduler.run.len(), 2); + assert_eq!(scheduler.resume_step, None); + } + + #[tokio::test] + async fn initialize_task_scheduler_with_receipted_instruction() { + let config = Resources::default(); + let (instruction1, instruction2, _) = + workflow_test_utils::related_wasm_instructions::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1.clone()), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let invocation_receipt = InvocationReceipt::new( + Invocation::new(task1.clone()).try_into().unwrap(), + InstructionResult::Ok(Ipld::Integer(4)), + Ipld::Null, + None, + UcanPrf::default(), + ); + let receipt = Receipt::try_with( + instruction1.clone().try_into().unwrap(), + &invocation_receipt, + ) + .unwrap(); + + let db = test_utils::db::MemoryDb::setup_connection_pool(Settings::load().unwrap().node()); + let mut conn = db.conn().unwrap(); + + let stored_receipt = + test_utils::db::MemoryDb::store_receipt(receipt.clone(), &mut conn).unwrap(); + + assert_eq!(receipt, stored_receipt); + + let workflow = Workflow::new(vec![task1.clone(), task2.clone()]); + let fetch_fn = |_rscs: Vec| { async { Ok(IndexMap::default()) } }.boxed(); + let scheduler = TaskScheduler::init(workflow, conn, fetch_fn).await.unwrap(); + let ran = scheduler.ran.as_ref().unwrap(); + + assert_eq!(scheduler.linkmap.len(), 1); + assert!(scheduler + .linkmap + .contains_key(&Cid::try_from(instruction1).unwrap())); + assert_eq!(ran.len(), 1); + assert_eq!(scheduler.run.len(), 1); + assert_eq!(scheduler.resume_step, Some(1)); + } + + #[tokio::test] + async fn initialize_task_scheduler_with_all_receipted_instruction() { + let config = Resources::default(); + let (instruction1, instruction2, _) = + workflow_test_utils::related_wasm_instructions::(); + + let task1 = Task::new( + RunInstruction::Expanded(instruction1.clone()), + config.clone().into(), + UcanPrf::default(), + ); + + let task2 = Task::new( + RunInstruction::Expanded(instruction2.clone()), + config.into(), + UcanPrf::default(), + ); + + let invocation_receipt1 = InvocationReceipt::new( + Invocation::new(task1.clone()).try_into().unwrap(), + InstructionResult::Ok(Ipld::Integer(4)), + Ipld::Null, + None, + UcanPrf::default(), + ); + let receipt1 = Receipt::try_with( + instruction1.clone().try_into().unwrap(), + &invocation_receipt1, + ) + .unwrap(); + + let invocation_receipt2 = InvocationReceipt::new( + Invocation::new(task2.clone()).try_into().unwrap(), + InstructionResult::Ok(Ipld::Integer(44)), + Ipld::Null, + None, + UcanPrf::default(), + ); + let receipt2 = Receipt::try_with( + instruction2.clone().try_into().unwrap(), + &invocation_receipt2, + ) + .unwrap(); + + let db = test_utils::db::MemoryDb::setup_connection_pool(Settings::load().unwrap().node()); + let mut conn = db.conn().unwrap(); + + let rows_inserted = + test_utils::db::MemoryDb::store_receipts(vec![receipt1, receipt2], &mut conn).unwrap(); + + assert_eq!(2, rows_inserted); + + let workflow = Workflow::new(vec![task1.clone(), task2.clone()]); + let fetch_fn = |_rscs: Vec| { async { Ok(IndexMap::default()) } }.boxed(); + let scheduler = TaskScheduler::init(workflow, conn, fetch_fn).await.unwrap(); + let ran = scheduler.ran.as_ref().unwrap(); + + assert_eq!(scheduler.linkmap.len(), 1); + assert!(!scheduler + .linkmap + .contains_key(&Cid::try_from(instruction1).unwrap()),); + assert!(scheduler + .linkmap + .contains_key(&Cid::try_from(instruction2).unwrap())); + assert_eq!(ran.len(), 2); + assert!(scheduler.run.is_empty()); + assert_eq!(scheduler.resume_step, None); + } +} diff --git a/homestar-runtime/src/schema.rs b/homestar-runtime/src/schema.rs deleted file mode 100644 index 92648aba..00000000 --- a/homestar-runtime/src/schema.rs +++ /dev/null @@ -1,12 +0,0 @@ -// @generated automatically by Diesel CLI. - -diesel::table! { - receipts (cid) { - cid -> Text, - ran -> Text, - out -> Binary, - meta -> Binary, - iss -> Nullable, - prf -> Binary, - } -} diff --git a/homestar-runtime/src/settings.rs b/homestar-runtime/src/settings.rs new file mode 100644 index 00000000..9cda09b0 --- /dev/null +++ b/homestar-runtime/src/settings.rs @@ -0,0 +1,121 @@ +//! Settings / Configuration. + +use config::{Config, ConfigError, Environment, File}; +use http::Uri; +use serde::Deserialize; +use std::path::PathBuf; + +/// Server settings. +#[derive(Clone, Debug, Deserialize)] +pub struct Node { + #[serde(default)] + pub(crate) network: Network, + #[serde(default)] + pub(crate) db: Database, +} + +/// Process monitoring settings. +#[derive(Clone, Debug, Deserialize)] +pub struct Monitoring { + /// Monitoring collection interval. + #[allow(dead_code)] + process_collector_interval: u64, +} + +#[derive(Debug, Deserialize)] +/// Application settings. +pub struct Settings { + monitoring: Monitoring, + node: Node, +} + +impl Settings { + /// Monitoring settings getter. + pub fn monitoring(&self) -> &Monitoring { + &self.monitoring + } + + /// Node + pub fn node(&self) -> &Node { + &self.node + } +} + +/// Network-related settings for a homestar node. +#[derive(Clone, Debug, Deserialize)] +pub struct Network { + /// + pub(crate) events_buffer_len: usize, + /// Address for [Swarm] to listen on. + /// + /// [Swarm]: libp2p::swarm::Swarm + #[serde(with = "http_serde::uri")] + pub(crate) listen_address: Uri, + /// Pub/sub hearbeat interval for mesh configuration. + pub(crate) pubsub_heartbeat_secs: u64, + /// Quorum for receipt records on the DHT. + pub(crate) receipt_quorum: usize, + /// Websocket-server host address. + #[serde(with = "http_serde::uri")] + pub(crate) websocket_host: Uri, + /// Websocket-server port. + pub(crate) websocket_port: u16, + /// Number of *bounded* clients to send messages to, used for a + /// [tokio::sync::broadcast::channel] + pub(crate) websocket_capacity: usize, +} + +/// Database-related settings for a homestar node. +#[derive(Clone, Debug, Deserialize)] +pub struct Database { + /// Maximum number of connections managed by the [pool]. + /// + /// [pool]: crate::db::Pool + pub(crate) max_pool_size: u32, +} + +impl Default for Network { + fn default() -> Self { + Self { + events_buffer_len: 100, + listen_address: Uri::from_static("/ip4/0.0.0.0/tcp/0"), + pubsub_heartbeat_secs: 10, + receipt_quorum: 2, + websocket_host: Uri::from_static("127.0.0.1"), + websocket_port: 1337, + websocket_capacity: 100, + } + } +} + +impl Default for Database { + fn default() -> Self { + Self { max_pool_size: 100 } + } +} + +impl Settings { + /// Load settings. + pub fn load() -> Result { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("config/settings.toml"); + // inject environment variables naming them properly on the settings + // e.g. [database] url="foo" + // would be injected with environment variable HOMESTAR_DATABASE_URL="foo" + // use one underscore as defined by the separator below + Self::build(path) + } + + /// Load settings from file string that must conform to a [PathBuf]. + pub fn load_from_file(file: String) -> Result { + let path = PathBuf::from(file); + Self::build(path) + } + + fn build(path: PathBuf) -> Result { + let s = Config::builder() + .add_source(File::with_name(&path.as_path().display().to_string())) + .add_source(Environment::with_prefix("HOMESTAR").separator("__")) + .build()?; + s.try_deserialize() + } +} diff --git a/homestar-runtime/src/tasks.rs b/homestar-runtime/src/tasks.rs new file mode 100644 index 00000000..94d7aa9b --- /dev/null +++ b/homestar-runtime/src/tasks.rs @@ -0,0 +1,33 @@ +#![allow(missing_docs)] + +//! Module for working with task-types and task-specific functionality. + +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use enum_assoc::Assoc; +use std::path::PathBuf; + +mod wasm; + +pub(crate) use wasm::*; + +const WASM_OP: &str = "wasm/run"; + +/// First-class registered task-types. +#[allow(missing_debug_implementations)] +#[derive(Assoc)] +#[func(pub fn ability(s: &str) -> Option)] +pub enum RegisteredTasks { + /// Basic `wasm/run` task-type. + #[assoc(ability = WASM_OP)] + WasmRun, +} + +/// Trait for loading files for different task-types directly. +#[async_trait] +pub trait FileLoad { + /// Load file asynchronously. + async fn load(file: PathBuf) -> Result> { + tokio::fs::read(file).await.map_err(|e| anyhow!(e)) + } +} diff --git a/homestar-runtime/src/tasks/wasm.rs b/homestar-runtime/src/tasks/wasm.rs new file mode 100644 index 00000000..939ac691 --- /dev/null +++ b/homestar-runtime/src/tasks/wasm.rs @@ -0,0 +1,57 @@ +use super::FileLoad; +use anyhow::Result; +use async_trait::async_trait; +use homestar_core::workflow::input::Args; +use homestar_wasm::{ + io::{Arg, Output}, + wasmtime::{world::Env, State, World}, +}; + +#[allow(missing_debug_implementations)] +pub(crate) struct WasmContext { + env: Env, +} + +impl WasmContext { + pub(crate) fn new(data: State) -> Result { + Ok(Self { + env: World::default(data)?, + }) + } + + /// Instantiate environment via [World] and execute on [Args]. + pub(crate) async fn run<'a>( + &mut self, + bytes: Vec, + fun_name: &'a str, + args: Args, + ) -> Result { + let env = World::instantiate_with_current_env(bytes, fun_name, &mut self.env).await?; + env.execute(args).await + } +} + +#[async_trait] +impl FileLoad for WasmContext {} + +#[cfg(test)] +mod test { + use super::*; + use std::path::PathBuf; + + fn fixtures(file: &str) -> PathBuf { + PathBuf::from(format!( + "{}/../homestar-wasm/fixtures/{file}", + env!("CARGO_MANIFEST_DIR") + )) + } + + #[tokio::test] + async fn load_wasm_file_as_bytes() { + let wat = WasmContext::load(fixtures("add_one_component.wat")) + .await + .unwrap(); + + assert!(!wat.is_empty()); + } +} diff --git a/homestar-runtime/src/test_utils/db.rs b/homestar-runtime/src/test_utils/db.rs index c35abd56..b5dbdcb5 100644 --- a/homestar-runtime/src/test_utils/db.rs +++ b/homestar-runtime/src/test_utils/db.rs @@ -1,13 +1,33 @@ -use diesel::{connection::SimpleConnection, Connection, SqliteConnection}; - -pub fn setup() -> anyhow::Result { - let mut conn = diesel::sqlite::SqliteConnection::establish(":memory:").unwrap(); - let source = diesel_migrations::FileBasedMigrations::find_migrations_directory()?; - let _ = diesel_migrations::MigrationHarness::run_pending_migrations(&mut conn, source).unwrap(); - begin(&mut conn)?; - Ok(conn) -} +use crate::{ + db::{Connection, Database, Pool}, + settings, +}; +use diesel::r2d2; +use std::sync::Arc; + +/// Sqlite in-memory [Database] [Pool]. +#[derive(Debug)] +pub struct MemoryDb(Arc); + +impl Database for MemoryDb { + fn setup_connection_pool(_settings: &settings::Node) -> Self { + let manager = r2d2::ConnectionManager::::new(":memory:"); + let pool = r2d2::Pool::builder() + .max_size(1) + .build(manager) + .expect("DATABASE_URL must be set to an SQLite DB file"); + + let source = diesel_migrations::FileBasedMigrations::find_migrations_directory().unwrap(); + let _ = diesel_migrations::MigrationHarness::run_pending_migrations( + &mut pool.get().unwrap(), + source, + ) + .unwrap(); + MemoryDb(Arc::new(pool)) + } -pub fn begin(conn: &mut SqliteConnection) -> Result<(), diesel::result::Error> { - conn.batch_execute("BEGIN;") + fn conn(&self) -> anyhow::Result { + let conn = self.0.get()?; + Ok(conn) + } } diff --git a/homestar-runtime/src/test_utils/mod.rs b/homestar-runtime/src/test_utils/mod.rs index b3f5a8d2..d041ac17 100644 --- a/homestar-runtime/src/test_utils/mod.rs +++ b/homestar-runtime/src/test_utils/mod.rs @@ -1,2 +1,4 @@ #[cfg(test)] pub mod db; +#[cfg(test)] +pub mod receipt; diff --git a/homestar-runtime/src/test_utils/receipt.rs b/homestar-runtime/src/test_utils/receipt.rs new file mode 100644 index 00000000..ac1fa9a1 --- /dev/null +++ b/homestar-runtime/src/test_utils/receipt.rs @@ -0,0 +1,41 @@ +//! Test utilities for working with [receipts]. +//! +//! [receipts]: crate::Receipt + +use crate::Receipt; +use homestar_core::{ + test_utils, + workflow::{prf::UcanPrf, receipt::Receipt as InvocationReceipt, InstructionResult, Pointer}, +}; +use libipld::{ + cid::Cid, + multihash::{Code, MultihashDigest}, + Ipld, Link, +}; + +const RAW: u64 = 0x55; + +/// Return both a `mocked` [Ucan Invocation Receipt] and a runtime [Receipt] +/// +/// [UCAN Invocation Receipt]: homestar_core::workflow::Receipt +pub fn receipts() -> (InvocationReceipt, Receipt) { + let h = Code::Blake3_256.digest(b"beep boop"); + let cid = Cid::new_v1(RAW, h); + let link: Link = Link::new(cid); + let local = InvocationReceipt::new( + Pointer::new_from_link(link), + InstructionResult::Ok(Ipld::Bool(true)), + Ipld::Null, + None, + UcanPrf::default(), + ); + let receipt = Receipt::try_with( + test_utils::workflow::instruction::() + .try_into() + .unwrap(), + &local, + ) + .unwrap(); + + (local, receipt) +} diff --git a/homestar-runtime/src/worker.rs b/homestar-runtime/src/worker.rs new file mode 100644 index 00000000..b7aa3da6 --- /dev/null +++ b/homestar-runtime/src/worker.rs @@ -0,0 +1,339 @@ +#[cfg(feature = "ipfs")] +use crate::workflow::settings::BackoffStrategy; +#[cfg(feature = "ipfs")] +use crate::IpfsCli; +use crate::{ + db::{Connection, Database}, + network::eventloop::Event, + scheduler::TaskScheduler, + tasks::{RegisteredTasks, WasmContext}, + workflow::{settings::Settings, Resource, WorkflowInfo}, + Db, Receipt, Workflow, +}; +use anyhow::{anyhow, bail, Result}; +use crossbeam::channel; +use futures::FutureExt; +#[cfg(feature = "ipfs")] +use futures::StreamExt; +use homestar_core::workflow::{ + prf::UcanPrf, InstructionResult, Pointer, Receipt as InvocationReceipt, +}; +use homestar_wasm::{io::Arg, wasmtime::State}; +use indexmap::IndexMap; +use libipld::{Cid, Ipld}; +use std::{ + collections::BTreeMap, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{sync::mpsc, task::JoinSet}; +#[cfg(feature = "ipfs")] +use tryhard::RetryFutureConfig; + +/// Worker that operates over a given [TaskScheduler]. +#[derive(Debug)] +pub struct Worker<'a> { + pub(crate) scheduler: TaskScheduler<'a>, + pub(crate) workflow_info: WorkflowInfo, +} + +impl<'a> Worker<'a> { + /// Instantiate a new [Worker] for a [Workflow]. + #[cfg(not(feature = "ipfs"))] + pub async fn new( + workflow: Workflow<'a, Arg>, + workflow_settings: &'a Settings, + conn: Connection, + ) -> Result> { + let fetch_fn = |rscs: Vec| { + async { Self::get_resources(rscs, workflow_settings).await }.boxed() + }; + + let scheduler = TaskScheduler::init(workflow.clone(), conn, fetch_fn).await?; + let workflow_len = workflow.len(); + let workflow_cid = Cid::try_from(workflow)?; + let workflow_info = WorkflowInfo::new( + workflow_cid, + scheduler.resume_step.map_or(0, |step| step), + workflow_len, + ); + Ok(Self { + scheduler, + workflow_info, + }) + } + + /// Instantiate a new [Worker] for a [Workflow]. + #[cfg(feature = "ipfs")] + pub async fn new( + workflow: Workflow<'a, Arg>, + workflow_settings: &'a Settings, + conn: Connection, + ipfs: &'a IpfsCli, + ) -> Result> { + let fetch_fn = |rscs: Vec| { + async { Self::get_resources(rscs, workflow_settings, ipfs).await }.boxed() + }; + + let scheduler = TaskScheduler::init(workflow.clone(), conn, fetch_fn).await?; + let workflow_len = workflow.len(); + let workflow_cid = Cid::try_from(workflow)?; + let workflow_info = WorkflowInfo::new( + workflow_cid, + scheduler.resume_step.map_or(0, |step| step), + workflow_len, + ); + Ok(Self { + scheduler, + workflow_info, + }) + } + + /// Run [Worker]'s tasks in task-queue with access to the [Db] object + /// to use a connection from the Database pool per run. + pub async fn run( + self, + db: Db, + event_sender: Arc>, + settings: Settings, + ) -> Result<()> { + self.run_queue(db, event_sender, settings).await + } + + #[cfg(feature = "ipfs")] + async fn get_resources( + resources: Vec, + settings: &'a Settings, + ipfs: &'a IpfsCli, + ) -> Result>> { + async fn fetch(rsc: Resource, client: IpfsCli) -> (Resource, Result>) { + match rsc { + Resource::Url(url) => { + let bytes = match (url.scheme(), url.domain(), url.path()) { + ("ipfs", _, _) => client.get_resource(&url).await, + (_, Some("ipfs.io"), _) => client.get_resource(&url).await, + (_, _, path) if path.contains("/ipfs/") || path.contains("/ipns/") => { + client.get_resource(&url).await + } + (_, Some(domain), _) => { + let split: Vec<&str> = domain.splitn(3, '.').collect(); + // subdomain-gateway case: + // + if let (Ok(_cid), "ipfs") = (Cid::try_from(split[0]), split[1]) { + client.get_resource(&url).await + } else { + // TODO: reqwest call + todo!() + } + } + // TODO: reqwest call + (_, _, _) => todo!(), + }; + (Resource::Url(url), bytes) + } + + Resource::Cid(cid) => { + let bytes = client.get_cid(cid).await; + (Resource::Cid(cid), bytes) + } + } + } + let num_requests = resources.len(); + futures::stream::iter(resources.into_iter().map(|rsc| async move { + // Have to enumerate configs here, as type variants are different + // and cannot be matched on. + match settings.retry_backoff_strategy { + BackoffStrategy::Exponential => { + tryhard::retry_fn(|| { + let rsc = rsc.clone(); + let client = ipfs.clone(); + tokio::spawn(async move { fetch(rsc, client).await }) + }) + .with_config( + RetryFutureConfig::new(settings.retries) + .exponential_backoff(Duration::from_millis( + settings.retry_initial_delay_ms, + )) + .max_delay(Duration::from_secs(settings.retry_max_delay_secs)), + ) + .await + } + BackoffStrategy::Fixed => { + tryhard::retry_fn(|| { + let rsc = rsc.clone(); + let client = ipfs.clone(); + tokio::spawn(async move { fetch(rsc, client).await }) + }) + .with_config( + RetryFutureConfig::new(settings.retries) + .fixed_backoff(Duration::from_millis(settings.retry_initial_delay_ms)) + .max_delay(Duration::from_secs(settings.retry_max_delay_secs)), + ) + .await + } + BackoffStrategy::Linear => { + tryhard::retry_fn(|| { + let rsc = rsc.clone(); + let client = ipfs.clone(); + tokio::spawn(async move { fetch(rsc, client).await }) + }) + .with_config( + RetryFutureConfig::new(settings.retries) + .linear_backoff(Duration::from_millis(settings.retry_initial_delay_ms)) + .max_delay(Duration::from_secs(settings.retry_max_delay_secs)), + ) + .await + } + BackoffStrategy::None => { + tryhard::retry_fn(|| { + let rsc = rsc.clone(); + let client = ipfs.clone(); + tokio::spawn(async move { fetch(rsc, client).await }) + }) + .with_config(RetryFutureConfig::new(settings.retries).no_backoff()) + .await + } + } + })) + .buffer_unordered(num_requests) + .collect::>() + .await + .into_iter() + .try_fold(IndexMap::default(), |mut acc, res| { + let inner = res?; + let answer = inner.1?; + acc.insert(inner.0, answer); + Ok::<_, anyhow::Error>(acc) + }) + } + + /// TODO: Client calls (only) over http(s). + #[cfg(not(feature = "ipfs"))] + async fn get_resources( + _resources: Vec, + _settings: &'a Settings, + ) -> Result> { + Ok(IndexMap::default()) + } + + async fn run_queue( + mut self, + db: Db, + event_sender: Arc>, + settings: Settings, + ) -> Result<()> { + for batch in self.scheduler.run.into_iter() { + let (mut set, _handles) = batch.into_iter().try_fold( + (JoinSet::new(), vec![]), + |(mut set, mut handles), node| { + let vertice = node.into_inner(); + let invocation_ptr = vertice.invocation; + let instruction = vertice.instruction; + let rsc = instruction.resource(); + let parsed = vertice.parsed; + let fun = parsed.fun().ok_or_else(|| anyhow!("no function defined"))?; + + let args = parsed.into_args(); + let meta = Ipld::Map(BTreeMap::from([ + ("op".into(), fun.to_string().into()), + ("workflow".into(), self.workflow_info.cid().into()) + ])); + + match RegisteredTasks::ability(&instruction.op().to_string()) { + Some(RegisteredTasks::WasmRun) => { + let wasm = self + .scheduler + .resources + .get(&Resource::Url(rsc.to_owned())) + .ok_or_else(|| anyhow!("resource not available"))? + .to_owned(); + + let instruction_ptr = Pointer::try_from(instruction)?; + let state = State::default(); + let mut wasm_ctx = WasmContext::new(state)?; + let resolved = + args.resolve(|cid| match self.scheduler.linkmap.get(&cid) { + Some(result) => Ok(result.to_owned()), + None => match Db::find_instruction( + Pointer::new(cid), + &mut db.conn()?, + ) { + Ok(found) => Ok(found.output_as_arg()), + Err(_e) => { + tracing::debug!( + "no related instruction receipt found in the DB" + ); + let (sender, receiver) = channel::bounded(1); + event_sender.blocking_send(Event::FindReceipt( + cid, + sender, + ))?; + let found = match receiver.recv_deadline( + Instant::now() + Duration::from_secs(settings.p2p_timeout_secs), + ) { + Ok((found_cid, found)) if found_cid == cid => { + found + } + Ok(_) => bail!("only one worker channel per worker"), + Err(err) => bail!("error returning invocation receipt for {cid}: {err}"), + }; + + Ok(found.output_as_arg()) + } + }, + })?; + + let handle = set.spawn(async move { + match wasm_ctx.run(wasm, &fun, resolved).await { + Ok(output) => { + Ok((output, instruction_ptr, invocation_ptr, meta)) + } + Err(e) => Err(anyhow!("cannot execute wasm module: {e}")), + } + }); + handles.push(handle); + } + None => tracing::error!( + "no valid task/instruction-type referenced by operation: {}", + instruction.op() + ), + }; + + Ok::<_, anyhow::Error>((set, handles)) + }, + )?; + + while let Some(res) = set.join_next().await { + let (executed, instruction_ptr, invocation_ptr, meta) = res??; + let output_to_store = Ipld::try_from(executed)?; + + let invocation_receipt = InvocationReceipt::new( + invocation_ptr, + InstructionResult::Ok(output_to_store), + Ipld::Null, + None, + UcanPrf::default(), + ); + + let mut receipt = Receipt::try_with(instruction_ptr, &invocation_receipt)?; + self.scheduler.linkmap.insert( + Cid::try_from(receipt.instruction())?, + receipt.output_as_arg(), + ); + + receipt.set_meta(meta); + + let stored_receipt = Db::store_receipt(receipt, &mut db.conn()?)?; + + // send internal event + event_sender + .send(Event::CapturedReceipt( + stored_receipt, + self.workflow_info.clone(), + )) + .await?; + } + } + Ok(()) + } +} diff --git a/homestar-runtime/src/workflow.rs b/homestar-runtime/src/workflow.rs new file mode 100644 index 00000000..eebf1100 --- /dev/null +++ b/homestar-runtime/src/workflow.rs @@ -0,0 +1,587 @@ +//! A [Workflow] is a declarative configuration of a series of +//! [UCAN Invocation] `Tasks`. +//! +//! [UCAN Invocation]: + +use crate::scheduler::ExecutionGraph; +use anyhow::{anyhow, bail}; +use dagga::{self, dot::DagLegend, Node}; +use homestar_core::workflow::{ + input::{Parse, Parsed}, + instruction::RunInstruction, + Instruction, Invocation, Pointer, Task, +}; +use homestar_wasm::io::Arg; +use indexmap::IndexMap; +use libipld::{ + cbor::DagCborCodec, + json::DagJsonCodec, + multihash::{Code, MultihashDigest}, + prelude::Codec, + serde::from_ipld, + Cid, Ipld, +}; +use std::{collections::BTreeMap, path::Path}; +use url::Url; + +pub(crate) mod settings; + +type Dag<'a> = dagga::Dag, usize>; + +const DAG_CBOR: u64 = 0x71; +const CID_KEY: &str = "cid"; +const TASKS_KEY: &str = "tasks"; +const PROGRESS_KEY: &str = "progress"; +const NUM_TASKS_KEY: &str = "num_tasks"; + +/// A resource can refer to a [URI] or [Cid] +/// being accessed. +/// +/// [URI]: +#[derive(Debug, Clone, Eq, Hash, PartialEq)] +pub enum Resource { + /// Resource fetched by [Url]. + Url(Url), + /// Resource fetched by [Cid]. + Cid(Cid), +} + +/// Ahead-of-time (AOT) context object, which includes the given +/// [Workflow] as a executable [Dag] (directed acyclic graph) and +/// the [Task] resources retrieved through IPFS Client or the DHT directly +/// ahead-of-time. +/// +/// [Dag]: dagga::Dag +#[derive(Debug, Clone)] +pub struct AOTContext<'a> { + dag: Dag<'a>, + resources: Vec, +} + +impl AOTContext<'static> { + /// Convert [Dag] to a [dot] file, to be read by graphviz, etc. + /// + /// [Dag]: dagga::Dag + /// [dot]: + pub fn dot(&self, name: &str, path: &Path) -> anyhow::Result<()> { + DagLegend::new(self.dag.nodes()) + .with_name(name) + .save_to( + path.to_str() + .ok_or_else(|| anyhow!("path is not correctly formatted"))?, + ) + .map_err(|e| anyhow!(e)) + } +} + +/// Vertex information for [Dag] [Node]. +/// +/// [Dag]: dagga::Dag +#[derive(Debug, Clone, PartialEq)] +pub struct Vertex<'a> { + pub(crate) instruction: Instruction<'a, Arg>, + pub(crate) parsed: Parsed, + pub(crate) invocation: Pointer, +} + +impl<'a> Vertex<'a> { + fn new( + instruction: Instruction<'a, Arg>, + parsed: Parsed, + invocation: Pointer, + ) -> Vertex<'a> { + Vertex { + instruction, + parsed, + invocation, + } + } +} + +/// Associated [Workflow] information, separated from [Workflow] struct in order +/// to relate to it as a key-value relationship of (workflow) +/// cid => [WorkflowInfo]. +/// +/// TODO: map of task cids completed +#[derive(Debug, Clone, PartialEq)] +pub struct WorkflowInfo { + pub(crate) cid: Cid, + pub(crate) progress: usize, + pub(crate) num_tasks: usize, +} + +impl WorkflowInfo { + /// Create a new [WorkflowInfo] given a [Cid], progress / step, and number + /// of tasks. + pub fn new(cid: Cid, progress: usize, num_tasks: usize) -> Self { + Self { + cid, + progress, + num_tasks, + } + } + + /// Create a default [WorkflowInfo] given a [Cid] and number of tasks. + pub fn default(cid: Cid, num_tasks: usize) -> Self { + Self { + cid, + progress: 0, + num_tasks, + } + } + + /// Get the [Cid] of a [Workflow] as a [String]. + pub fn cid(&self) -> String { + self.cid.to_string() + } + + /// Set the progress / step of the [WorkflowInfo]. + pub fn set_progress(&mut self, progress: usize) { + self.progress = progress; + } + + /// Increment the progress / step of the [WorkflowInfo]. + pub fn increment_progress(&mut self) { + self.progress += 1; + } +} + +impl From for Ipld { + fn from(workflow: WorkflowInfo) -> Self { + Ipld::Map(BTreeMap::from([ + (CID_KEY.into(), Ipld::Link(workflow.cid)), + ( + PROGRESS_KEY.into(), + Ipld::Integer(workflow.progress as i128), + ), + ( + NUM_TASKS_KEY.into(), + Ipld::Integer(workflow.num_tasks as i128), + ), + ])) + } +} + +impl TryFrom for WorkflowInfo { + type Error = anyhow::Error; + + fn try_from(ipld: Ipld) -> Result { + let map = from_ipld::>(ipld)?; + let cid = from_ipld( + map.get(CID_KEY) + .ok_or_else(|| anyhow!("no `cid` set"))? + .to_owned(), + )?; + let progress = from_ipld( + map.get(PROGRESS_KEY) + .ok_or_else(|| anyhow!("no `progress` set"))? + .to_owned(), + )?; + let num_tasks = from_ipld( + map.get(NUM_TASKS_KEY) + .ok_or_else(|| anyhow!("no `num_tasks` set"))? + .to_owned(), + )?; + + Ok(Self { + cid, + progress, + num_tasks, + }) + } +} + +impl TryFrom for Vec { + type Error = anyhow::Error; + + fn try_from(receipt: WorkflowInfo) -> Result { + let receipt_ipld = Ipld::from(receipt); + DagCborCodec.encode(&receipt_ipld) + } +} + +impl TryFrom> for WorkflowInfo { + type Error = anyhow::Error; + + fn try_from(bytes: Vec) -> Result { + let ipld: Ipld = DagCborCodec.decode(&bytes)?; + ipld.try_into() + } +} + +/// Workflow composed of [tasks]. +/// +/// [tasks]: Task +#[derive(Debug, Clone, PartialEq)] +pub struct Workflow<'a, T> { + tasks: Vec>, +} + +impl<'a> Workflow<'a, Arg> { + /// Create a new [Workflow] given a set of tasks. + pub fn new(tasks: Vec>) -> Self { + Self { tasks } + } + + /// Length of workflow given a series of [tasks]. + /// + /// [tasks]: Task + pub fn len(&self) -> usize { + self.tasks.len() + } + + /// Whether [Workflow] contains [tasks] or not. + /// + /// [tasks]: Task + pub fn is_empty(&self) -> bool { + self.tasks.is_empty() + } + + /// Convert the [Workflow] into an batch-separated [ExecutionGraph]. + pub fn graph(self) -> anyhow::Result> { + let aot = self.aot()?; + match aot.dag.build_schedule() { + Ok(schedule) => Ok(ExecutionGraph { + schedule: schedule.batches, + resources: aot.resources, + }), + Err(e) => bail!("schedule could not be built from given workflow: {e}"), + } + } + + /// Return workflow as stringified Json. + pub fn to_json(&self) -> anyhow::Result { + let encoded = DagJsonCodec.encode(&Ipld::from(self.to_owned()))?; + let s = std::str::from_utf8(&encoded) + .map_err(|e| anyhow!("cannot stringify encoded value: {e}"))?; + Ok(s.to_string()) + } + + fn aot(self) -> anyhow::Result> { + let lookup_table = self.lookup_table()?; + + let (dag, resources) = + self.tasks.into_iter().enumerate().try_fold( + (Dag::default(), vec![]), + |(mut dag, mut resources), (i, task)| { + // Clone as we're owning the struct going backward. + let ptr: Pointer = Invocation::::from(task.clone()).try_into()?; + let instr_cid = task.instruction_cid()?.to_string(); + + let RunInstruction::Expanded(instr) = task.into_instruction() else { + bail!("workflow tasks/instructions must be expanded / inlined") + }; + + // TODO: check if op is runnable on current node + // TODO LATER: check if op is registered on the network + + resources.push(Resource::Url(instr.resource().to_owned())); + + let parsed = instr.input().parse()?; + let reads = parsed.args().deferreds().into_iter().fold( + vec![], + |mut in_flow_reads, cid| { + if let Some(v) = lookup_table.get(&cid) { + in_flow_reads.push(*v) + } else { + resources.push(Resource::Url(instr.resource().to_owned())); + } + in_flow_reads + }, + ); + + let node = Node::new(Vertex::new(instr.to_owned(), parsed, ptr)) + .with_name(instr_cid) + .with_result(i); + + dag.add_node(node.with_reads(reads)); + Ok::<_, anyhow::Error>((dag, resources)) + }, + )?; + + Ok(AOTContext { dag, resources }) + } + + /// Generate an [IndexMap] lookup table of task instruction CIDs to a + /// unique enumeration. + fn lookup_table(&self) -> anyhow::Result> { + self.tasks + .iter() + .enumerate() + .try_fold(IndexMap::new(), |mut acc, (i, t)| { + acc.insert(t.instruction_cid()?, i); + Ok::<_, anyhow::Error>(acc) + }) + } +} + +impl From> for Ipld { + fn from(workflow: Workflow<'_, Arg>) -> Self { + Ipld::Map(BTreeMap::from([( + TASKS_KEY.into(), + Ipld::List( + workflow + .tasks + .into_iter() + .map(Ipld::from) + .collect::>(), + ), + )])) + } +} + +impl TryFrom for Workflow<'_, Arg> { + type Error = anyhow::Error; + + fn try_from(ipld: Ipld) -> Result { + let map = from_ipld::>(ipld)?; + let ipld = map + .get(TASKS_KEY) + .ok_or_else(|| anyhow!("no `tasks` set"))?; + + let tasks = if let Ipld::List(tasks) = ipld { + let tasks = tasks.iter().fold(vec![], |mut acc, ipld| { + acc.push(ipld.try_into().unwrap()); + acc + }); + tasks + } else { + bail!("unexpected conversion type") + }; + + Ok(Self { tasks }) + } +} + +impl TryFrom> for Cid { + type Error = anyhow::Error; + + fn try_from(workflow: Workflow<'_, Arg>) -> Result { + let ipld: Ipld = workflow.into(); + let bytes = DagCborCodec.encode(&ipld)?; + let hash = Code::Sha3_256.digest(&bytes); + Ok(Cid::new_v1(DAG_CBOR, hash)) + } +} + +#[cfg(test)] +mod test { + use super::*; + use homestar_core::{ + test_utils, + workflow::{config::Resources, instruction::RunInstruction, prf::UcanPrf}, + }; + use std::path::Path; + + #[test] + fn dag_to_dot() { + let config = Resources::default(); + let instruction1 = test_utils::workflow::wasm_instruction::(); + let (instruction2, _) = test_utils::workflow::wasm_instruction_with_nonce::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let workflow = Workflow::new(vec![task1, task2]); + let aot = workflow.aot().unwrap(); + + aot.dot("test", Path::new("test.dot")).unwrap(); + assert!(Path::new("test.dot").exists()); + } + + #[test] + fn build_parallel_schedule() { + let config = Resources::default(); + let instruction1 = test_utils::workflow::wasm_instruction::(); + let (instruction2, _) = test_utils::workflow::wasm_instruction_with_nonce::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let tasks = vec![task1.clone(), task2.clone()]; + + let workflow = Workflow::new(tasks); + let dag = workflow.aot().unwrap().dag; + + let instr1 = task1.instruction_cid().unwrap().to_string(); + let instr2 = task2.instruction_cid().unwrap().to_string(); + + dagga::assert_batches(&[format!("{}, {}", instr2, instr1).as_str()], dag); + } + + #[test] + fn build_seq_schedule() { + let config = Resources::default(); + let (instruction1, instruction2, _) = + test_utils::workflow::related_wasm_instructions::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let workflow = Workflow::new(vec![task1.clone(), task2.clone()]); + let dag = workflow.aot().unwrap().dag; + + let instr1 = task1.instruction_cid().unwrap().to_string(); + let instr2 = task2.instruction_cid().unwrap().to_string(); + + // separate + dagga::assert_batches(&[&instr1, &instr2], dag); + } + + #[test] + fn build_mixed_graph() { + let config = Resources::default(); + let (instruction1, instruction2, instruction3) = + test_utils::workflow::related_wasm_instructions::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.clone().into(), + UcanPrf::default(), + ); + let task3 = Task::new( + RunInstruction::Expanded(instruction3), + config.clone().into(), + UcanPrf::default(), + ); + + let (instruction4, _) = test_utils::workflow::wasm_instruction_with_nonce::(); + let task4 = Task::new( + RunInstruction::Expanded(instruction4), + config.into(), + UcanPrf::default(), + ); + + let tasks = vec![task1.clone(), task2.clone(), task3.clone(), task4.clone()]; + let workflow = Workflow::new(tasks); + + let instr1 = task1.instruction_cid().unwrap().to_string(); + let instr2 = task2.instruction_cid().unwrap().to_string(); + let instr3 = task3.instruction_cid().unwrap().to_string(); + let instr4 = task4.instruction_cid().unwrap().to_string(); + + let schedule = workflow.graph().unwrap().schedule; + let nodes = schedule + .into_iter() + .fold(vec![], |mut acc: Vec, vec| { + if vec.len() == 1 { + acc.push(vec.first().unwrap().name().to_string()) + } else { + let mut tmp = vec![]; + for node in vec { + tmp.push(node.name().to_string()); + } + acc.push(tmp.join(", ")) + } + + acc + }); + + assert!( + nodes + == vec![ + format!("{}, {}", instr1, instr4), + instr2.clone(), + instr3.clone() + ] + || nodes == vec![format!("{}, {}", instr4, instr1), instr2, instr3] + ); + } + + #[test] + fn workflow_to_json() { + let config = Resources::default(); + let (instruction1, instruction2, _) = + test_utils::workflow::related_wasm_instructions::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let workflow = Workflow::new(vec![task1.clone(), task2.clone()]); + + let json_string = workflow.to_json().unwrap(); + + let json_val = json::from(json_string.clone()); + assert_eq!(json_string, json_val.to_string()); + } + + #[test] + fn ipld_roundtrip_workflow() { + let config = Resources::default(); + let (instruction1, instruction2, _) = + test_utils::workflow::related_wasm_instructions::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let workflow = Workflow::new(vec![task1.clone(), task2.clone()]); + let ipld = Ipld::from(workflow.clone()); + + assert_eq!(workflow, ipld.try_into().unwrap()) + } + + #[test] + fn ipld_roundtrip_workflow_info() { + let config = Resources::default(); + let (instruction1, instruction2, _) = + test_utils::workflow::related_wasm_instructions::(); + let task1 = Task::new( + RunInstruction::Expanded(instruction1), + config.clone().into(), + UcanPrf::default(), + ); + let task2 = Task::new( + RunInstruction::Expanded(instruction2), + config.into(), + UcanPrf::default(), + ); + + let workflow = Workflow::new(vec![task1.clone(), task2.clone()]); + let mut workflow_info = + WorkflowInfo::default(Cid::try_from(workflow.clone()).unwrap(), workflow.len()); + let ipld = Ipld::from(workflow_info.clone()); + assert_eq!(workflow_info, ipld.try_into().unwrap()); + workflow_info.increment_progress(); + workflow_info.increment_progress(); + assert_eq!(workflow_info.progress, 2); + } +} diff --git a/homestar-runtime/src/workflow/settings.rs b/homestar-runtime/src/workflow/settings.rs new file mode 100644 index 00000000..d928144d --- /dev/null +++ b/homestar-runtime/src/workflow/settings.rs @@ -0,0 +1,37 @@ +//! Workflow settings for a worker's run/execution. + +/// Workflow settings. +#[derive(Debug, Clone, PartialEq)] +pub struct Settings { + pub(crate) retries: u32, + pub(crate) retry_backoff_strategy: BackoffStrategy, + pub(crate) retry_max_delay_secs: u64, + pub(crate) retry_initial_delay_ms: u64, + pub(crate) p2p_timeout_secs: u64, +} + +impl Default for Settings { + fn default() -> Self { + Self { + retries: 10, + retry_backoff_strategy: BackoffStrategy::Exponential, + retry_max_delay_secs: 60, + retry_initial_delay_ms: 500, + p2p_timeout_secs: 60, + } + } +} + +/// Backoff strategies supported for workflows. +#[allow(dead_code)] +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum BackoffStrategy { + /// Exponential backoff: the delay will double each time. + Exponential, + /// Fixed backoff: the delay wont change between attempts. + Fixed, + /// Linear backoff: the delay will scale linearly with the number of attempts. + Linear, + /// No backoff: forcing just leveraging retries. + None, +} diff --git a/homestar-wasm/Cargo.toml b/homestar-wasm/Cargo.toml index fc99346a..7c747061 100644 --- a/homestar-wasm/Cargo.toml +++ b/homestar-wasm/Cargo.toml @@ -1,12 +1,11 @@ [package] name = "homestar-wasm" version = "0.1.0" -description = "" -keywords = [] -categories = [] - +description = "Homestar Wasm / Wasmtime implementation and IPLD <=> WIT interpreter" +keywords = ["wasm", "wasmtime", "wit", "ipld", "ipvm"] +categories = ["wasm", "execution-engines"] include = ["/src", "README.md", "LICENSE"] -license = "Apache" +license = { workspace = true } readme = "README.md" edition = { workspace = true } rust-version = { workspace = true } @@ -21,7 +20,9 @@ doctest = true crate-type = ["cdylib", "rlib"] [dependencies] -anyhow = "1.0" +# return to version.workspace = true after the following issue is fixed: +# https://github.com/DevinR528/cargo-sort/issues/47 +anyhow = { workspace = true } atomic_refcell = "0.1" enum-as-inner = "0.5" heck = "0.4" @@ -33,19 +34,25 @@ rust_decimal = "1.28" serde_ipld_dagcbor = "0.2" stacker = "0.1" thiserror = "1.0" -tracing = "0.1" -wasi-cap-std-sync = { git = "https://github.com/bytecodealliance/preview2-prototyping" } -wasi-common = { git = "https://github.com/bytecodealliance/preview2-prototyping" } -wasmparser = "0.101" -wasmtime = { git = "https://github.com/bytecodealliance/wasmtime", features = ["async", "component-model", "default"] } -wasmtime-component-util = { git = "https://github.com/bytecodealliance/wasmtime" } +tracing = { workspace = true } +wasi-cap-std-sync = "8.0" +wasi-common = "8.0" +wasmparser = "0.104" +wasmtime = { version = "8.0", features = ["async", "component-model", "default"] } +wasmtime-component-util = "8.0" wat = "1.0" -wit-component = "0.7" +wit-component = "0.8" [dev-dependencies] criterion = "0.4" +image = "0.24" tokio = { version = "1.25", default_features = false } tokio-test = "0.4" [features] default = [] + +[package.metadata.docs.rs] +all-features = true +# defines the configuration attribute `docsrs` +rustdoc-args = ["--cfg", "docsrs"] diff --git a/homestar-wasm/fixtures/homestar_guest_wasm.wasm b/homestar-wasm/fixtures/homestar_guest_wasm.wasm index 0ca7d7e2..b96fd619 100755 Binary files a/homestar-wasm/fixtures/homestar_guest_wasm.wasm and b/homestar-wasm/fixtures/homestar_guest_wasm.wasm differ diff --git a/homestar-wasm/src/io.rs b/homestar-wasm/src/io.rs index 98831728..e6fda776 100644 --- a/homestar-wasm/src/io.rs +++ b/homestar-wasm/src/io.rs @@ -1,3 +1,5 @@ +//! IO (input/output) types for the Wasm execution. + use anyhow::anyhow; use enum_as_inner::EnumAsInner; use homestar_core::workflow::{ @@ -53,12 +55,17 @@ impl input::Parse for Input { fn parse(&self) -> anyhow::Result> { if let Input::Ipld(ref ipld) = self { let map = from_ipld::>(ipld.to_owned())?; + + let func = map + .get("func") + .ok_or_else(|| anyhow!("wrong task input format: {ipld:?}"))?; + let wasm_args = map .get("args") .ok_or_else(|| anyhow!("wrong task input format: {ipld:?}"))?; let args: Args = wasm_args.to_owned().try_into()?; - Ok(Parsed::with(args)) + Ok(Parsed::with_fn(from_ipld::(func.to_owned())?, args)) } else { Err(anyhow!("unexpected task input")) } diff --git a/homestar-wasm/src/lib.rs b/homestar-wasm/src/lib.rs index 86f153f0..d2b14177 100644 --- a/homestar-wasm/src/lib.rs +++ b/homestar-wasm/src/lib.rs @@ -2,14 +2,18 @@ #![warn(missing_debug_implementations, missing_docs, rust_2018_idioms)] #![deny(unreachable_pub, private_in_public)] -//! homestar-wasm is enables a Wasm runtime and execution engine for Homestar. +//! homestar-wasm wraps and extends a [Wasmtime] runtime and acts as the defacto +//! execution engine for Homestar. +//! +//! Related crates/packages: +//! +//! - [homestar-core] +//! - [homestar-runtime] +//! +//! [homestar-core]: homestar_core +//! [homestar-runtime]: +//! [Wasmtime]: -/// pub mod io; -/// Test utilities. pub mod test_utils; -/// All interaction with [wasmtime] runtime, types, and values. pub mod wasmtime; - -/// -pub use homestar_core; diff --git a/homestar-wasm/src/test_utils/mod.rs b/homestar-wasm/src/test_utils/mod.rs index 67ed9115..70feb9ea 100644 --- a/homestar-wasm/src/test_utils/mod.rs +++ b/homestar-wasm/src/test_utils/mod.rs @@ -1,2 +1,3 @@ -/// Test-utilities for Wasm components. +//! Test-utilities for Wasm components. + pub mod component; diff --git a/homestar-wasm/src/wasmtime/config.rs b/homestar-wasm/src/wasmtime/config.rs index 07788997..a8ddbf6c 100644 --- a/homestar-wasm/src/wasmtime/config.rs +++ b/homestar-wasm/src/wasmtime/config.rs @@ -1,3 +1,5 @@ +//! Configuration for Wasm/wasmtime execution. + use crate::wasmtime; use homestar_core::workflow::config::Resources; diff --git a/homestar-wasm/src/wasmtime/ipld.rs b/homestar-wasm/src/wasmtime/ipld.rs index b878dee8..67c5be6c 100644 --- a/homestar-wasm/src/wasmtime/ipld.rs +++ b/homestar-wasm/src/wasmtime/ipld.rs @@ -1,5 +1,9 @@ //! Convert (bidirectionally) between [Ipld] and [wasmtime::component::Val]s //! and [wasmtime::component::Type]s. +//! +//! tl;dr: [Ipld] <=> [wasmtime::component::Val] IR. +//! +//! [Ipld]: libipld::Ipld use anyhow::{anyhow, Result}; use atomic_refcell::{AtomicRef, AtomicRefCell, AtomicRefMut}; @@ -305,6 +309,22 @@ impl RuntimeVal { _ => RuntimeVal::new(Val::Float64(v)), }, Ipld::String(v) => RuntimeVal::new(Val::String(Box::from(v))), + Ipld::Bytes(v) if matches!(interface_ty.inner(), Some(Type::List(_))) => { + let inner = interface_ty + .inner() + .ok_or_else(|| anyhow!("component type mismatch: expected >"))?; + + // already pattern matched against + let list_inst = inner.unwrap_list(); + + let vec = v.into_iter().fold(vec![], |mut acc, elem| { + let RuntimeVal(value, _tags) = RuntimeVal::new(Val::U8(elem)); + acc.push(value); + acc + }); + + RuntimeVal::new(list_inst.new_val(vec.into_boxed_slice())?) + } Ipld::Bytes(v) => RuntimeVal::new(Val::String(Box::from(Base::Base64.encode(v)))), Ipld::Link(v) => match v.version() { cid::Version::V0 => RuntimeVal::new(Val::String(Box::from( @@ -454,15 +474,29 @@ impl TryFrom for Ipld { })?; Ipld::Map(inner) } - RuntimeVal(Val::List(v), _) => { - let inner = v.iter().try_fold(vec![], |mut acc, elem| { - let ipld = Ipld::try_from(RuntimeVal::new(elem.to_owned()))?; - acc.push(ipld); - Ok::<_, Self::Error>(acc) - })?; + RuntimeVal(Val::List(v), _) => match v.first() { + Some(Val::U8(_)) => { + let inner = v.iter().try_fold(vec![], |mut acc, elem| { + if let Val::U8(v) = elem { + acc.push(v.to_owned()); + Ok::<_, Self::Error>(acc) + } else { + Err(anyhow!("expected all u8 types"))? + } + })?; - Ipld::List(inner) - } + Ipld::Bytes(inner) + } + Some(_) => { + let inner = v.iter().try_fold(vec![], |mut acc, elem| { + let ipld = Ipld::try_from(RuntimeVal::new(elem.to_owned()))?; + acc.push(ipld); + Ok::<_, Self::Error>(acc) + })?; + Ipld::List(inner) + } + None => Ipld::List(vec![]), + }, RuntimeVal(Val::Union(u), tags) if !tags.empty() => { let inner = Ipld::try_from(RuntimeVal::new(u.payload().to_owned()))?; diff --git a/homestar-wasm/src/wasmtime/mod.rs b/homestar-wasm/src/wasmtime/mod.rs index 44dfeea8..b7e69b7d 100644 --- a/homestar-wasm/src/wasmtime/mod.rs +++ b/homestar-wasm/src/wasmtime/mod.rs @@ -4,13 +4,8 @@ //! [Wasmtime]: //! [Ipld]: libipld::Ipld -/// pub mod config; -/// [Ipld] <=> [wasmtime::component::Val] IR. -/// -/// [Ipld]: libipld::Ipld pub mod ipld; -/// Wasmtime component initialzation and execution of Wasm function(s). -mod world; +pub mod world; -pub use world::*; +pub use world::{State, World}; diff --git a/homestar-wasm/src/wasmtime/world.rs b/homestar-wasm/src/wasmtime/world.rs index 0ac55e2c..39903972 100644 --- a/homestar-wasm/src/wasmtime/world.rs +++ b/homestar-wasm/src/wasmtime/world.rs @@ -1,7 +1,7 @@ //! [Wasmtime] shim for parsing Wasm components, instantiating //! a module, and executing a Wasm function dynamically. //! -//! [Wasmtime]: https://docs.rs/wasmtime/latest/wasmtime/ +//! [Wasmtime]: use super::ipld::{InterfaceType, RuntimeVal}; use crate::io::{Arg, Output}; @@ -22,7 +22,7 @@ const UNIT_OF_COMPUTE_INSTRUCTIONS: u64 = 100_000; // our error set. /// Incoming `state` from host runtime. -#[derive(Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct State { fuel: u64, } @@ -49,80 +49,93 @@ impl State { /// wasmtime [Instance], [Engine], [Linker], and [Store]. #[allow(missing_debug_implementations)] pub struct Env { - bindings: World, + bindings: Option, engine: Engine, - instance: Instance, + instance: Option, linker: Linker, store: Store, } impl Env { - fn new( - bindings: World, - engine: Engine, - instance: Instance, - linker: Linker, - store: Store, - ) -> Env { - Env { - bindings, + fn new(engine: Engine, linker: Linker, store: Store) -> Env { + Self { + bindings: None, engine, - instance, + instance: None, linker, store, } } fn set_bindings(&mut self, bindings: World) { - self.bindings = bindings; + self.bindings = Some(bindings); } fn set_instance(&mut self, instance: Instance) { - self.instance = instance; + self.instance = Some(instance); } /// Execute Wasm function dynamically given a list ([Args]) of [Ipld] or /// [wasmtime::component::Val] arguments and returning [Output] results. /// Types must conform to [Wit] IDL types when Wasm was compiled/generated. /// - /// [Wit]: https://github.com/WebAssembly/component-model/blob/main/design/mvp/WIT.md + /// [Wit]: /// [Ipld]: libipld::Ipld pub async fn execute(&mut self, args: Args) -> Result where T: Send, { - let param_typs = self.bindings.func().params(&self.store); - let result_typs = self.bindings.func().results(&self.store); + let param_types = self + .bindings + .as_mut() + .ok_or_else(|| anyhow!("bindings not yet instantiated for wasm environment"))? + .func() + .params(&self.store); + let result_types = self + .bindings + .as_mut() + .ok_or_else(|| anyhow!("bindings not yet instantiated for wasm environment"))? + .func() + .results(&self.store); let params: Vec = iter::zip( - param_typs.iter(), + param_types.iter(), args.into_inner().into_iter(), ) .try_fold(vec![], |mut acc, (typ, arg)| { let v = match arg { Input::Ipld(ipld) => RuntimeVal::try_from(ipld, &InterfaceType::from(typ))?.value(), - // TODO: Match within `InvocationResult`(s). - Input::Arg(val) => val.into_inner().into_value().map_err(|e| anyhow!(e))?, + Input::Arg(val) => match val.into_inner() { + Arg::Ipld(ipld) => { + RuntimeVal::try_from(ipld, &InterfaceType::from(typ))?.value() + } + Arg::Value(v) => v, + }, Input::Deferred(await_promise) => bail!(anyhow!( "deferred task not yet resolved for {}: {}", await_promise.result(), - await_promise.task_cid() + await_promise.instruction_cid() )), }; acc.push(v); Ok::<_, anyhow::Error>(acc) })?; - let mut results_alloc: Vec = result_typs + let mut results_alloc: Vec = result_types .iter() .map(|_res| component::Val::Bool(false)) .collect(); self.bindings + .as_mut() + .ok_or_else(|| anyhow!("bindings not yet instantiated for wasm environment"))? .func() .call_async(&mut self.store, ¶ms, &mut results_alloc) .await?; + self.bindings + .as_mut() + .ok_or_else(|| anyhow!("bindings not yet instantiated for wasm environment"))? .func() .post_return_async(&mut self.store) .await?; @@ -137,12 +150,12 @@ impl Env { } /// Return `wasmtime` bindings. - pub fn bindings(&self) -> &World { + pub fn bindings(&self) -> &Option { &self.bindings } /// Return the initialized [wasmtime::component::Instance]. - pub fn instance(&self) -> Instance { + pub fn instance(&self) -> Option { self.instance } @@ -164,17 +177,33 @@ impl Env { pub struct World(Func); impl World { + /// Instantiate a default [environment] given a configuration + /// for a [World], given [State]. + /// + /// [environment]: Env + pub fn default(data: State) -> Result> { + let config = Self::configure(); + let engine = Engine::new(&config)?; + let linker = Self::define_linker(&engine); + + let mut store = Store::new(&engine, data); + store.add_fuel(store.data().fuel)?; + + // Configures a `Store` to yield execution of async WebAssembly code + // periodically and not cause extended polling. + store.out_of_fuel_async_yield(u64::MAX, UNIT_OF_COMPUTE_INSTRUCTIONS); + + let env = Env::new(engine, linker, store); + Ok(env) + } + /// Instantiates the provided `module` using the specified /// parameters, wrapping up the result in a [Env] structure /// that translates between wasm and the host, and gives access /// for future invocations to use the already-initialized linker, store. /// /// Used when first initiating a module of a workflow. - pub async fn instantiate<'a>( - bytes: Vec, - fun_name: String, - data: State, - ) -> Result> { + pub async fn instantiate(bytes: Vec, fun_name: &str, data: State) -> Result> { let config = Self::configure(); let engine = Engine::new(&config)?; let linker = Self::define_linker(&engine); @@ -191,7 +220,9 @@ impl World { let instance = linker.instantiate_async(&mut store, &component).await?; let bindings = Self::new(&mut store, &instance, fun_name)?; - let env = Env::new(bindings, engine, instance, linker, store); + let mut env = Env::new(engine, linker, store); + env.set_bindings(bindings); + env.set_instance(instance); Ok(env) } @@ -201,11 +232,11 @@ impl World { /// the instance for the Wasm component. /// /// [environment]: Env - pub async fn instantiate_with_current_env( + pub async fn instantiate_with_current_env<'a, T>( bytes: Vec, - fun_name: String, - env: &mut Env, - ) -> Result<&mut Env> + fun_name: &'a str, + env: &'a mut Env, + ) -> Result<&'a mut Env> where T: Send, { @@ -232,6 +263,7 @@ impl World { config.wasm_component_model(true); config.async_support(true); config.cranelift_nan_canonicalization(true); + config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); // Most Wasm instructions consume 1 unit of fuel. // Some instructions, such as nop, drop, block, and loop, consume 0 @@ -258,13 +290,13 @@ impl World { fn new( mut store: impl wasmtime::AsContextMut, instance: &Instance, - fun_name: String, + fun_name: &str, ) -> Result { let mut store_ctx = store.as_context_mut(); let mut exports = instance.exports(&mut store_ctx); let mut __exports = exports.root(); let func = __exports - .func(&fun_name) + .func(fun_name) .or_else(|| __exports.func(&fun_name.to_kebab_case())) .or_else(|| __exports.func(&fun_name.to_snake_case())) .ok_or_else(|| anyhow!("function not found"))?; diff --git a/homestar-wasm/tests/execute_wasm.rs b/homestar-wasm/tests/execute_wasm.rs index d52ec5de..03c4d182 100644 --- a/homestar-wasm/tests/execute_wasm.rs +++ b/homestar-wasm/tests/execute_wasm.rs @@ -1,9 +1,9 @@ +use homestar_core::workflow::{ + input::{Args, Parse}, + pointer::{Await, AwaitResult}, + Input, InstructionResult, Pointer, +}; use homestar_wasm::{ - homestar_core::workflow::{ - input::{Args, Parse}, - pointer::{Await, AwaitResult, InvocationPointer}, - Input, InvocationResult, - }, io::{Arg, Output}, wasmtime::{State, World}, }; @@ -22,13 +22,13 @@ fn fixtures(file: &str) -> PathBuf { #[tokio::test] async fn test_execute_wat() { - let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::Integer(1)]), - )]))); + let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ("args".into(), Ipld::List(vec![Ipld::Integer(1)])), + ]))); let wat = fs::read(fixtures("add_one_component.wat")).unwrap(); - let mut env = World::instantiate(wat, "add-one".to_string(), State::default()) + let mut env = World::instantiate(wat, "add-one", State::default()) .await .unwrap(); let res = env @@ -41,19 +41,19 @@ async fn test_execute_wat() { #[tokio::test] async fn test_execute_wat_from_non_component() { let wat = fs::read(fixtures("add_one.wat")).unwrap(); - let env = World::instantiate(wat, "add_one".to_string(), State::default()).await; + let env = World::instantiate(wat, "add_one", State::default()).await; assert!(env.is_err()); } #[tokio::test] async fn test_execute_wasm_underscore() { - let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::Integer(1)]), - )]))); + let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ("args".into(), Ipld::List(vec![Ipld::Integer(1)])), + ]))); let wasm = fs::read(fixtures("add_one.wasm")).unwrap(); - let mut env = World::instantiate(wasm, "add_one".to_string(), State::default()) + let mut env = World::instantiate(wasm, "add_one", State::default()) .await .unwrap(); let res = env @@ -65,13 +65,13 @@ async fn test_execute_wasm_underscore() { #[tokio::test] async fn test_execute_wasm_hyphen() { - let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::Integer(10)]), - )]))); + let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ("args".into(), Ipld::List(vec![Ipld::Integer(10)])), + ]))); let wasm = fs::read(fixtures("add_one.wasm")).unwrap(); - let mut env = World::instantiate(wasm, "add-one".to_string(), State::default()) + let mut env = World::instantiate(wasm, "add-one", State::default()) .await .unwrap(); let res = env @@ -84,19 +84,22 @@ async fn test_execute_wasm_hyphen() { #[tokio::test] async fn test_wasm_wrong_fun() { let wasm = fs::read(fixtures("add_one.wasm")).unwrap(); - let env = World::instantiate(wasm, "add-onez".to_string(), State::default()).await; + let env = World::instantiate(wasm, "add-onez", State::default()).await; assert!(env.is_err()); } #[tokio::test] async fn test_append_string() { - let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::String("Natural Science".to_string())]), - )]))); + let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("append-string".to_string())), + ( + "args".into(), + Ipld::List(vec![Ipld::String("Natural Science".to_string())]), + ), + ]))); let wasm = fs::read(fixtures("homestar_guest_wasm.wasm")).unwrap(); - let mut env = World::instantiate(wasm, "append-string".to_string(), State::default()) + let mut env = World::instantiate(wasm, "append-string", State::default()) .await .unwrap(); @@ -120,13 +123,13 @@ async fn test_matrix_transpose() { Ipld::List(vec![Ipld::Integer(4), Ipld::Integer(5), Ipld::Integer(6)]), Ipld::List(vec![Ipld::Integer(7), Ipld::Integer(8), Ipld::Integer(9)]), ]); - let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![ipld_inner.clone()]), - )]))); + let ipld = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("transpose".to_string())), + ("args".into(), Ipld::List(vec![ipld_inner.clone()])), + ]))); let wasm = fs::read(fixtures("homestar_guest_wasm.wasm")).unwrap(); - let mut env = World::instantiate(wasm, "transpose".to_string(), State::default()) + let mut env = World::instantiate(wasm, "transpose", State::default()) .await .unwrap(); @@ -139,10 +142,10 @@ async fn test_matrix_transpose() { assert_ne!(transposed_ipld, ipld_inner); - let ipld_transposed_map = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![transposed_ipld]), - )]))); + let ipld_transposed_map = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("transpose".to_string())), + ("args".into(), Ipld::List(vec![transposed_ipld])), + ]))); let retransposed = env .execute(ipld_transposed_map.parse().unwrap().try_into().unwrap()) @@ -156,20 +159,23 @@ async fn test_matrix_transpose() { #[tokio::test] async fn test_execute_wasms_in_seq() { - let ipld_int = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::Integer(1)]), - )]))); - - let ipld_str = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::String("Natural Science".to_string())]), - )]))); + let ipld_int = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("add_one".to_string())), + ("args".into(), Ipld::List(vec![Ipld::Integer(1)])), + ]))); + + let ipld_str = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("append_string".to_string())), + ( + "args".into(), + Ipld::List(vec![Ipld::String("Natural Science".to_string())]), + ), + ]))); let wasm1 = fs::read(fixtures("add_one.wasm")).unwrap(); let wasm2 = fs::read(fixtures("homestar_guest_wasm.wasm")).unwrap(); - let mut env = World::instantiate(wasm1, "add_one".to_string(), State::default()) + let mut env = World::instantiate(wasm1, "add_one", State::default()) .await .unwrap(); @@ -180,7 +186,7 @@ async fn test_execute_wasms_in_seq() { assert_eq!(res, Output::Value(wasmtime::component::Val::S32(2))); - let env2 = World::instantiate_with_current_env(wasm2, "append_string".to_string(), &mut env) + let env2 = World::instantiate_with_current_env(wasm2, "append_string", &mut env) .await .unwrap(); @@ -197,28 +203,70 @@ async fn test_execute_wasms_in_seq() { ); } +#[tokio::test] +async fn test_multiple_args() { + let wasm = fs::read(fixtures("homestar_guest_wasm.wasm")).unwrap(); + + let ipld_str = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("join-strings".to_string())), + ( + "args".into(), + Ipld::List(vec![ + Ipld::String("Round".to_string()), + Ipld::String("about".to_string()), + ]), + ), + ]))); + + let mut env = World::instantiate(wasm, "join-strings", State::default()) + .await + .unwrap(); + + let res = env + .execute(ipld_str.parse().unwrap().try_into().unwrap()) + .await + .unwrap(); + + assert_eq!( + res, + Output::Value(wasmtime::component::Val::String("Roundabout".into())) + ); +} + #[tokio::test] async fn test_execute_wasms_in_seq_with_threaded_result() { - let ipld_step_1 = Input::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::Integer(1)]), - )]))); + let ipld_step_1 = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("join-strings".to_string())), + ( + "args".into(), + Ipld::List(vec![ + Ipld::String("Round".to_string()), + Ipld::String("about".to_string()), + ]), + ), + ]))); let h = Code::Blake3_256.digest(b"beep boop"); let cid = Cid::new_v1(0x55, h); let link: Link = Link::new(cid); - let invoked_task = InvocationPointer::new_from_link(link); + let invoked_instr = Pointer::new_from_link(link); - let promise = Await::new(invoked_task, AwaitResult::Ok); + let promise = Await::new(invoked_instr, AwaitResult::Ok); - let ipld_step_2 = Input::::Ipld(Ipld::Map(BTreeMap::from([( - "args".into(), - Ipld::List(vec![Ipld::try_from(promise).unwrap()]), - )]))); + let ipld_step_2 = Input::::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("join-strings".to_string())), + ( + "args".into(), + Ipld::List(vec![ + Ipld::try_from(promise).unwrap(), + Ipld::String("about".to_string()), + ]), + ), + ]))); - let wasm1 = fs::read(fixtures("add_one.wasm")).unwrap(); + let wasm1 = fs::read(fixtures("homestar_guest_wasm.wasm")).unwrap(); - let mut env = World::instantiate(wasm1.clone(), "add_one".to_string(), State::default()) + let mut env = World::instantiate(wasm1.clone(), "join-strings", State::default()) .await .unwrap(); @@ -227,9 +275,12 @@ async fn test_execute_wasms_in_seq_with_threaded_result() { .await .unwrap(); - assert_eq!(res, Output::Value(wasmtime::component::Val::S32(2))); + assert_eq!( + res, + Output::Value(wasmtime::component::Val::String("Roundabout".into())) + ); - let env2 = World::instantiate_with_current_env(wasm1, "add-one".to_string(), &mut env) + let env2 = World::instantiate_with_current_env(wasm1, "join-strings", &mut env) .await .unwrap(); @@ -238,12 +289,84 @@ async fn test_execute_wasms_in_seq_with_threaded_result() { // Short-circuit resolve with known value. let resolved = parsed .resolve(|_| { - Ok(InvocationResult::Ok(Arg::Value( - wasmtime::component::Val::S32(2), + Ok(InstructionResult::Ok(Arg::Value( + wasmtime::component::Val::String("RoundRound".into()), ))) }) .unwrap(); let res2 = env2.execute(resolved).await.unwrap(); - assert_eq!(res2, Output::Value(wasmtime::component::Val::S32(3))); + assert_eq!( + res2, + Output::Value(wasmtime::component::Val::String("RoundRoundabout".into())) + ); +} + +#[tokio::test] +async fn test_execute_wasms_with_multiple_inits() { + let ipld_step_1 = Input::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("join-strings".to_string())), + ( + "args".into(), + Ipld::List(vec![ + Ipld::String("Round".to_string()), + Ipld::String("about".to_string()), + ]), + ), + ]))); + + let h = Code::Blake3_256.digest(b"beep boop"); + let cid = Cid::new_v1(0x55, h); + let link: Link = Link::new(cid); + let invoked_instr = Pointer::new_from_link(link); + + let promise = Await::new(invoked_instr, AwaitResult::Ok); + + let ipld_step_2 = Input::::Ipld(Ipld::Map(BTreeMap::from([ + ("func".into(), Ipld::String("join-strings".to_string())), + ( + "args".into(), + Ipld::List(vec![ + Ipld::try_from(promise).unwrap(), + Ipld::String("about".to_string()), + ]), + ), + ]))); + + let wasm1 = fs::read(fixtures("homestar_guest_wasm.wasm")).unwrap(); + + let mut env = World::instantiate(wasm1.clone(), "join-strings", State::default()) + .await + .unwrap(); + + let res = env + .execute(ipld_step_1.parse().unwrap().try_into().unwrap()) + .await + .unwrap(); + + assert_eq!( + res, + Output::Value(wasmtime::component::Val::String("Roundabout".into())) + ); + + let mut env2 = World::instantiate(wasm1, "join-strings", State::default()) + .await + .unwrap(); + + let parsed: Args = ipld_step_2.parse().unwrap().try_into().unwrap(); + + // Short-circuit resolve with known value. + let resolved = parsed + .resolve(|_| { + Ok(InstructionResult::Ok(Arg::Ipld(Ipld::String( + "RoundRound".into(), + )))) + }) + .unwrap(); + + let res2 = env2.execute(resolved).await.unwrap(); + assert_eq!( + res2, + Output::Value(wasmtime::component::Val::String("RoundRoundabout".into())) + ); } diff --git a/migrations/2022-12-11-183928_create_receipts/down.sql b/migrations/2022-12-11-183928_create_receipts/down.sql deleted file mode 100644 index 92d4139a..00000000 --- a/migrations/2022-12-11-183928_create_receipts/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE receipts; -DROP INDEX ran_index; diff --git a/migrations/2022-12-11-183928_create_receipts/up.sql b/migrations/2022-12-11-183928_create_receipts/up.sql deleted file mode 100644 index b5aa656d..00000000 --- a/migrations/2022-12-11-183928_create_receipts/up.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TABLE receipts ( - cid TEXT NOT NULL PRIMARY KEY, - ran TEXT NOT NULL, - out BLOB NOT NULL, - meta BLOB NOT NULL, - iss TEXT, - prf BLOB NOT NULL -); - -CREATE INDEX ran_index ON receipts (ran); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 292fe499..5d56faf9 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "stable" +channel = "nightly"