refactor: worker tests, cargo nextest, and better handle on workflow …

…progress (#137)

Includes:

* foregin_key relationships and join table for many to many workflow(s)
to receipts(s)
* make `progress` defined by receipts (as Cids) completed
* handling of db, fallback to libp2p for workflow information and
progress
* tests, tests, tests

.github/workflows/coverage.yml

@@ -41,6 +41,7 @@ jobs:
           LLVM_PROFILE_FILE: "homestar-%p-%m.profraw"
           RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
           RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+        # covering nexttest's doc-test issue
         run: cargo test --all-features
 
       - name: Install grcov

.github/workflows/tests_and_checks.yml

@@ -110,11 +110,14 @@ jobs:
           override: true
           toolchain: ${{ matrix.rust-toolchain }}
 
-      - name: Run Tests
-        run: cargo test --all-features
+      - name: Install nextest
+        uses: taiki-e/install-action@nextest
 
       - name: Run Tests
-        run: cargo test --no-default-features
+        run: cargo nextest run --all-features
+
+      - name: Run Doc Tests
+        run: cargo test --doc
 
   run-tests-no-default-features:
     runs-on: ubuntu-latest
@@ -147,5 +150,8 @@ jobs:
           override: true
           toolchain: ${{ matrix.rust-toolchain }}
 
+      - name: Install nextest
+        uses: taiki-e/install-action@nextest
+
       - name: Run Tests
-        run: cargo test --no-default-features
+        run: cargo nextest run --no-default-features

Cargo.toml

@@ -27,6 +27,10 @@ tracing = "0.1"
 debug-assertions = true
 split-debuginfo = "unpacked"
 
+[profile.release]
+# compiling with link time optimizations (LTO)
+lto = true
+
 [profile.release.package.homestar-core]
 # Tell `rustc` to optimize for small code size.
 opt-level = "s"
@@ -46,7 +50,7 @@ debug-assertions = false
 # Perform optimizations on all codegen units.
 codegen-units = 1
 # Tell `rustc` to optimize for small code size.
-opt-level = "z" # or 'z' to optimize "aggressively" for size
+opt-level = "z" # 'z' to optimize "aggressively" for size
 # Strip debug symbols
 # "symbols" issue: https://github.com/rust-lang/rust/issues/93294
 strip = "symbols"

README.md

@@ -39,19 +39,79 @@
 
 ## Outline
 
+- [Workspace](#workspace)
 - [Testing the Project](#testing-the-project)
+- [Running the Runtime on Docker](#running-the-runtime-on-docker)
 - [Contributing](#contributing)
 - [Getting Help](#getting-help)
 - [External Resources](#external-resources)
 - [License](#license)
 
+## Workspace
+
+This repository is comprised of a few library packages and a binary that
+represents the `homestar` runtime.
+
+### Core Crates
+
+- [homestar-core](./homestar-core)
+- [homestar-wasm](./homestar-wasm)
+
+### Runtime Crate
+
+- [homestar-runtime](./homestar-runtime)
+
+### Non-published, Helper Crates
+
+- [homestar-guest-wasm](./homestar-guest-wasm)
 
 ## Testing the Project
 
-- Run tests
+- Running the tests:
+
+We recommend using [cargo nextest][cargo-nextest], which is installed via
+[our Nix flake](#nix) or can be [installed separately][cargo-nextest-install].
+
+  ```console
+  cargo nextest run --all-features --no-capture
+  ```
+
+Otherwise, the above command looks like this using the default `cargo test`:
 
   ```console
-  cargo test --all-features
+  cargo test --all-features -- --nocapture
+  ```
+
+## Running the Runtime on Docker
+
+We recommend setting your [Docker Engine][docker-engine] configuration
+with `experimental` and `buildkit` set to `true`, for example:
+
+``` json
+{
+  "builder": {
+    "gc": {
+      "defaultKeepStorage": "20GB",
+      "enabled": true
+    }
+  },
+  "experimental": true,
+  "features": {
+    "buildkit": true
+  }
+}
+```
+
+- Build a multi-plaform Docker image via [buildx][buildx]:
+
+  ```console
+  docker buildx build --platform=linux/amd64,linux/arm64 -t homestar-runtime --progress=plain .
+  ```
+
+- Run a Docker image (depending on your platform):
+
+  ```console
+  docker run --platform=linux/arm64 -t homestar-runtime
   ```
 
 ## Contributing
@@ -89,7 +149,7 @@ hooks. Please run this before every commit and/or push.
 
 - We recommend leveraging [cargo-watch][cargo-watch],
   [cargo-expand][cargo-expand] and [irust][irust] for Rust development.
-- We recommend using [cargo-udeps][cargo-udeps] for removing unused dependencies
+- We also recommend using [cargo-udeps][cargo-udeps] for removing unused dependencies
   before commits and pull-requests.
 
 ### Conventional Commits
@@ -138,10 +198,11 @@ submitted for inclusion in the work by you, as defined in the Apache-2.0
 license, shall be dual licensed as above, without any additional terms or
 conditions.
 
-
 [apache]: https://www.apache.org/licenses/LICENSE-2.0
 [blog-1]: https://fission.codes/blog/ipfs-thing-breaking-down-ipvm/
 [cargo-expand]: https://github.com/dtolnay/cargo-expand
+[cargo-nextest]: https://nexte.st/index.html
+[cargo-nextest-install]: https://nexte.st/book/installation.html
 [cargo-udeps]: https://github.com/est31/cargo-udeps
 [cargo-watch]: https://github.com/watchexec/cargo-watch
 [cod-ipvm]: https://www.youtube.com/watch?v=3y1RB8wt_YY

flake.nix

@@ -38,10 +38,12 @@
         cargo-installs = with pkgs; [
           cargo-deny
           cargo-expand
+          cargo-nextest
           cargo-outdated
           cargo-sort
           cargo-udeps
           cargo-watch
+          twiggy
         ];
       in rec
       {

homestar-core/src/test_utils/workflow.rs

@@ -42,7 +42,7 @@ where
     Ipld: From<T>,
     T: Clone,
 {
-    let wasm = "bafkreidztuwoszw2dfnzufjpsjmzj67x574qcdm2autnhnv43o3t4zmh7i".to_string();
+    let wasm = "bafybeihzvrlcfqf6ffbp2juhuakspxj2bdsc54cabxnuxfvuqy5lvfxapy".to_string();
     let resource = Url::parse(format!("ipfs://{wasm}").as_str()).unwrap();
 
     let instr = Instruction::new(

homestar-core/src/workflow/pointer.rs

@@ -159,7 +159,7 @@ impl TryFrom<&Ipld> for Await {
 /// [Task]: super::Task
 /// [Instruction]: super::Instruction
 /// [Receipt]: super::Receipt
-#[derive(Clone, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Serialize, Deserialize, Hash)]
 #[diesel(sql_type = Text)]
 pub struct Pointer(Cid);
 
@@ -228,7 +228,7 @@ impl<'a> From<&'a Pointer> for Cow<'a, Pointer> {
 
 impl ToSql<Text, Sqlite> for Pointer {
     fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Sqlite>) -> serialize::Result {
-        out.set_value(self.cid().to_string_of_base(Base::Base32Lower)?);
+        out.set_value(self.cid().to_string());
         Ok(IsNull::No)
     }
 }

homestar-guest-wasm/Cargo.toml

@@ -16,3 +16,7 @@ image = "0.24"
 doc = false
 bench = false
 crate-type = ["cdylib", "rlib"]
+
+[features]
+default = []
+run_image_tests = []

homestar-guest-wasm/src/lib.rs

@@ -86,6 +86,7 @@ export_homestar!(Component);
 #[cfg(test)]
 mod test {
     use super::*;
+    #[cfg(feature = "run_image_tests")]
     use std::path::Path;
 
     #[test]
@@ -110,6 +111,7 @@ mod test {
     }
 
     #[test]
+    #[cfg(feature = "run_image_tests")]
     fn blur() {
         let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap();
         let mut buffer: Vec<u8> = Vec::new();
@@ -131,6 +133,7 @@ mod test {
     }
 
     #[test]
+    #[cfg(feature = "run_image_tests")]
     fn crop() {
         let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap();
         let mut buffer: Vec<u8> = Vec::new();
@@ -152,6 +155,7 @@ mod test {
     }
 
     #[test]
+    #[cfg(feature = "run_image_tests")]
     fn grayscale() {
         let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap();
         let mut buffer: Vec<u8> = Vec::new();
@@ -173,6 +177,7 @@ mod test {
     }
 
     #[test]
+    #[cfg(feature = "run_image_tests")]
     fn rotate() {
         let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap();
         let mut buffer: Vec<u8> = Vec::new();
@@ -194,6 +199,7 @@ mod test {
     }
 
     #[test]
+    #[cfg(feature = "run_image_tests")]
     fn mixed() {
         let img = image::open(Path::new("./fixtures/synthcat.jpg")).unwrap();
         let mut buffer: Vec<u8> = Vec::new();

homestar-runtime/migrations/2023-06-04-135955_create_workflows/down.sql

@@ -0,0 +1 @@
+DROP TABLE workflows;

homestar-runtime/migrations/2023-06-04-135955_create_workflows/up.sql

@@ -0,0 +1,4 @@
+CREATE TABLE workflows (
+  cid         TEXT NOT NULL PRIMARY KEY,
+  num_tasks   INTEGER NOT NULL
+);

homestar-runtime/migrations/2023-06-04-183928_create_receipts/up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE receipts (
+  cid          TEXT NOT NULL PRIMARY KEY,
+  ran          TEXT NOT NULL,
+  instruction  TEXT NOT NULL,
+  out          BLOB NOT NULL,
+  meta         BLOB NOT NULL,
+  issuer       TEXT,
+  prf          BLOB NOT NULL,
+  version      TEXT NOT NULL
+);
+
+CREATE INDEX instruction_index ON receipts (instruction);

homestar-runtime/migrations/2023-06-07-023422_create_workflows_receipts/down.sql

@@ -0,0 +1 @@
+DROP TABLE workflows_receipts;

homestar-runtime/migrations/2023-06-07-023422_create_workflows_receipts/up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE workflows_receipts (
+  workflow_cid TEXT NOT NULL REFERENCES workflows(cid),
+  receipt_cid TEXT NOT NULL REFERENCES receipts(cid),
+  PRIMARY KEY(workflow_cid, receipt_cid)
+);