Add 262 JSON tests (#662)

* Add the `javy-test-macros` crate

This commit introduces the `javy-test-macros` crate to the existing
family of crates.

This crate is intended to be a dev-dependency crate used to test javy
through the embedding API.

As of this commit, this crate exposes the bare minimum functionality to
test the implementation of JSON.{parse,stringify} introduced in
#651.

* Test the JSON.{parse, stringify} implementation

This commit adds `javy-test-macros` as a development dependency in the
`javy` crate and adds 262 test for Javy's custom implementation of
`JSON.{parse,stringigy}`.

The 262 suite is pulled in as a submodule.

In order to test the custom implementation, this commit explicitly calls
`config.override_json_parse_and_stringify` when creating the default
runtime used in Javy. The override configuration was introduced in
#651.

* chore: Checkout submodules in the CI workflow

To pull-in 262 test suite.

* chore: Remove `--test-threads=1`

This was added for development purposes.

* chore: Clippy fixes

* chore: Define wasi runner in ci

* Gate 262 tests under the json feature

* Vet dependencies

* chore: Remove stale TODO

* Remove commented out test

There's nothing in the spec stating that the key must be a string.

* Fix Cargo.toml

* Address review comments

* Fix unbalanced double quote

.github/workflows/ci.yml

@@ -17,6 +17,8 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
 
       - uses: ./.github/actions/ci-shared-setup
         with:
@@ -32,6 +34,8 @@ jobs:
         run: cargo build -p javy-core --release --target=wasm32-wasi
 
       - name: Test
+        env: 
+          CARGO_TARGET_WASM32_WASI_RUNNER: wasmtime --dir=.
         run: cargo hack wasi test --workspace --exclude=javy-cli --each-feature -- --nocapture
 
       - name: Lint

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "wpt/upstream"]
 	path = wpt/upstream
 	url = https://github.com/web-platform-tests/wpt
+[submodule "crates/javy/test262"]
+	path = crates/javy/test262
+	url = [email protected]:tc39/test262.git

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "crates/apis",
   "crates/core",
   "crates/cli",
+  "crates/javy-test-macros",
 ]
 resolver = "2"
 

Makefile

@@ -22,7 +22,7 @@ docs:
 	cargo doc --package=javy-core --open --target=wasm32-wasi
 
 test-javy:
-	cargo wasi test --package=javy --features json,messagepack -- --nocapture
+	CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime --dir=."  cargo wasi test --package=javy --features json,messagepack -- --nocapture
 
 test-core:
 	cargo wasi test --package=javy-core -- --nocapture

crates/core/src/runtime.rs

@@ -7,6 +7,7 @@ pub(crate) fn new_runtime() -> Result<Runtime> {
         .text_encoding(true)
         .redirect_stdout_to_stderr(true)
         .javy_stream_io(true)
+        .override_json_parse_and_stringify(true)
         .javy_json(true);
 
     Runtime::new(std::mem::take(config))

crates/javy-test-macros/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "javy-test-macros"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[lib]
+proc-macro = true
+doctest = false
+
+[dependencies]
+anyhow = { workspace = true }
+proc-macro2 = "1.0.85"
+quote = "1.0.36"
+syn = { version = "2.0.66", features = ["full"] }

crates/javy-test-macros/README.md

@@ -0,0 +1,3 @@
+# Macro helpers for testing Javy
+
+See `src/lib.rs` for more details and usage.

crates/javy-test-macros/src/lib.rs

@@ -0,0 +1,174 @@
+use anyhow::bail;
+/// Macros for testing Javy.
+///
+/// Helper macros to define Test262 tests or tests that exercise different
+/// configuration combinations.
+///
+/// Currently only defining Test262 tests for JSON is supported.
+///
+/// Usage
+///
+/// ```rust
+/// t262!(path = "path/to/262/directory")
+/// ```
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use std::path::{Path, PathBuf};
+use syn::{parse_macro_input, Ident, LitStr, Result};
+
+struct Config262 {
+    root: PathBuf,
+}
+
+impl Config262 {
+    fn validate(&self) -> anyhow::Result<()> {
+        let path: Box<Path> = self.root.clone().into();
+        if path.is_dir() {
+            Ok(())
+        } else {
+            bail!("Invalid path")
+        }
+    }
+}
+
+impl Default for Config262 {
+    fn default() -> Self {
+        Self {
+            root: PathBuf::new(),
+        }
+    }
+}
+
+#[proc_macro]
+pub fn t262(stream: TokenStream) -> TokenStream {
+    let mut config = Config262::default();
+
+    let config_parser = syn::meta::parser(|meta| {
+        if meta.path.is_ident("path") {
+            let lit: Option<LitStr> = Some(meta.value()?.parse()?);
+
+            if let Some(s) = lit {
+                config.root = PathBuf::from(s.clone().value());
+            } else {
+                return Err(meta.error("Expected String literal"));
+            }
+            config.validate().map_err(|e| meta.error(e))
+        } else {
+            Err(meta.error("Unsupported property"))
+        }
+    });
+
+    parse_macro_input!(stream with config_parser);
+
+    match expand(&config) {
+        Ok(tok) => tok,
+        Err(e) => e.into_compile_error().into(),
+    }
+}
+
+// Should this test be ignored?
+fn ignore(test_name: &str) -> bool {
+    [
+        // A bit unfortunate, currently simd-json returns `0` for `'-0'`;
+        // I think this is a bug in simd-json itself.
+        "test_parse_text_negative_zero",
+        // Realms are not supported by QuickJS
+        "test_stringify_replacer_array_proxy_revoked_realm",
+        "test_stringify_value_bigint_cross_realm",
+        // TODO
+        // Currently the conversion between non-utf8 string encodings is lossy.
+        // There's probably a way to improve the interop.
+        "test_stringify_value_string_escape_unicode",
+    ]
+    .contains(&test_name)
+}
+
+fn expand(config: &Config262) -> Result<TokenStream> {
+    let harness = config.root.join("harness");
+    let harness_str = harness.into_os_string().into_string().unwrap();
+    let json_parse = config
+        .root
+        .join("test")
+        .join("built-ins")
+        .join("JSON")
+        .join("parse");
+
+    let json_stringify = config
+        .root
+        .join("test")
+        .join("built-ins")
+        .join("JSON")
+        .join("stringify");
+
+    let parse_tests = gen_tests(&json_parse, &harness_str, "parse");
+    let stringify_tests = gen_tests(&json_stringify, &harness_str, "stringify");
+
+    Ok(quote! {
+        #parse_tests
+        #stringify_tests
+    }
+    .into())
+}
+
+fn gen_tests(
+    dir: &PathBuf,
+    harness_str: &String,
+    prefix: &'static str,
+) -> proc_macro2::TokenStream {
+    let parse_dir = std::fs::read_dir(dir).expect("parse directory to be available");
+    let spec = parse_dir.filter_map(|e| e.ok()).map(move |entry| {
+            let path = entry.path();
+            let path_str = path.clone().into_os_string().into_string().unwrap();
+            let name = path.file_stem().unwrap().to_str().unwrap();
+            let name = name.replace('.', "_");
+            let name = name.replace('-', "_");
+            let test_name = Ident::new(&format!("test_{}_{}", prefix, name), Span::call_site());
+            let ignore = ignore(&test_name.to_string());
+
+            let attrs = if ignore {
+                quote! {
+                    #[ignore]
+                }
+            } else {
+                quote! {}
+            };
+
+            quote! {
+                #[test]
+                #attrs
+                #[allow(non_snake_case)]
+                fn #test_name() {
+                    let mut config = ::javy::Config::default();
+                    config
+                        .override_json_parse_and_stringify(true);
+                    let runtime = ::javy::Runtime::new(config).expect("runtime to be created");
+                    let harness_path = ::std::path::PathBuf::from(#harness_str);
+
+                    let helpers = vec![
+                        harness_path.join("sta.js"),
+                        harness_path.join("assert.js"),
+                        harness_path.join("compareArray.js"),
+                        harness_path.join("propertyHelper.js"),
+                        harness_path.join("isConstructor.js")
+                    ];
+                    runtime.context().with(|this| {
+                        for helper in helpers {
+                            let helper_contents = ::std::fs::read(helper).expect("helper path to exist");
+                            let r: ::javy::quickjs::Result<()> = this.eval_with_options(helper_contents, ::javy::quickjs::context::EvalOptions::default()).expect("helper evaluation to succeed");
+                            assert!(r.is_ok());
+                        }
+
+                        let test_contents = ::std::fs::read(&#path_str).expect("test file contents to be available");
+                        let r: ::javy::quickjs::Result<()> = this.eval_with_options(test_contents, ::javy::quickjs::context::EvalOptions::default());
+                        assert!(r.is_ok(), "{}", ::javy::val_to_string(this.clone(), this.catch()).unwrap());
+                    });
+
+                }
+            }
+        });
+
+    quote! {
+        #(#spec)*
+    }
+}

crates/javy/Cargo.toml

@@ -23,6 +23,9 @@ bitflags = "2.5.0"
 fastrand = "2.1.0"
 simd-json = { version = "0.13.10", optional = true, default-features = false, features = ["big-int-as-float", "serde_impl"] }
 
+[dev-dependencies]
+javy-test-macros = { path = "../javy-test-macros/" }
+
 [features]
 export_alloc_fns = []
 messagepack = ["rmp-serde", "serde-transcode"]