Merge wasmerio#1380

1380: Test refactor r=syrusakbary a=syrusakbary

<!-- 
Prior to submitting a PR, review the CONTRIBUTING.md document for recommendations on how to test:
https://github.com/wasmerio/wasmer/blob/master/CONTRIBUTING.md#pull-requests

-->

# Description

This PR refactors the way we do testing in our infrastructure, simplifying test generation specially for spectests for each of the backends.

<!-- 
Provide details regarding the change including motivation,
links to related issues, and the context of the PR.
-->

# Review

- [ ] Add a short description of the the change to the CHANGELOG.md file


Co-authored-by: Syrus <[email protected]>

ATTRIBUTIONS.md

@@ -10,15 +10,14 @@ These are the projects that were used as inspiration and/or that we are using co
 - [WAVM](https://github.com/wavm/wavm): for their great integration and testing framework
 - [greenwasm](https://github.com/Kimundi/greenwasm): for their [spectests framework](https://github.com/Kimundi/greenwasm/tree/master/greenwasm-spectest)
 - [wasmtime](https://github.com/CraneStation/wasmtime):
-  - For their [mmap implementation](https://github.com/CraneStation/wasmtime/blob/3f24098edc81cd9bf0f877fb7fba018cad0f039e/lib/runtime/src/mmap.rs)
-  - For the implementation of the `__jit_debug_register_code` function
-    in Rust, the structure of using Cranelift with the GDB JIT
-    interface including implementation details regarding the structure
-    of generating debug information for each function with Cranelift
-    (for example, the sorting of the extended basic blocks before
-    processing the instructions), and the API for transforming DWARF
-    see [wasm-debug's attribution file](https://github.com/wasmerio/wasm-debug/blob/master/ATTRIBUTIONS.md)
-    for more information
+  For their [mmap implementation](https://github.com/CraneStation/wasmtime/blob/3f24098edc81cd9bf0f877fb7fba018cad0f039e/lib/runtime/src/mmap.rs), the wast test implementation and the implementation of the `__jit_debug_register_code` function
+  in Rust, the structure of using Cranelift with the GDB JIT
+  interface including implementation details regarding the structure
+  of generating debug information for each function with Cranelift
+  (for example, the sorting of the extended basic blocks before
+  processing the instructions), and the API for transforming DWARF
+  see [wasm-debug's attribution file](https://github.com/wasmerio/wasm-debug/blob/master/ATTRIBUTIONS.md)
+  for more information
 - [stackoverflow](https://stackoverflow.com/a/45795699/1072990): to create an efficient HashMap with pair keys
 - [Emscripten](https://github.com/kripken/emscripten): for emtests test sources to ensure compatibility
 - [The WebAssembly spec](https://github.com/WebAssembly/spec/tree/master/test): for implementation details of WebAssembly and spectests

Cargo.toml

@@ -59,18 +59,23 @@ members = [
     "examples/parallel",
     "examples/plugin-for-example",
     "examples/parallel-guest",
+    "tests/test-generator",
     "tests/generate-wasi-tests",
     "tests/generate-emscripten-tests",
+    "tests/wast",
 ]
 
 [build-dependencies]
-wabt = "0.9.1"
+anyhow = "1.0.19"
 generate-emscripten-tests = { path = "tests/generate-emscripten-tests" }
 generate-wasi-tests = { path = "tests/generate-wasi-tests" }
+test-generator = { path = "tests/test-generator" }
 glob = "0.3"
 rustc_version = "0.2"
 
 [dev-dependencies]
+anyhow = "1.0.19"
+wasmer-wast = { path = "tests/wast" }
 criterion = "0.3"
 glob = "0.3"
 libc = "0.2.60" # for `tests/dev-utils`'s Stdout capturing

Makefile

@@ -35,17 +35,17 @@ generate: generate-emtests generate-wasitests
 
 # Spectests
 spectests-singlepass:
-	WASMER_TEST_SINGLEPASS=1 cargo test test_run_spectests --release --no-default-features --features "wasi backend-singlepass" -- --nocapture --test-threads 1
+	WASMER_TEST_SINGLEPASS=1 cargo test singlepass::spec --release --no-default-features --features "wasi backend-singlepass" -- --nocapture
 
 spectests-cranelift:
-	WASMER_TEST_CRANELFIT=1 cargo test test_run_spectests --release --no-default-features --features "wasi backend-cranelift" -- --nocapture
+	WASMER_TEST_CRANELFIT=1 cargo test cranelift::spec --release --no-default-features --features "wasi backend-cranelift" -- --nocapture
 
 spectests-llvm:
-	WASMER_TEST_LLVM=1 cargo test test_run_spectests --release --no-default-features --features "wasi backend-llvm wasmer-llvm-backend/test" -- --nocapture
+	WASMER_TEST_LLVM=1 cargo test llvm::spec --release --no-default-features --features "wasi backend-llvm wasmer-llvm-backend/test" -- --nocapture --test-threads=1
 
 spectests-all:
 	WASMER_TEST_CRANELIFT=1 WASMER_TEST_LLVM=1 WASMER_TEST_SINGLEPASS=1 \
-	cargo test test_run_spectests --release --no-default-features --features "wasi backend-cranelift backend-singlepass backend-llvm wasmer-llvm-backend/test" -- --nocapture --test-threads 1
+	cargo test spec --release --no-default-features --features "wasi backend-cranelift backend-singlepass backend-llvm wasmer-llvm-backend/test" 
 
 
 spectests: spectests-singlepass spectests-cranelift spectests-llvm

build.rs

@@ -4,8 +4,108 @@
 
 use generate_emscripten_tests;
 use generate_wasi_tests;
+use std::env;
+use std::fmt::Write;
+use std::fs;
+use std::path::PathBuf;
+use std::process::Command;
+use test_generator::{
+    build_ignores_from_textfile, extract_name, test_directory, test_directory_module,
+    with_test_module, Test, Testsuite,
+};
+
+/// Given a Testsuite and a path, process the path in case is a wast
+/// file.
+fn wast_processor(out: &mut Testsuite, p: PathBuf) -> Option<Test> {
+    let ext = p.extension()?;
+    // Only look at wast files.
+    if ext != "wast" {
+        return None;
+    }
+
+    // Ignore files starting with `.`, which could be editor temporary files
+    if p.file_stem()?.to_str()?.starts_with(".") {
+        return None;
+    }
+
+    let testname = extract_name(&p);
+    let body = format!(
+        "crate::run_wast(r#\"{}\"#, \"{}\")",
+        p.display(),
+        out.path.get(0).unwrap()
+    );
+
+    Some(Test {
+        name: testname.to_string(),
+        body: body.to_string(),
+    })
+}
+
+fn main() -> anyhow::Result<()> {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=test/ignores.txt");
 
-fn main() {
     generate_wasi_tests::build();
     generate_emscripten_tests::build();
+
+    let out_dir = PathBuf::from(
+        env::var_os("OUT_DIR").expect("The OUT_DIR environment variable must be set"),
+    );
+    let ignores = build_ignores_from_textfile("tests/ignores.txt".into())?;
+    let mut out = Testsuite {
+        buffer: String::new(),
+        path: vec![],
+        ignores: ignores,
+    };
+
+    for compiler in &["singlepass", "cranelift", "llvm"] {
+        writeln!(out.buffer, "#[cfg(feature=\"backend-{}\")]", compiler);
+        writeln!(out.buffer, "#[cfg(test)]")?;
+        writeln!(out.buffer, "#[allow(non_snake_case)]")?;
+        with_test_module(&mut out, compiler, |mut out| {
+            with_test_module(&mut out, "spec", |out| {
+                let spec_tests = test_directory(out, "tests/spectests", wast_processor)?;
+                // Skip running spec_testsuite tests if the submodule isn't checked
+                // out.
+                // if spec_tests > 0 {
+                //     test_directory_module(
+                //         out,
+                //         "tests/spec_testsuite/proposals/simd",
+                //         wast_processor,
+                //     )?;
+                //     test_directory_module(
+                //         out,
+                //         "tests/spec_testsuite/proposals/multi-value",
+                //         wast_processor,
+                //     )?;
+                //     test_directory_module(
+                //         out,
+                //         "tests/spec_testsuite/proposals/reference-types",
+                //         wast_processor,
+                //     )?;
+                //     test_directory_module(
+                //         out,
+                //         "tests/spec_testsuite/proposals/bulk-memory-operations",
+                //         wast_processor,
+                //     )?;
+                // } else {
+                //     println!(
+                //         "cargo:warning=The spec testsuite is disabled. To enable, run `git submodule \
+                //     update --remote`."
+                //     );
+                // }
+                Ok(())
+            })?;
+            Ok(())
+        })?;
+    }
+
+    // println!("{}", out.buffer);
+    // std::process::exit(1);
+    // Write out our auto-generated tests and opportunistically format them with
+    // `rustfmt` if it's installed.
+    let output = out_dir.join("generated_tests.rs");
+    fs::write(&output, out.buffer)?;
+    Command::new("rustfmt").arg(&output).status();
+    Ok(())
 }

ci/docker/aarch64-linux-android/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:19.04
+FROM ubuntu:19.10
 
 RUN dpkg --add-architecture i386 && \
     apt-get update && \

ci/docker/x86_64-linux-android/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:19.04
+FROM ubuntu:19.10
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

lib/runtime-core/src/import.rs

@@ -178,6 +178,11 @@ impl ImportObject {
         }
         out
     }
+
+    /// Returns true if the ImportObject contains namespace with the provided name.
+    pub fn contains_namespace(&self, name: &str) -> bool {
+        self.map.lock().unwrap().borrow().contains_key(name)
+    }
 }
 
 /// Iterator for an `ImportObject`'s exports.

tests/emtest.rs

@@ -1,2 +1,2 @@
-pub mod dev_utils;
 mod emtests;
+pub mod utils;

tests/emtests/_common.rs

@@ -37,7 +37,7 @@ macro_rules! assert_emscripten_output {
             EmscriptenGlobals,
             generate_emscripten_env,
         };
-        use crate::dev_utils::stdio::StdioCapturer;
+        use crate::utils::stdio::StdioCapturer;
 
         let wasm_bytes = include_bytes!($file);
         let backend = $crate::emtests::_common::get_backend().expect("Please set one of `WASMER_TEST_CRANELIFT`, `WASMER_TEST_LLVM`, or `WASMER_TEST_SINGELPASS` to `1`.");