Cover 100% of core spec tests, support WIT codegen

Here is a more detailed breakdown for specific changes:
1. WebAssembly binary parser change to make it compliant with the spec. Files changed:
./Sources/WasmKit/Parser/
./Tests/WasmKitTests/Parser/

2. WebAssembly interpreter changes to make it compliant with the spec. No JIT or assembly code generation done in the interpreter. Files changed:
./Sources/WasmKit/Execution/
./Sources/WasmKit/Types/
./Tests/WasmKitTests/Execution/

3. WASI system interface changes to make it compliant with the spec.
./Sources/WASI/
./Sources/SystemExtras/
./Tests/WASITests/
./IntegrationTests/WASI/

4 .WIT parser change to make it compliant with the Component Model specification. Files changed:
./Sources/WIT/
./Tests/WITTests/

5. WIT code generation change. We should mention that this only generates Swift code statically and no binary code is generated. It’s similar to protoc in Protocol Buffers or Swift OpenAPI Generator projects that are already open-source. No “hot reloading” or dynamic code generation is done (i.e. when application is running).
Files changed:
./Plugins/
./Sources/WITExtractor/
./Sources/WITOverlayGenerator/
./Sources/WasmKit/Component/
./Tests/WITExtractorTests/
./Tests/WITOverlayGeneratorTests/
./Tests/WITExtractorPluginTests/

Co-authored-by: Yuta Saito <[email protected]>

.gitignore

@@ -1,9 +1,13 @@
+.DS_Store
+
 .build
 *.xcodeproj
 .swiftpm
 
-/Tests/LinuxMain.swift
-/Tests/WAKitTests/XCTestManifests.swift
-
 /spectest
 .vscode
+
+/Tests/default.json
+/Tests/WITOverlayGeneratorTests/Compiled/
+/Tests/WITOverlayGeneratorTests/Generated/
+Tests/WITExtractorPluginTests/Fixtures/*/Package.resolved

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "Vendor/Mint"]
-	path = Vendor/Mint
-	url = https://github.com/yonaskolb/Mint.git
-[submodule "Vendor/spec"]
-	path = Vendor/spec
-	url = https://github.com/WebAssembly/spec
+[submodule "Vendor/testsuite"]
+	path = Vendor/testsuite
+	url = https://github.com/WebAssembly/testsuite.git
+[submodule "Vendor/wasi-testsuite"]
+	path = Vendor/wasi-testsuite
+	url = https://github.com/WebAssembly/wasi-testsuite.git

.swift-format

@@ -0,0 +1,7 @@
+{
+  "version": 1,
+  "lineLength": 200,
+  "indentation": {
+    "spaces": 4
+  }
+}

CI/Sources/os-check.sh

@@ -0,0 +1,19 @@
+is_amazonlinux2() {
+  if [ -f /etc/os-release ]; then
+    source /etc/os-release
+    if [ "$ID" == "amzn" ]; then
+      return 0
+    fi
+  fi
+  return 1
+}
+
+is_debian_family() {
+  if [ -f /etc/os-release ]; then
+    source /etc/os-release
+    if [ "$ID_LIKE" == "debian" ]; then
+      return 0
+    fi
+  fi
+  return 1
+}

CI/check-spectest.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# A CI script to run "make spectest" with wabt installed.
+#
+
+set -eu -o pipefail
+source "$(dirname $0)/Sources/os-check.sh"
+
+install_tools() {
+  if ! which make curl cmake ninja python3 xz > /dev/null; then
+    apt update && apt install -y curl build-essential cmake ninja-build python3 xz-utils
+  fi
+
+  if ! which wat2wasm > /dev/null; then
+    local build_dir=$(mktemp -d /tmp/WasmKit-wabt.XXXXXX)
+    mkdir -p $build_dir
+    curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.33/wabt-1.0.33.tar.xz | tar xJ --strip-components=1 -C $build_dir
+    cmake -B $build_dir/build -GNinja -DBUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local $build_dir
+    cmake --build $build_dir/build --target install
+  fi
+
+  echo "Use wat2wasm $(wat2wasm --version): $(which wat2wasm)"
+  echo "Use wasm2wat $(wasm2wat --version): $(which wasm2wat)"
+}
+
+# Currently wabt is unavailable in amazonlinux2, so we skip the spectest on it.
+if is_amazonlinux2; then
+  echo "Skip spectest on amazonlinux2"
+  exit 0
+fi
+
+set -e
+
+install_tools
+
+SOURCE_DIR="$(cd $(dirname $0)/.. && pwd)"
+exec make -C $SOURCE_DIR spectest

CI/check-wasi-testsuite.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# A CI script to run wasi-testsuite
+#
+
+set -eu -o pipefail
+source "$(dirname $0)/Sources/os-check.sh"
+
+install_tools() {
+  if is_amazonlinux2; then
+    amazon-linux-extras install -y python3.8
+    ln -s /usr/bin/python3.8 /usr/bin/python3
+  elif is_debian_family; then
+    apt-get update
+    apt-get install -y python3-pip
+  else
+    echo "Unknown OS"
+    exit 1
+  fi
+}
+
+install_tools
+
+SOURCE_DIR="$(cd $(dirname $0)/.. && pwd)"
+(
+  cd $SOURCE_DIR && \
+  python3 -m pip install -r ./Vendor/wasi-testsuite/test-runner/requirements.txt && \
+  exec ./IntegrationTests/WASI/run-tests.sh
+)

Documentation/ComponentModel/CanonicalABI.md

@@ -0,0 +1,79 @@
+# Implementation notes on Canonical ABI
+
+Component model defines a high-level interface between components. The interface defined by the WIT is mapped to the low-level core values and memory operations. The mapping is called the Canonical ABI.
+The key idea of the Canonical ABI is to define a set of operations that can be used to translate between the WIT values and the core values.
+
+Each WIT type has 2 key operations, [lift and lower](https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md#lifting-and-lowering-values):
+
+- Lift: Translates core values to a WIT value.
+- Lower: Translates a WIT value to core values.
+
+Considering Component A and B, where A is calling a function exported by B that takes a string and returns an integer, the following diagram shows how the operations are used.
+
+```mermaid
+graph LR;
+    subgraph CA[Component A]
+        F1["(string) -> u32"]
+
+        Import["(i32, i32) -> i32"]
+        F1 --> |"lower"| Import
+    end
+
+    subgraph CB[Component B]
+        F2["(string) -> u32"]
+        Export["(i32, i32) -> i32"]
+
+        Export --> |"lift"| F2
+    end
+
+    Import --> |"invoke"| Export
+```
+
+```mermaid
+graph RL;
+    subgraph CA[Component A]
+        F1["(string) -> u32"]
+
+        Import["(i32, i32) -> i32"]
+        Import --> |"lift"| F1
+    end
+
+    subgraph CB[Component B]
+        F2["(string) -> u32"]
+        Export["(i32, i32) -> i32"]
+
+        F2 --> |"lower"| Export
+    end
+
+    Export --> |"return"| Import
+```
+
+## Lifting
+
+Lifting operation translates core values to a WIT value. It is used when calling a WIT-typed function from a core-typed function, or when returning a WIT value from a core-typed function. The operation can be split into 2 parts:
+
+1. [Flat Lifting](https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md#flat-lifting): Translates a list of core values to a WIT value.
+2. [Loading](https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md#loading): Reads a WIT value from the memory.
+
+The loading operation is only used when the value is too large to be passed as a function argument, or too large to be returned from a function. Currently, the number of return value known as [`MAX_FLAT_RESULTS`](https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md#flattening) is limited to 1 in the core-level signature so that the Canonical ABI can be implemented without the multi-value proposal.
+
+## Lowering
+
+Lowering operation translates a WIT value to core values. It is used when calling a core-typed function from a WIT-typed function, or when returning a core value from a WIT-typed function. The operation can be split into 2 parts:
+
+1. [Flat Lowering](https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md#flat-lowering): Translates a WIT value to a list of core values.
+2. [Storing](https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md#storing): Writes a WIT value to the memory.
+
+The same as the loading operation, the storing operation is only used when the WIT value is too large to be passed as a function argument, or too large to be returned from a function.
+
+## Code sharing for static and dynamic operations
+
+There are 3 places that the Canonical ABI needs to be implemented:
+
+1. Code Generator: Statically generate the code for lifting and lowering at the Swift level for guest components and host runtime.
+2. Host Runtime: Dynamically exchange the WIT values between guest components based on the given WIT definition at runtime.
+3. AOT/JIT Compiler: Statically generate the code for lifting and lowering with the given WIT definition at runtime.
+
+To reduce the code duplication and maintenance cost, WasmKit uses the same ABI modeling code that describes the essential logic of each lifting, lowering, loading, and storing operation in abstract ways. (See [`Sources/WIT/CanonicalABI/`](../../Sources/WIT/CanonicalABI))
+
+The ABI modeling code is designed to be used in both static and dynamic contexts. In the static context, each operation is performed at the meta-level, which means the operation is not actually executed but only used to construct the sequence of instructions. In the dynamic context, the operation is actually executed.

Documentation/ComponentModel/SemanticsNotes.md

@@ -0,0 +1,84 @@
+# Component Model semantics notes
+
+## Identifier namespace
+
+Function and type names in an interface should be unique within the interface. Each interface has its own
+namespace. A world has its own namespace for interface and function. An interface defined in a world with
+the same name with another interface defined in a package should be distinguished.
+
+```wit
+package ns:pkg
+interface iface {
+  type my-type = u8
+}
+world w {
+  interface ns-pkg-iface {
+    type my-type = u32
+  }
+}
+```
+In Swift, we can't use kebab-case, `:`, and `/` in identifiers, so we need to transform the identifiers defined in WIT
+to PascalCase by replacing `-`, `:`, and `/` and upcase the first letter following those symbols.
+Therefore, our overlay code generator cannot accept the above WIT definition, while it conflicts `ns:pkg/iface`
+and `ns-pkg-iface`. In the future, we can implement name escaping, but it requires careful transformation, so
+we postponed its implementation for now.
+
+## World
+
+A World corresponds to a component, in Swift toolchain, a linked WebAssembly binary after wasm-ld.
+A component contains only single World. A world can include other worlds, but items in the included Worlds
+are flattened into the including World, it doesn't violate single-world rule.
+
+## Import
+
+A World can import `interface` and bare functions.
+A function imported through `interface` defined in package-level has module name
+`my-namespace:my-pkg/my-interface`. The namespace and package names are where the interface
+is originally defined. Alias names in top-level use are not used in the import name.
+A function imported through `interface` defined in world-level has module name `my-interface`.
+A bare function defined directly in world like `import f: func()` has module name `$root`.
+
+## Resource methods
+
+A resource method can be defined within a `resource` definition. The Component Model proposal does not
+explicitly specifies which component is responsible to provide the resource method definition, but usually a component
+that exposes an interface that includes the resource type definition in WIT level is expected to provide the resource
+methods. Consider the following example:
+```
+package example:http
+interface handler {
+  record header-entry {
+    key: string,
+    value: string,
+  }
+  resource blob {
+    constructor(bytes: list<u8>)
+    size: func() -> u32
+  }
+  record message {
+    body: own<blob>,
+    headers: list<header-entry>,
+  }
+  handle: func(request: message) -> message
+}
+world service {
+  export handler
+}
+world middleware {
+  import handler
+  export handler
+}
+```
+
+In this case, both `service` and `middleware` components are responsible to provide the following implementations:
+
+- `example:http/handler#[constructor]blob`
+- `example:http/handler#[dtor]blob`
+- `example:http/handler#[method]blob.size`
+- `example:http/handler#handle`
+
+A type defined in `handler` interface can be shared between export and import interfaces unless it transitively
+uses a `resource` type. In this case, `header-entry` type can be shared, but `message` and `blob` types can't.
+This is because each resource type in import and export has its own constructor, destructor, and methods implementations
+even though they both have the same raw representation. A `message` passing to or returned from an imported function
+should call imported implementations and vice vasa.

Examples/wasm/factorial.wat

@@ -0,0 +1,11 @@
+  (func $fac (export "fac") (param i64) (result i64)
+    (if (result i64) (i64.eqz (local.get 0))
+      (then (i64.const 1))
+      (else
+        (i64.mul
+          (local.get 0)
+          (call $fac (i64.sub (local.get 0) (i64.const 1)))
+        )
+      )
+    )
+  )

Examples/wasm/fib.wat

@@ -0,0 +1,46 @@
+(module
+  (type (;0;) (func))
+  (type (;1;) (func (param i32) (result i32)))
+  (func (;0;) (type 0))
+  (func (;1;) (type 1) (param i32) (result i32)
+    (local i32)
+    i32.const 1
+    local.set 1
+    block  ;; label = @1
+      local.get 0
+      i32.const 2
+      i32.lt_s
+      br_if 0 (;@1;)
+      local.get 0
+      i32.const 2
+      i32.add
+      local.set 0
+      i32.const 1
+      local.set 1
+      loop  ;; label = @2
+        local.get 0
+        i32.const -3
+        i32.add
+        call 1
+        local.get 1
+        i32.add
+        local.set 1
+        local.get 0
+        i32.const -2
+        i32.add
+        local.tee 0
+        i32.const 3
+        i32.gt_s
+        br_if 0 (;@2;)
+      end
+    end
+    local.get 1)
+  (table (;0;) 1 1 funcref)
+  (memory (;0;) 2)
+  (global (;0;) (mut i32) (i32.const 66560))
+  (global (;1;) i32 (i32.const 66560))
+  (global (;2;) i32 (i32.const 1024))
+  (export "memory" (memory 0))
+  (export "__heap_base" (global 1))
+  (export "__data_end" (global 2))
+  (export "fib" (func 1)))

Examples/wasm/host.wat

@@ -0,0 +1,18 @@
+(module
+  (global (export "global_i32") i32 (i32.const 666))
+  (global (export "global_i64") i64 (i64.const 666))
+  (global (export "global_f32") f32 (f32.const 666))
+  (global (export "global_f64") f64 (f64.const 666))
+
+  (table (export "table") 10 20 funcref)
+
+  (memory (export "memory") 1 2)
+
+  (func (export "print"))
+  (func (export "print_i32") (param i32))
+  (func (export "print_i64") (param i64))
+  (func (export "print_f32") (param f32))
+  (func (export "print_f64") (param f64))
+  (func (export "print_i32_f32") (param i32 f32))
+  (func (export "print_f64_f64") (param f64 f64))
+)