|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Ecosystem verification for round 5: mcpp 2026.9.6.2, compat:spirv-headers, |
| 3 | +# and ggml-org:llamacpp's backend-vulkan feature. |
| 4 | +# |
| 5 | +# xlings subos verify-962 --sandbox --cmd \ |
| 6 | +# "MCPP_VERIFY_VERSION=2026.9.6.2 bash <this file>" |
| 7 | +# |
| 8 | +# and once on the host (MCPP_VERIFY_HOST=1), where a real device answers. |
| 9 | +# |
| 10 | +# WHAT A SANDBOX CAN AND CANNOT DECIDE HERE, AND WHY THE SPLIT IS DIFFERENT |
| 11 | +# FROM ROUND 4's. |
| 12 | +# |
| 13 | +# Round 4 split on /dev: the sandbox has fourteen entries and none is a GPU, so |
| 14 | +# any program reaching for one reports "no device" whatever the build did. |
| 15 | +# Round 5 has a software device -- Mesa's lavapipe, a payload -- so the sandbox |
| 16 | +# CAN run a kernel. What it cannot do is run llama.cpp on it without being told |
| 17 | +# to: ggml keeps only Vulkan devices whose type is not `eCpu`, and lavapipe |
| 18 | +# reports exactly that type while advertising every feature the backend |
| 19 | +# requires. Upstream's own selector (GGML_VK_VISIBLE_DEVICES) names a device by |
| 20 | +# index, and using it is what makes the sandbox leg meaningful rather than a |
| 21 | +# test of the runner's hardware. |
| 22 | +# |
| 23 | +# EVERY CRITERION HERE NAMES THE OBJECT IT SELECTED. Round 4 produced four |
| 24 | +# defects of one shape -- a check that chose its own object and did not say |
| 25 | +# which -- so each section prints the path, version or device it read. |
| 26 | +set -u |
| 27 | + |
| 28 | +VER="${MCPP_VERIFY_VERSION:?set MCPP_VERIFY_VERSION}" |
| 29 | +STORE="${MCPP_VERIFY_BIN:-$HOME/.xlings/data/xpkgs/xim-x-mcpp/$VER/bin/mcpp}" |
| 30 | +SRC="${MCPP_VERIFY_SRC:-}" # a checkout of llama.cpp-m at b10069.1+ |
| 31 | +XL="${XLINGS_BIN:-$(command -v xlings)}" |
| 32 | +MODEL="${LLAMACPP_TEST_MODEL:-}" |
| 33 | + |
| 34 | +fails=0 |
| 35 | +fail() { printf 'ASSERT-FAIL: %s\n' "$1"; fails=$((fails + 1)); } |
| 36 | +ok() { printf 'ok: %s\n' "$1"; } |
| 37 | +section() { printf '\n== %s ==\n' "$1"; } |
| 38 | + |
| 39 | +work=$(mktemp -d) |
| 40 | +trap 'rm -rf "$work"' EXIT |
| 41 | + |
| 42 | +# -- A. identity and mirror -------------------------------------------------- |
| 43 | +section "A. identity" |
| 44 | +if [ ! -x "$STORE" ]; then |
| 45 | + fail "no released binary at $STORE" |
| 46 | +else |
| 47 | + got=$("$STORE" --version 2>&1 | head -1) |
| 48 | + if [ "$got" = "mcpp $VER" ]; then ok "$got from $STORE" |
| 49 | + else fail "version is '$got' at $STORE"; fi |
| 50 | +fi |
| 51 | + |
| 52 | +# The SETTING, not the tool's banner: `xlings config` renders through a ui |
| 53 | +# layer that prints nothing into a pipe, so a command substitution reads empty |
| 54 | +# and reports a mirror that is in fact set. |
| 55 | +"$STORE" self config --mirror "${MCPP_VERIFY_MIRROR:-CN}" >/dev/null 2>&1 || true |
| 56 | +"$XL" config --mirror "${MCPP_VERIFY_MIRROR:-CN}" >/dev/null 2>&1 || true |
| 57 | +xm=$(python3 -c "import json,os;print(json.load(open(os.path.expanduser('~/.xlings/.xlings.json'))).get('mirror',''))" 2>/dev/null) |
| 58 | +if [ "$xm" = "${MCPP_VERIFY_MIRROR:-CN}" ]; then ok "xlings mirror is $xm" |
| 59 | +else fail "xlings mirror is '$xm'"; fi |
| 60 | + |
| 61 | +"$STORE" index update >/dev/null 2>&1 || true |
| 62 | +snap=$(cat "$HOME/.mcpp/registry/data/mcpplibs/.xlings-index-version" 2>/dev/null || echo unknown) |
| 63 | +ok "index snapshot $snap" |
| 64 | + |
| 65 | +# -- B. compat:spirv-headers ------------------------------------------------- |
| 66 | +section "B. compat:spirv-headers" |
| 67 | +# The criterion reads a DEFINITION back, not an exit status. A package that |
| 68 | +# resolved and exposed the wrong include root compiles nothing, so "it built" |
| 69 | +# would pass on an empty tree. |
| 70 | +mkdir -p "$work/spirv/tests" |
| 71 | +cat > "$work/spirv/mcpp.toml" <<'TOML' |
| 72 | +[package] |
| 73 | +name = "spirvprobe" |
| 74 | +version = "0.1.0" |
| 75 | +[dependencies.compat] |
| 76 | +spirv-headers = "1.4.357.0" |
| 77 | +TOML |
| 78 | +cat > "$work/spirv/tests/probe.cpp" <<'CPP' |
| 79 | +#include <spirv/unified1/spirv.hpp> |
| 80 | +import std; |
| 81 | +int main() { |
| 82 | + std::println("magic={:#x} opcap={} glcompute={}", |
| 83 | + (unsigned)spv::MagicNumber, |
| 84 | + (unsigned)spv::OpCapability, |
| 85 | + (unsigned)spv::ExecutionModelGLCompute); |
| 86 | + return spv::MagicNumber == 0x07230203u ? 0 : 1; |
| 87 | +} |
| 88 | +CPP |
| 89 | +out=$(cd "$work/spirv" && "$STORE" test 2>&1) |
| 90 | +if printf '%s' "$out" | grep -q 'magic=0x7230203 opcap=17 glcompute=5'; then |
| 91 | + ok "compat:spirv-headers delivers the Khronos layout and definitions" |
| 92 | +else |
| 93 | + fail "compat:spirv-headers probe did not read the definitions back" |
| 94 | + printf '%s\n' "$out" | tail -12 |
| 95 | +fi |
| 96 | + |
| 97 | +# -- C. the feature is additive ---------------------------------------------- |
| 98 | +section "C. a CPU consumer acquires nothing of the Vulkan backend" |
| 99 | +# The criterion is the RESOLUTION, not "the CPU build still works". A manifest |
| 100 | +# that made every consumer download a shader compiler would also still work. |
| 101 | +if [ -z "$SRC" ] || [ ! -f "$SRC/mcpp.toml" ]; then |
| 102 | + fail "set MCPP_VERIFY_SRC to a llama.cpp-m checkout" |
| 103 | +else |
| 104 | + cpu="$work/cpu"; cp -r "$SRC" "$cpu"; rm -rf "$cpu/target" |
| 105 | + (cd "$cpu" && "$STORE" build >/dev/null 2>&1) |
| 106 | + res=$(ls "$cpu"/target/*/*/resolution.json 2>/dev/null | head -1) |
| 107 | + if [ -z "$res" ]; then |
| 108 | + fail "no resolution.json under $cpu/target" |
| 109 | + else |
| 110 | + ok "reading $res" |
| 111 | + if grep -qE '"(compat[.:]vulkan|compat[.:]spirv-headers|shaderc|mesa-lavapipe)' "$res"; then |
| 112 | + fail "the CPU build's resolution names a Vulkan-backend package" |
| 113 | + grep -oE '"(compat[.:][a-z-]+|xim:[a-z-]+)"' "$res" | sort -u | head -20 |
| 114 | + else |
| 115 | + ok "no Vulkan-backend package appears in the CPU resolution" |
| 116 | + fi |
| 117 | + fi |
| 118 | +fi |
| 119 | + |
| 120 | +# -- D. the Vulkan backend builds, and the shaders are edges ----------------- |
| 121 | +section "D. backend-vulkan" |
| 122 | +if [ -z "$SRC" ] || [ ! -f "$SRC/mcpp.toml" ]; then |
| 123 | + fail "set MCPP_VERIFY_SRC to a llama.cpp-m checkout" |
| 124 | +else |
| 125 | + vk="$work/vk"; cp -r "$SRC" "$vk"; rm -rf "$vk/target" |
| 126 | + if (cd "$vk" && "$STORE" build --features backend-vulkan >"$work/vk.log" 2>&1); then |
| 127 | + ok "backend-vulkan builds" |
| 128 | + else |
| 129 | + fail "backend-vulkan build failed" |
| 130 | + tail -20 "$work/vk.log" |
| 131 | + fi |
| 132 | + |
| 133 | + # The shaders are GRAPH EDGES, and the count is the criterion. A build |
| 134 | + # program that generated them itself would leave the same artifacts here |
| 135 | + # and no edges in build.ninja. |
| 136 | + ninja=$(ls "$vk"/target/*/*/build.ninja 2>/dev/null | head -1) |
| 137 | + comps=$(ls "$vk"/third_party/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/*.comp 2>/dev/null | wc -l) |
| 138 | + if [ -n "$ninja" ] && [ "$comps" -gt 100 ]; then |
| 139 | + ok "reading $ninja against $comps vendored shaders" |
| 140 | + rules=$(grep -c '^rule mcpp_action_' "$ninja" || true) |
| 141 | + if [ "$rules" -ge "$((comps + 2))" ]; then |
| 142 | + ok "$rules declared action rules for $comps shaders plus the generator and header" |
| 143 | + else |
| 144 | + fail "$rules action rules for $comps shaders; expected at least $((comps + 2))" |
| 145 | + fi |
| 146 | + objs=$(grep -c '\.comp\.o :' "$ninja" || true) |
| 147 | + if [ "$objs" -eq "$comps" ]; then |
| 148 | + ok "$objs generated shader sources joined the compile set" |
| 149 | + else |
| 150 | + fail "$objs generated shader objects for $comps shaders" |
| 151 | + fi |
| 152 | + else |
| 153 | + fail "no build.ninja or no vendored shaders under $vk" |
| 154 | + fi |
| 155 | + |
| 156 | + # The archive must CARRY them. A static library that linked nothing still |
| 157 | + # builds, which is how the engine defect this round fixed stayed quiet. |
| 158 | + ar_bin=$(command -v ar || true) |
| 159 | + lib=$(ls "$vk"/target/*/*/bin/libllama.a 2>/dev/null | head -1) |
| 160 | + if [ -n "$ar_bin" ] && [ -n "$lib" ]; then |
| 161 | + members=$("$ar_bin" t "$lib" | grep -c '\.comp\.o$' || true) |
| 162 | + if [ "$members" -eq "$comps" ]; then |
| 163 | + ok "$lib carries $members shader objects" |
| 164 | + else |
| 165 | + fail "$lib carries $members shader objects, expected $comps" |
| 166 | + fi |
| 167 | + else |
| 168 | + fail "no libllama.a or no ar to read it" |
| 169 | + fi |
| 170 | +fi |
| 171 | + |
| 172 | +# -- E. the device answers --------------------------------------------------- |
| 173 | +section "E. the device decode equals the host decode" |
| 174 | +if [ -z "$MODEL" ] || [ ! -f "$MODEL" ]; then |
| 175 | + fail "set LLAMACPP_TEST_MODEL to a .gguf (tests/support/fetch_model.py writes one)" |
| 176 | +elif [ -z "$SRC" ]; then |
| 177 | + fail "set MCPP_VERIFY_SRC to a llama.cpp-m checkout" |
| 178 | +else |
| 179 | + icd=$(find "$HOME/.mcpp/registry/data/xpkgs/xim-x-mesa-lavapipe" \ |
| 180 | + "$HOME/.xlings/data/xpkgs/xim-x-mesa-lavapipe" \ |
| 181 | + -name 'lvp_icd.*.json' -print -quit 2>/dev/null || true) |
| 182 | + if [ -n "$icd" ]; then |
| 183 | + ok "software device ICD at $icd" |
| 184 | + else |
| 185 | + ok "no lavapipe payload found; using whatever driver this machine has" |
| 186 | + fi |
| 187 | + # GGML_VK_VISIBLE_DEVICES is the selector, not a workaround: ggml drops |
| 188 | + # `eCpu` devices by type, and a software implementation is one. |
| 189 | + env_prefix=(env "LLAMACPP_TEST_MODEL=$MODEL" "GGML_VK_VISIBLE_DEVICES=0") |
| 190 | + [ -n "$icd" ] && env_prefix+=("VK_DRIVER_FILES=$icd") |
| 191 | + if (cd "$work/vk" 2>/dev/null && "${env_prefix[@]}" \ |
| 192 | + "$STORE" test vulkan_decode --features backend-vulkan >"$work/dec.log" 2>&1); then |
| 193 | + : |
| 194 | + fi |
| 195 | + if grep -q 'LLAMACPP_VULKAN_TEST=PASS' "$work/dec.log" 2>/dev/null; then |
| 196 | + ok "$(grep -m1 'vulkan device:' "$work/dec.log")" |
| 197 | + ok "$(grep -m1 'host token:' "$work/dec.log")" |
| 198 | + ok "$(grep -m1 -E 'offloaded [1-9][0-9]*/[1-9][0-9]* layers to GPU' "$work/dec.log")" |
| 199 | + else |
| 200 | + fail "the device decode did not pass" |
| 201 | + tail -20 "$work/dec.log" 2>/dev/null |
| 202 | + fi |
| 203 | +fi |
| 204 | + |
| 205 | +printf '\n== summary ==\n' |
| 206 | +if [ "$fails" -eq 0 ]; then |
| 207 | + printf 'PASS: 0 assertions failed\n' |
| 208 | +else |
| 209 | + printf 'FAIL: %s assertion(s) failed\n' "$fails" |
| 210 | +fi |
| 211 | +exit "$fails" |
0 commit comments