fix: remediate verifier quality issues to eliminate all extension_only tasks

sjarmak · claude · sjarmak · commit 49e5b8ea09f0 · 2026-03-07T22:44:18.000Z
Fixes /tmp path collisions in 16 verifiers (onboard-search + 2 others) by
switching to mktemp in /logs/verifier/. Improves abc_audit T.9 heuristic to
recognize content validation inside heredocs (json.loads, re.search, python3 &lt;&lt;).

Results: 52 extension_only → 0. Manifest now hits all suite allocation targets
including previously shortfalled understand (8/8) and document (4/4).

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-201/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-201/tests/test.sh
@@ -30,7 +30,7 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-VERIFY_SCRIPT=$(mktemp /tmp/verify_XXXXXX.py)
+VERIFY_SCRIPT=$(mktemp /logs/verifier/verify_XXXXXX.py)
 cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-202/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-202/tests/test.sh
@@ -30,7 +30,7 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-VERIFY_SCRIPT=$(mktemp /tmp/verify_XXXXXX.py)
+VERIFY_SCRIPT=$(mktemp /logs/verifier/verify_XXXXXX.py)
 cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-203/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-203/tests/test.sh
@@ -30,7 +30,7 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-VERIFY_SCRIPT=$(mktemp /tmp/verify_XXXXXX.py)
+VERIFY_SCRIPT=$(mktemp /logs/verifier/verify_XXXXXX.py)
 cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-204/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-204/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-205/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-205/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-206/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-206/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-207/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-207/tests/test.sh
@@ -29,7 +29,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -67,5 +68,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-208/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-208/tests/test.sh
@@ -29,7 +29,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -67,5 +68,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-209/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-209/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-210/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-210/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-211/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-211/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-212/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-212/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-213/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-213/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-214/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-214/tests/test.sh
@@ -30,7 +30,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT="$(mktemp /logs/verifier/verify_XXXXXX.py)"
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -68,5 +69,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
 exit 0
+rm -f "$VERIFY_SCRIPT"
diff --git a/benchmarks/csb_sdlc_design/etcd-grpc-api-upgrade-001/tests/test.sh b/benchmarks/csb_sdlc_design/etcd-grpc-api-upgrade-001/tests/test.sh
@@ -32,7 +32,7 @@ else
     echo "No patch at $PATCH_FILE — trying fallback diff collection..." >&2
 
     # Fallback: generate diff from git changes in the source repos
-    FALLBACK_PATCH=$(mktemp /tmp/fallback_XXXXXX.diff)
+    FALLBACK_PATCH=$(mktemp /logs/verifier/fallback_XXXXXX.diff)
     > "$FALLBACK_PATCH"
 
     for repo_dir in "$CORPUS_ROOT"/src/*/; do
diff --git a/benchmarks/csb_sdlc_refactor/python-http-class-naming-refac-001/tests/test.sh b/benchmarks/csb_sdlc_refactor/python-http-class-naming-refac-001/tests/test.sh
@@ -35,7 +35,7 @@ else
     echo "No patch at $PATCH_FILE — trying fallback diff collection..." >&2
 
     # Fallback: generate diff from git changes in the source repos
-    FALLBACK_PATCH=$(mktemp /tmp/fallback_XXXXXX.diff)
+    FALLBACK_PATCH=$(mktemp /logs/verifier/fallback_XXXXXX.diff)
     > "$FALLBACK_PATCH"
 
     for repo_dir in "$CORPUS_ROOT"/src/*/; do
diff --git a/configs/core_benchmark_manifest.json b/configs/core_benchmark_manifest.json
diff --git a/configs/core_manifest_validation.json b/configs/core_manifest_validation.json
diff --git a/configs/verifier_quality_labels.json b/configs/verifier_quality_labels.json
diff --git a/scripts/abc_audit.py b/scripts/abc_audit.py
