feat: curate oracle_answer.json for all 130 new MCP tasks (142-271)

- Populated oracle_answer.json files for all 130 stub tasks across 10 suites:
  ccb_mcp_incident (142-150), ccb_mcp_domain (151-160),
  ccb_mcp_security (161-170), ccb_mcp_crossrepo_tracing (171-181),
  ccb_mcp_compliance (182-194), ccb_mcp_migration (195-207),
  ccb_mcp_crossorg (208-222), ccb_mcp_org (223-237),
  ccb_mcp_platform (238-252), ccb_mcp_crossrepo (253-271)
- Synced required_files into task_spec.json via new sync_oracle_files.py
  (preserves evaluation.checks/search_pattern, unlike hydrate_task_specs.py)
- Added 4 new fixture repo_sets: multi-org-go, nodejs-web-stack,
  prometheus-monitoring, python-ml-stack
- Added oracle_curation_log.json to .gitignore
- Fixed curate_oracle.py backoff: max 120s, base delay 1.0s

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sjarmak

.gitignore

@@ -43,6 +43,9 @@ harbor-ccb_crossrepo-dataset/
 vendor/DependEval/
 vendor/dependeval_repos/
 
+# Oracle curation logs (auto-generated, not needed for benchmark)
+oracle_curation_log.json
+
 # Credentials
 *.key
 *.pem

benchmarks/ccb_mcp_compliance/ccx-compliance-057/tests/task_spec.json

@@ -5,10 +5,13 @@
   "category": "F",
   "mcp_suite": "ccb_mcp_compliance",
   "prd": {
-    "user_story": "As a developer, I want to: Find ALL files in `grafana/grafana` that form the SSO settings control across 4 layers: 1. Feature Flag Definition — the registry where `ssoSettingsApi` is defined and generated constants. 2. SSO Settings Infrastructure — Service interface, Reloadable interface, SSOSettings data model, and SSOSettingsStore database layer. 3. API and Authentication Wiring — REST API endpoint registration with access control middleware, SocialService provider, and authentication client registration. 4. Access Control and DI Registration — access control evaluators and the ProvideService dependency injection function.",
-    "constraints": ["Provide specific file paths and repository names in your answer.", "Write your findings to /workspace/answer.json."],
+    "user_story": "As a developer, I want to: Find ALL files in `grafana/grafana` that form the SSO settings control across 4 layers: 1. Feature Flag Definition \u2014 the registry where `ssoSettingsApi` is defined and generated constants. 2. SSO Settings Infrastructure \u2014 Service interface, Reloadable interface, SSOSettings data model, and SSOSettingsStore database layer. 3. API and Authentication Wiring \u2014 REST API endpoint registration with access control middleware, SocialService provider, and authentication client registration. 4. Access Control and DI Registration \u2014 access control evaluators and the ProvideService dependency injection function.",
+    "constraints": [
+      "Provide specific file paths and repository names in your answer.",
+      "Write your findings to /workspace/answer.json."
+    ],
     "success_definition": "Agent successfully identifies relevant files and symbols across all repos in the grafana-observability fixture.",
-    "seed_prompt": "Find ALL files in `grafana/grafana` that form the SSO settings control across 4 layers: 1. Feature Flag Definition — the registry where `ssoSettingsApi` is defined and generated constants. 2. SSO Settings Infrastructure — Service interface, Reloadable interface, SSOSettings data model, and SSOSettingsStore database layer. 3. API and Authentication Wiring — REST API endpoint registration with access control middleware, SocialService provider, and authentication client registration. 4. Access Control and DI Registration — access control evaluators and the ProvideService dependency injection function."
+    "seed_prompt": "Find ALL files in `grafana/grafana` that form the SSO settings control across 4 layers: 1. Feature Flag Definition \u2014 the registry where `ssoSettingsApi` is defined and generated constants. 2. SSO Settings Infrastructure \u2014 Service interface, Reloadable interface, SSOSettings data model, and SSOSettingsStore database layer. 3. API and Authentication Wiring \u2014 REST API endpoint registration with access control middleware, SocialService provider, and authentication client registration. 4. Access Control and DI Registration \u2014 access control evaluators and the ProvideService dependency injection function."
   },
   "artifacts": {
     "repo_set_id": "grafana-observability",
@@ -20,20 +23,26 @@
     }
   },
   "evaluation": {
-    "modes": ["deterministic"],
+    "modes": [
+      "deterministic"
+    ],
     "checks": [
-  {
-    "type": "file_set_match",
-    "params": {
-      "search_pattern": "",
-      "file_filter": ""
-    }
-  }
-],
+      {
+        "type": "file_set_match",
+        "params": {
+          "search_pattern": "SocialService OR ProvideService",
+          "file_filter": ""
+        }
+      }
+    ],
     "eval_script": "/tests/eval.sh",
     "pass_exit_code": 0
   },
   "logging": {
-    "required_metrics": ["oracle_coverage", "time_to_first_oracle_hit_ms", "unique_repos_touched"]
+    "required_metrics": [
+      "oracle_coverage",
+      "time_to_first_oracle_hit_ms",
+      "unique_repos_touched"
+    ]
   }
 }

benchmarks/ccb_mcp_compliance/ccx-compliance-115/tests/task_spec.json

@@ -56,7 +56,7 @@
       {
         "type": "file_set_match",
         "params": {
-          "search_pattern": "",
+          "search_pattern": "Compliance OR Audit OR Django",
           "file_filter": ""
         }
       }

benchmarks/ccb_mcp_compliance/ccx-compliance-118/tests/task_spec.json

@@ -68,7 +68,7 @@
       {
         "type": "file_set_match",
         "params": {
-          "search_pattern": "",
+          "search_pattern": "RelatedFieldListFilter OR ForeignKey OR ListFilter OR ChangeList",
           "file_filter": ""
         }
       }

benchmarks/ccb_mcp_compliance/ccx-compliance-182/tests/task_spec.json

@@ -6,33 +6,54 @@
   "mcp_suite": "ccb_mcp_compliance",
   "prd": {
     "user_story": "As a developer, I want to: Audit kubernetes/kubernetes for completeness of RBAC verb handling. Find all Go source files that define the set of allowed verbs (get, list, watch, create, update, patch, delete, deletecollection) and validate that all API handlers enforce these verb constraints.",
-    "constraints": ["Provide specific file paths and repository names in your answer.", "Write your findings to /workspace/answer.json."],
+    "constraints": [
+      "Provide specific file paths and repository names in your answer.",
+      "Write your findings to /workspace/answer.json."
+    ],
     "success_definition": "Agent successfully identifies relevant files and symbols across all repos in the kubernetes-ecosystem fixture.",
     "seed_prompt": "Audit kubernetes/kubernetes for completeness of RBAC verb handling. Find all Go source files that define the set of allowed verbs (get, list, watch, create, update, patch, delete, deletecollection) and validate that all API handlers enforce these verb constraints."
   },
   "artifacts": {
     "repo_set_id": "kubernetes-ecosystem",
     "oracle": {
-      "required_files": [],
+      "required_files": [
+        "sg-evals/kubernetes--v1.32.0/pkg/apis/rbac/v1alpha1/zz_generated.conversion.go",
+        "sg-evals/kubernetes--v1.32.0/plugin/pkg/auth/authorizer/rbac/rbac_test.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/registry/rbac/clusterrole/policybased/storage_test.go",
+        "sg-evals/kubernetes--v1.32.0/test/integration/authutil/authutil.go",
+        "sg-evals/kubernetes--v1.32.0/test/e2e/storage/drivers/csi.go",
+        "sg-evals/kubernetes--v1.32.0/cmd/kubeadm/app/phases/addons/dns/dns_test.go",
+        "sg-evals/api--v0.32.0/rbac/v1/types_swagger_doc_generated.go",
+        "sg-evals/api--v0.32.0/rbac/v1beta1/types.go",
+        "sg-evals/api--v0.32.0/rbac/v1/generated.proto",
+        "sg-evals/api--v0.32.0/rbac/v1beta1/generated.proto",
+        "sg-evals/client-go--v0.32.0/applyconfigurations/internal/internal.go"
+      ],
       "required_symbols": [],
       "required_references": [],
       "dependency_chains": []
     }
   },
   "evaluation": {
-    "modes": ["deterministic"],
+    "modes": [
+      "deterministic"
+    ],
     "checks": [
-  {
-    "type": "file_set_match",
-    "params": {
-      "search_pattern": ""
-    }
-  }
-],
+      {
+        "type": "file_set_match",
+        "params": {
+          "search_pattern": "ClusterRole"
+        }
+      }
+    ],
     "eval_script": "/tests/eval.sh",
     "pass_exit_code": 0
   },
   "logging": {
-    "required_metrics": ["oracle_coverage", "time_to_first_oracle_hit_ms", "unique_repos_touched"]
+    "required_metrics": [
+      "oracle_coverage",
+      "time_to_first_oracle_hit_ms",
+      "unique_repos_touched"
+    ]
   }
 }

benchmarks/ccb_mcp_compliance/ccx-compliance-183/tests/task_spec.json

@@ -6,33 +6,55 @@
   "mcp_suite": "ccb_mcp_compliance",
   "prd": {
     "user_story": "As a developer, I want to: Find all Go source files in kubernetes/kubernetes that enforce the API deprecation policy: the deprecated API version warning injection, the removal version validation, and the minimum supported API version checks in the API server.",
-    "constraints": ["Provide specific file paths and repository names in your answer.", "Write your findings to /workspace/answer.json."],
+    "constraints": [
+      "Provide specific file paths and repository names in your answer.",
+      "Write your findings to /workspace/answer.json."
+    ],
     "success_definition": "Agent successfully identifies relevant files and symbols across all repos in the kubernetes-ecosystem fixture.",
     "seed_prompt": "Find all Go source files in kubernetes/kubernetes that enforce the API deprecation policy: the deprecated API version warning injection, the removal version validation, and the minimum supported API version checks in the API server."
   },
   "artifacts": {
     "repo_set_id": "kubernetes-ecosystem",
     "oracle": {
-      "required_files": [],
+      "required_files": [
+        "sg-evals/kubernetes--v1.32.0/staging/src/k8s.io/apiserver/pkg/endpoints/deprecation/deprecation.go",
+        "sg-evals/kubernetes--v1.32.0/staging/src/k8s.io/apiextensions-apiserver/test/integration/deprecation_test.go",
+        "sg-evals/kubernetes--v1.32.0/staging/src/k8s.io/apiserver/pkg/registry/rest/update.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/registry/core/node/strategy.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/registry/core/serviceaccount/strategy.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/registry/discovery/endpointslice/strategy.go",
+        "sg-evals/kubernetes--v1.32.0/cmd/kubeadm/app/util/config/upgradeconfiguration.go",
+        "sg-evals/kubernetes--v1.32.0/cmd/kubeadm/app/cmd/options/generic.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/api/pod/warnings.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/api/service/warnings.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/api/persistentvolumeclaim/util.go",
+        "sg-evals/etcd-io-etcd/server/embed/config.go"
+      ],
       "required_symbols": [],
       "required_references": [],
       "dependency_chains": []
     }
   },
   "evaluation": {
-    "modes": ["deterministic"],
+    "modes": [
+      "deterministic"
+    ],
     "checks": [
-  {
-    "type": "file_set_match",
-    "params": {
-      "search_pattern": ""
-    }
-  }
-],
+      {
+        "type": "file_set_match",
+        "params": {
+          "search_pattern": "Kubernetes OR Deprecation OR Policy"
+        }
+      }
+    ],
     "eval_script": "/tests/eval.sh",
     "pass_exit_code": 0
   },
   "logging": {
-    "required_metrics": ["oracle_coverage", "time_to_first_oracle_hit_ms", "unique_repos_touched"]
+    "required_metrics": [
+      "oracle_coverage",
+      "time_to_first_oracle_hit_ms",
+      "unique_repos_touched"
+    ]
   }
 }

benchmarks/ccb_mcp_compliance/ccx-compliance-184/tests/task_spec.json

@@ -6,33 +6,56 @@
   "mcp_suite": "ccb_mcp_compliance",
   "prd": {
     "user_story": "As a developer, I want to: Find all Go source files in kubernetes/kubernetes that validate pod security context fields: the runAsNonRoot enforcement, the privileged container check, the capabilities validation, and the seccompProfile enforcement.",
-    "constraints": ["Provide specific file paths and repository names in your answer.", "Write your findings to /workspace/answer.json."],
+    "constraints": [
+      "Provide specific file paths and repository names in your answer.",
+      "Write your findings to /workspace/answer.json."
+    ],
     "success_definition": "Agent successfully identifies relevant files and symbols across all repos in the kubernetes-ecosystem fixture.",
     "seed_prompt": "Find all Go source files in kubernetes/kubernetes that validate pod security context fields: the runAsNonRoot enforcement, the privileged container check, the capabilities validation, and the seccompProfile enforcement."
   },
   "artifacts": {
     "repo_set_id": "kubernetes-ecosystem",
     "oracle": {
-      "required_files": [],
+      "required_files": [
+        "sg-evals/kubernetes--v1.32.0/pkg/apis/core/validation/validation.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/apis/core/validation/validation_test.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/apis/core/types.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/securitycontext/accessors.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/securitycontext/util.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/kubelet/kuberuntime/security_context_others.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/kubelet/kuberuntime/security_context_windows.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/kubelet/kuberuntime/security_context_others_test.go",
+        "sg-evals/kubernetes--v1.32.0/pkg/kubelet/kuberuntime/security_context_windows_test.go",
+        "sg-evals/kubernetes--v1.32.0/staging/src/k8s.io/pod-security-admission/policy/check_runAsNonRoot.go",
+        "sg-evals/kubernetes--v1.32.0/staging/src/k8s.io/pod-security-admission/test/fixtures_runAsNonRoot.go",
+        "sg-evals/kubernetes--v1.32.0/staging/src/k8s.io/api/core/v1/types.go",
+        "sg-evals/kubernetes--v1.32.0/cmd/kubelet/app/server.go"
+      ],
       "required_symbols": [],
       "required_references": [],
       "dependency_chains": []
     }
   },
   "evaluation": {
-    "modes": ["deterministic"],
+    "modes": [
+      "deterministic"
+    ],
     "checks": [
-  {
-    "type": "file_set_match",
-    "params": {
-      "search_pattern": ""
-    }
-  }
-],
+      {
+        "type": "file_set_match",
+        "params": {
+          "search_pattern": "Kubernetes OR Security OR Context"
+        }
+      }
+    ],
     "eval_script": "/tests/eval.sh",
     "pass_exit_code": 0
   },
   "logging": {
-    "required_metrics": ["oracle_coverage", "time_to_first_oracle_hit_ms", "unique_repos_touched"]
+    "required_metrics": [
+      "oracle_coverage",
+      "time_to_first_oracle_hit_ms",
+      "unique_repos_touched"
+    ]
   }
 }

benchmarks/ccb_mcp_compliance/ccx-compliance-185/tests/task_spec.json

@@ -6,33 +6,55 @@
   "mcp_suite": "ccb_mcp_compliance",
   "prd": {
     "user_story": "As a developer, I want to: Find all Go source files in prometheus/prometheus that enforce Prometheus metric naming conventions: the metric name validation (snake_case, unit suffix), the label name validation, and the help text presence check in the client registration.",
-    "constraints": ["Provide specific file paths and repository names in your answer.", "Write your findings to /workspace/answer.json."],
+    "constraints": [
+      "Provide specific file paths and repository names in your answer.",
+      "Write your findings to /workspace/answer.json."
+    ],
     "success_definition": "Agent successfully identifies relevant files and symbols across all repos in the prometheus-monitoring fixture.",
     "seed_prompt": "Find all Go source files in prometheus/prometheus that enforce Prometheus metric naming conventions: the metric name validation (snake_case, unit suffix), the label name validation, and the help text presence check in the client registration."
   },
   "artifacts": {
     "repo_set_id": "prometheus-monitoring",
     "oracle": {
-      "required_files": [],
+      "required_files": [
+        "prometheus/prometheus/config/config.go",
+        "prometheus/prometheus/config/config_test.go",
+        "prometheus/prometheus/scrape/scrape.go",
+        "prometheus/prometheus/scrape/scrape_test.go",
+        "prometheus/prometheus/scrape/manager_test.go",
+        "prometheus/prometheus/model/labels/labels_common.go",
+        "prometheus/prometheus/model/rulefmt/rulefmt.go",
+        "prometheus/prometheus/model/textparse/protobufparse.go",
+        "prometheus/prometheus/cmd/prometheus/main.go",
+        "prometheus/prometheus/storage/remote/codec.go",
+        "prometheus/prometheus/notifier/manager.go",
+        "prometheus/prometheus/web/ui/mantine-ui/src/promql/utils.ts"
+      ],
       "required_symbols": [],
       "required_references": [],
       "dependency_chains": []
     }
   },
   "evaluation": {
-    "modes": ["deterministic"],
+    "modes": [
+      "deterministic"
+    ],
     "checks": [
-  {
-    "type": "file_set_match",
-    "params": {
-      "search_pattern": ""
-    }
-  }
-],
+      {
+        "type": "file_set_match",
+        "params": {
+          "search_pattern": "Prometheus OR Metric OR Naming"
+        }
+      }
+    ],
     "eval_script": "/tests/eval.sh",
     "pass_exit_code": 0
   },
   "logging": {
-    "required_metrics": ["oracle_coverage", "time_to_first_oracle_hit_ms", "unique_repos_touched"]
+    "required_metrics": [
+      "oracle_coverage",
+      "time_to_first_oracle_hit_ms",
+      "unique_repos_touched"
+    ]
   }
 }