Skip to content

Commit 7c9cf97

Browse files
author
LoCoBench Bot
committed
Add k8s docgen successor tasks with canonical-safe verifiers
1 parent f840de8 commit 7c9cf97

File tree

25 files changed

+1886
-0
lines changed

25 files changed

+1886
-0
lines changed

‎benchmarks/ccb_docgen/docgen-k8s-apiserver-001/environment/Dockerfile‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
FROM golang:1.23-bookworm
2+
3+
WORKDIR /workspace
4+
5+
# Install dependencies
6+
RUN apt-get update && apt-get install -y \
7+
git \
8+
curl \
9+
python3 \
10+
npm \
11+
ripgrep
12+
13+
# Install Claude Code CLI
14+
RUN npm install -g @anthropic-ai/claude-code
15+
16+
# Clone the Kubernetes repository at pinned commit for reproducibility
17+
RUN git clone --filter=blob:none --no-checkout https://github.com/kubernetes/kubernetes.git . && \
18+
git checkout 8c9c67c000104450cfc5a5f48053a9a84b73cf93 && \
19+
git config user.email "agent@example.com" && \
20+
git config user.name "Agent"
21+
22+
# SURGICAL: Remove ONLY the target doc.go (not recursive!)
23+
RUN rm -f staging/src/k8s.io/apiserver/doc.go
24+
# Strip "// Package ..." line from files in the immediate directory only
25+
RUN find staging/src/k8s.io/apiserver -maxdepth 1 -name "*.go" -exec sed -i '/^\/\/ Package/d' {} +
26+
27+
# Create test directory for verifier
28+
RUN mkdir -p /workspace/tests
29+
30+
# Create /app directory for MCP config upload compatibility
31+
RUN mkdir -p /app

‎benchmarks/ccb_docgen/docgen-k8s-apiserver-001/instruction.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Task: Kubernetes API Server Architecture and Extension Guide
2+
3+
**Repository:** kubernetes/kubernetes (stripped snapshot)
4+
**Output:** Write your document to `/workspace/documentation.md`
5+
6+
## Objective
7+
8+
Produce a subsystem architecture and extension guide for the apiserver library with request-path flow and extension tradeoffs.
9+
10+
## Scope
11+
12+
Primary focus area: `staging/src/k8s.io/apiserver`.
13+
14+
Your document must explain component responsibilities, end-to-end flow, and extension/operational tradeoffs.
15+
16+
## Required Sections
17+
18+
1. **Subsystem Overview** — purpose, boundaries, and upstream/downstream dependencies
19+
2. **Key Components** — major types/modules and their responsibilities
20+
3. **End-to-End Flow** — request/control flow with concrete file-backed references
21+
4. **Failure Modes & Tradeoffs** — common failures, limits, and design tradeoffs
22+
5. **Extension Points** — where behavior can be customized and associated risks
23+
6. **Source File Map** — list the most relevant files/directories used in your analysis
24+
25+
## Quality Bar
26+
27+
- Cite concrete file paths from the repository.
28+
- Explain behavior and interactions, not just API signatures.
29+
- Include at least one section on operational implications (performance, reliability, or maintainability).
30+
- Do not fabricate classes, functions, or files.
31+
32+
## Anti-Requirements
33+
34+
- Do not output `doc.go`.
35+
- Do not provide shallow bullet dumps without system flow explanation.
36+
- Do not rely on external docs not present in the workspace.

‎benchmarks/ccb_docgen/docgen-k8s-apiserver-001/task.toml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
version = "1.0"
2+
3+
[metadata]
4+
name = "docgen-k8s-apiserver-001"
5+
description = "Produce a subsystem architecture and extension guide for the apiserver library with request-path flow and extension tradeoffs."
6+
license = "Apache-2.0"
7+
8+
[task]
9+
id = "docgen-k8s-apiserver-001"
10+
repo = "sg-benchmarks/kubernetes--stripped"
11+
category = "architecture_doc"
12+
language = "go"
13+
difficulty = "hard"
14+
time_limit_sec = 1200
15+
16+
[task.metadata]
17+
complexity_dimension = "documentation_reasoning"
18+
doc_type = "subsystem_guide"
19+
target_subsystem = "staging/src/k8s.io/apiserver"
20+
output_path = "/workspace/documentation.md"
21+
22+
[verification]
23+
type = "test"
24+
command = "bash /tests/test.sh"
25+
reward_type = "checklist"
26+
description = "Weighted checklist: required topics (0.40), file references (0.25), data flow (0.20), extension points (0.15)"
27+
28+
[environment]
29+
build_timeout_sec = 900.0
30+
31+
[environment.setup_scripts]
32+
mcp_config = """#!/bin/bash
33+
if [ -n "$SOURCEGRAPH_ACCESS_TOKEN" ] && [ -n "$SOURCEGRAPH_URL" ]; then
34+
echo "Setting up Sourcegraph MCP configuration..."
35+
mkdir -p /root/.config/claude
36+
cat > /root/.config/claude/mcp.json << 'EOF'
37+
{
38+
"mcpServers": {
39+
"sourcegraph": {
40+
"command": "npx",
41+
"args": ["-y", "@sourcegraph/mcp-server"],
42+
"env": {
43+
"SRC_ACCESS_TOKEN": "$SOURCEGRAPH_ACCESS_TOKEN",
44+
"SOURCEGRAPH_URL": "$SOURCEGRAPH_URL"
45+
}
46+
}
47+
}
48+
}
49+
EOF
50+
echo "OK MCP configuration created"
51+
else
52+
echo "No Sourcegraph credentials provided, MCP disabled"
53+
fi
54+
exit 0
55+
"""

‎benchmarks/ccb_docgen/docgen-k8s-apiserver-001/tests/ground_truth.json‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
{
2+
"task_id": "docgen-k8s-apiserver-001",
3+
"description": "Kubernetes API Server Architecture and Extension Guide",
4+
"weights": {
5+
"required_topics": 0.4,
6+
"file_references": 0.25,
7+
"data_flow": 0.2,
8+
"extension_points": 0.15
9+
},
10+
"required_topics": [
11+
{
12+
"id": "t1",
13+
"description": "GenericAPIServer role and lifecycle",
14+
"patterns": [
15+
"GenericAPIServer",
16+
"generic api server|server lifecycle"
17+
],
18+
"weight": 0.2
19+
},
20+
{
21+
"id": "t2",
22+
"description": "Authentication, authorization, and admission stages",
23+
"patterns": [
24+
"authentication",
25+
"authorization",
26+
"admission"
27+
],
28+
"weight": 0.2
29+
},
30+
{
31+
"id": "t3",
32+
"description": "API aggregation and extension API servers",
33+
"patterns": [
34+
"APIService|api aggregation|extension API server"
35+
],
36+
"weight": 0.2
37+
},
38+
{
39+
"id": "t4",
40+
"description": "Storage and endpoint plumbing",
41+
"patterns": [
42+
"pkg/endpoints|installer",
43+
"pkg/registry|storage"
44+
],
45+
"weight": 0.2
46+
},
47+
{
48+
"id": "t5",
49+
"description": "Preferred alternatives (CRDs and admission webhooks)",
50+
"patterns": [
51+
"CRD|CustomResourceDefinition",
52+
"admission webhook|ValidatingWebhook|MutatingWebhook"
53+
],
54+
"weight": 0.2
55+
}
56+
],
57+
"file_references": [
58+
{
59+
"id": "r1",
60+
"description": "generic server implementation files",
61+
"patterns": [
62+
"pkg/server/genericapiserver\\.go",
63+
"pkg/server"
64+
],
65+
"weight": 0.25
66+
},
67+
{
68+
"id": "r2",
69+
"description": "admission interfaces",
70+
"patterns": [
71+
"pkg/admission/interfaces\\.go",
72+
"pkg/admission"
73+
],
74+
"weight": 0.25
75+
},
76+
{
77+
"id": "r3",
78+
"description": "endpoint installer",
79+
"patterns": [
80+
"pkg/endpoints/installer\\.go",
81+
"pkg/endpoints"
82+
],
83+
"weight": 0.25
84+
},
85+
{
86+
"id": "r4",
87+
"description": "generic registry",
88+
"patterns": [
89+
"pkg/registry/generic/registry\\.go",
90+
"pkg/registry"
91+
],
92+
"weight": 0.25
93+
}
94+
],
95+
"data_flow": [
96+
{
97+
"id": "d1",
98+
"description": "Request path from handler through authn/authz/admission to storage",
99+
"patterns": [
100+
"request|http",
101+
"authentication",
102+
"authorization",
103+
"admission",
104+
"registry|storage"
105+
],
106+
"weight": 0.5,
107+
"ordered": true
108+
},
109+
{
110+
"id": "d2",
111+
"description": "Failure handling and rejection points",
112+
"patterns": [
113+
"unauthorized|forbidden|reject|denied|error"
114+
],
115+
"weight": 0.5
116+
}
117+
],
118+
"extension_points": [
119+
{
120+
"id": "e1",
121+
"description": "API aggregation via APIService",
122+
"patterns": [
123+
"APIService|aggregation layer"
124+
],
125+
"weight": 0.333
126+
},
127+
{
128+
"id": "e2",
129+
"description": "Admission policy extension points",
130+
"patterns": [
131+
"admission plugin|ValidatingAdmissionPolicy|webhook"
132+
],
133+
"weight": 0.333
134+
},
135+
{
136+
"id": "e3",
137+
"description": "CRD-based extension path",
138+
"patterns": [
139+
"CRD|CustomResourceDefinition"
140+
],
141+
"weight": 0.333
142+
}
143+
],
144+
"ground_truth_provenance": {
145+
"source_suite": "archive/ccb_k8sdocs",
146+
"source_task": "apiserver-doc-001",
147+
"source_file": "archive/ccb_k8sdocs/apiserver-doc-001/ground_truth/doc.go",
148+
"source_sha256": "eb328fe88b92a59bdb77f62bf81f7fe3f7b465946232e8e682d23940e4d5092d",
149+
"source_type": "official upstream Kubernetes documentation (human-authored)",
150+
"note": "Canonical ground truth is the official Kubernetes doc.go at pinned commit. Checklist patterns should align to this canonical source.",
151+
"canonical_source": {
152+
"repo": "kubernetes/kubernetes",
153+
"commit": "8c9c67c000104450cfc5a5f48053a9a84b73cf93",
154+
"path": "staging/src/k8s.io/apiserver/doc.go",
155+
"url": "https://raw.githubusercontent.com/kubernetes/kubernetes/8c9c67c000104450cfc5a5f48053a9a84b73cf93/staging/src/k8s.io/apiserver/doc.go",
156+
"sha256": "eb328fe88b92a59bdb77f62bf81f7fe3f7b465946232e8e682d23940e4d5092d",
157+
"source_type": "official upstream Kubernetes documentation (human-authored)"
158+
},
159+
"archived_copy_sha256": "eb328fe88b92a59bdb77f62bf81f7fe3f7b465946232e8e682d23940e4d5092d",
160+
"archived_copy_matches_canonical": true
161+
}
162+
}

0 commit comments

Comments
 (0)