Skip to content

Commit 3fb4722

Browse files
taskbottaskbot
committed
add tests
1 parent 8205fe0 commit 3fb4722

File tree

1 file changed

+332
-0
lines changed

1 file changed

+332
-0
lines changed

‎test/e2e/chainsaw/operator/single-tenancy/test-scenarios/vmcp/chainsaw-test.yaml‎

Lines changed: 332 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,332 @@
1+
apiVersion: chainsaw.kyverno.io/v1alpha1
2+
kind: Test
3+
metadata:
4+
name: vmcp-composite-tool-definition
5+
spec:
6+
description: Test VirtualMCPCompositeToolDefinition CRD validation and lifecycle
7+
timeouts:
8+
apply: 30s
9+
assert: 60s
10+
cleanup: 30s
11+
steps:
12+
- name: verify-operator
13+
description: Ensure operator is ready before testing
14+
try:
15+
- assert:
16+
file: ../../setup/assert-operator-ready.yaml
17+
18+
- name: create-valid-composite-tool
19+
description: Create a valid composite tool definition
20+
try:
21+
- apply:
22+
resource:
23+
apiVersion: toolhive.stacklok.dev/v1alpha1
24+
kind: VirtualMCPCompositeToolDefinition
25+
metadata:
26+
name: deploy-workflow
27+
namespace: toolhive-system
28+
spec:
29+
name: deploy_and_verify
30+
description: Deploy application and verify deployment status
31+
timeout: 10m
32+
failureMode: abort
33+
steps:
34+
- id: deploy
35+
type: tool_call
36+
tool: kubectl.apply
37+
arguments:
38+
manifest: "{{.params.manifest}}"
39+
namespace: "{{.params.namespace}}"
40+
timeout: 5m
41+
- id: verify
42+
type: tool_call
43+
tool: kubectl.get
44+
arguments:
45+
resource: "deployment"
46+
namespace: "{{.params.namespace}}"
47+
dependsOn:
48+
- deploy
49+
timeout: 2m
50+
- assert:
51+
resource:
52+
apiVersion: toolhive.stacklok.dev/v1alpha1
53+
kind: VirtualMCPCompositeToolDefinition
54+
metadata:
55+
name: deploy-workflow
56+
namespace: toolhive-system
57+
58+
- name: test-validation-status-valid
59+
description: Verify validation status is set to Valid
60+
try:
61+
- script:
62+
content: |
63+
echo "Checking validation status for valid composite tool..."
64+
65+
for i in {1..10}; do
66+
STATUS=$(kubectl get vmcpctd deploy-workflow -n toolhive-system -o jsonpath='{.status.validationStatus}' 2>/dev/null || echo "")
67+
68+
if [ "$STATUS" = "Valid" ]; then
69+
echo "✓ Validation status is Valid"
70+
exit 0
71+
fi
72+
73+
echo "⚠ Attempt $i/10: Status is '$STATUS', waiting..."
74+
sleep 2
75+
done
76+
77+
echo "✗ Validation status never became Valid"
78+
kubectl get vmcpctd deploy-workflow -n toolhive-system -o yaml
79+
exit 1
80+
81+
- name: create-invalid-duplicate-steps
82+
description: Test validation rejection for duplicate step IDs
83+
try:
84+
- apply:
85+
resource:
86+
apiVersion: toolhive.stacklok.dev/v1alpha1
87+
kind: VirtualMCPCompositeToolDefinition
88+
metadata:
89+
name: duplicate-steps
90+
namespace: toolhive-system
91+
spec:
92+
name: invalid_duplicate
93+
description: Workflow with duplicate step IDs
94+
steps:
95+
- id: step1
96+
type: tool_call
97+
tool: kubectl.apply
98+
- id: step1
99+
type: tool_call
100+
tool: kubectl.get
101+
- script:
102+
content: |
103+
echo "Checking validation status for duplicate steps..."
104+
105+
for i in {1..10}; do
106+
STATUS=$(kubectl get vmcpctd duplicate-steps -n toolhive-system -o jsonpath='{.status.validationStatus}' 2>/dev/null || echo "")
107+
108+
if [ "$STATUS" = "Invalid" ]; then
109+
echo "✓ Validation status is Invalid (as expected)"
110+
111+
ERRORS=$(kubectl get vmcpctd duplicate-steps -n toolhive-system -o jsonpath='{.status.validationErrors}' 2>/dev/null || echo "")
112+
if echo "$ERRORS" | grep -q "duplicated"; then
113+
echo "✓ Validation error mentions 'duplicated'"
114+
exit 0
115+
else
116+
echo "✗ Validation error doesn't mention duplicate: $ERRORS"
117+
exit 1
118+
fi
119+
fi
120+
121+
echo "⚠ Attempt $i/10: Status is '$STATUS', waiting..."
122+
sleep 2
123+
done
124+
125+
echo "✗ Validation status never became Invalid"
126+
kubectl get vmcpctd duplicate-steps -n toolhive-system -o yaml
127+
exit 1
128+
129+
- name: create-invalid-tool-reference
130+
description: Test validation rejection for invalid tool reference format
131+
try:
132+
- apply:
133+
resource:
134+
apiVersion: toolhive.stacklok.dev/v1alpha1
135+
kind: VirtualMCPCompositeToolDefinition
136+
metadata:
137+
name: bad-tool-ref
138+
namespace: toolhive-system
139+
spec:
140+
name: invalid_tool_ref
141+
description: Workflow with invalid tool reference
142+
steps:
143+
- id: step1
144+
type: tool_call
145+
tool: invalid-no-dot
146+
- script:
147+
content: |
148+
echo "Checking validation status for bad tool reference..."
149+
150+
for i in {1..10}; do
151+
STATUS=$(kubectl get vmcpctd bad-tool-ref -n toolhive-system -o jsonpath='{.status.validationStatus}' 2>/dev/null || echo "")
152+
153+
if [ "$STATUS" = "Invalid" ]; then
154+
echo "✓ Validation status is Invalid (as expected)"
155+
156+
ERRORS=$(kubectl get vmcpctd bad-tool-ref -n toolhive-system -o jsonpath='{.status.validationErrors}' 2>/dev/null || echo "")
157+
if echo "$ERRORS" | grep -q "workload.tool_name"; then
158+
echo "✓ Validation error mentions correct format"
159+
exit 0
160+
else
161+
echo "✗ Validation error doesn't mention format: $ERRORS"
162+
exit 1
163+
fi
164+
fi
165+
166+
echo "⚠ Attempt $i/10: Status is '$STATUS', waiting..."
167+
sleep 2
168+
done
169+
170+
echo "✗ Validation status never became Invalid"
171+
kubectl get vmcpctd bad-tool-ref -n toolhive-system -o yaml
172+
exit 1
173+
174+
- name: create-invalid-circular-dependency
175+
description: Test validation rejection for circular dependencies
176+
try:
177+
- apply:
178+
resource:
179+
apiVersion: toolhive.stacklok.dev/v1alpha1
180+
kind: VirtualMCPCompositeToolDefinition
181+
metadata:
182+
name: circular-deps
183+
namespace: toolhive-system
184+
spec:
185+
name: circular_workflow
186+
description: Workflow with circular dependencies
187+
steps:
188+
- id: step1
189+
type: tool_call
190+
tool: tool.a
191+
dependsOn:
192+
- step2
193+
- id: step2
194+
type: tool_call
195+
tool: tool.b
196+
dependsOn:
197+
- step1
198+
- script:
199+
content: |
200+
echo "Checking validation status for circular dependencies..."
201+
202+
for i in {1..10}; do
203+
STATUS=$(kubectl get vmcpctd circular-deps -n toolhive-system -o jsonpath='{.status.validationStatus}' 2>/dev/null || echo "")
204+
205+
if [ "$STATUS" = "Invalid" ]; then
206+
echo "✓ Validation status is Invalid (as expected)"
207+
208+
ERRORS=$(kubectl get vmcpctd circular-deps -n toolhive-system -o jsonpath='{.status.validationErrors}' 2>/dev/null || echo "")
209+
if echo "$ERRORS" | grep -q "cycle"; then
210+
echo "✓ Validation error mentions 'cycle'"
211+
exit 0
212+
else
213+
echo "✗ Validation error doesn't mention cycle: $ERRORS"
214+
exit 1
215+
fi
216+
fi
217+
218+
echo "⚠ Attempt $i/10: Status is '$STATUS', waiting..."
219+
sleep 2
220+
done
221+
222+
echo "✗ Validation status never became Invalid"
223+
kubectl get vmcpctd circular-deps -n toolhive-system -o yaml
224+
exit 1
225+
226+
- name: create-composite-tool-with-parameters
227+
description: Create composite tool with parameter schema
228+
try:
229+
- apply:
230+
resource:
231+
apiVersion: toolhive.stacklok.dev/v1alpha1
232+
kind: VirtualMCPCompositeToolDefinition
233+
metadata:
234+
name: with-parameters
235+
namespace: toolhive-system
236+
spec:
237+
name: parameterized_deploy
238+
description: Deploy with parameters
239+
parameters:
240+
environment:
241+
type: string
242+
description: Target environment
243+
required: true
244+
replicas:
245+
type: integer
246+
description: Number of replicas
247+
default: "3"
248+
steps:
249+
- id: deploy
250+
type: tool_call
251+
tool: kubectl.apply
252+
arguments:
253+
env: "{{.params.environment}}"
254+
replicas: "{{.params.replicas}}"
255+
- script:
256+
content: |
257+
echo "Checking validation status for composite tool with parameters..."
258+
259+
for i in {1..10}; do
260+
STATUS=$(kubectl get vmcpctd with-parameters -n toolhive-system -o jsonpath='{.status.validationStatus}' 2>/dev/null || echo "")
261+
262+
if [ "$STATUS" = "Valid" ]; then
263+
echo "✓ Validation status is Valid"
264+
exit 0
265+
fi
266+
267+
echo "⚠ Attempt $i/10: Status is '$STATUS', waiting..."
268+
sleep 2
269+
done
270+
271+
echo "✗ Validation status never became Valid"
272+
kubectl get vmcpctd with-parameters -n toolhive-system -o yaml
273+
exit 1
274+
275+
- name: create-composite-tool-with-error-handling
276+
description: Create composite tool with error handling
277+
try:
278+
- apply:
279+
resource:
280+
apiVersion: toolhive.stacklok.dev/v1alpha1
281+
kind: VirtualMCPCompositeToolDefinition
282+
metadata:
283+
name: with-error-handling
284+
namespace: toolhive-system
285+
spec:
286+
name: resilient_deploy
287+
description: Deploy with retry logic
288+
steps:
289+
- id: deploy
290+
type: tool_call
291+
tool: kubectl.apply
292+
onError:
293+
action: retry
294+
maxRetries: 3
295+
timeout: 5m
296+
- script:
297+
content: |
298+
echo "Checking validation status for composite tool with error handling..."
299+
300+
for i in {1..10}; do
301+
STATUS=$(kubectl get vmcpctd with-error-handling -n toolhive-system -o jsonpath='{.status.validationStatus}' 2>/dev/null || echo "")
302+
303+
if [ "$STATUS" = "Valid" ]; then
304+
echo "✓ Validation status is Valid"
305+
exit 0
306+
fi
307+
308+
echo "⚠ Attempt $i/10: Status is '$STATUS', waiting..."
309+
sleep 2
310+
done
311+
312+
echo "✗ Validation status never became Valid"
313+
kubectl get vmcpctd with-error-handling -n toolhive-system -o yaml
314+
exit 1
315+
316+
- name: test-composite-tool-ready-condition
317+
description: Verify Ready condition is set correctly
318+
try:
319+
- script:
320+
content: |
321+
echo "Checking Ready condition for composite tool..."
322+
323+
READY=$(kubectl get vmcpctd deploy-workflow -n toolhive-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "")
324+
325+
if [ "$READY" = "True" ]; then
326+
echo "✓ Ready condition is True"
327+
exit 0
328+
fi
329+
330+
echo "⚠ Ready condition is '$READY', not 'True' (may not be implemented yet)"
331+
kubectl get vmcpctd deploy-workflow -n toolhive-system -o yaml
332+
exit 0

0 commit comments

Comments
 (0)