diff --git a/api/mcp.go b/api/mcp.go index 504de608..b379a25a 100644 --- a/api/mcp.go +++ b/api/mcp.go @@ -23,6 +23,17 @@ import ( "strings" ) +// JSON-RPC error codes AgentMesh itself returns on an MCP stream, as opposed +// to errors from the MCP backend it forwards. JSON-RPC 2.0 reserves -32768 to +// -32000 for the protocol, so these are outside that range. +const ( + // JSONRPCCodeUnauthorized answers a request the caller's credential does not + // authorize: the caller reached the provider and was identified, but the task + // in its biscuit does not allow this method or tool. The same request under + // the same task is denied again. + JSONRPCCodeUnauthorized = 1001 +) + // DefaultMaxMCPBodyInspectBytes is the default maximum HTTP request body size // buffered when inspecting an MCP JSON-RPC 2.0 payload for tool-level policy // evaluation (4 MiB). diff --git a/internal/node/mcp.go b/internal/node/mcp.go index dca899bf..b8e0ea5f 100644 --- a/internal/node/mcp.go +++ b/internal/node/mcp.go @@ -199,8 +199,41 @@ func NewMCPHandler(node *AgentMeshNode) http.Handler { // requiredLabels, when non-empty, fail-closed verifies the peer's control-plane-attested // labels (see checkPeerLabels) before the tool is invoked; nil means no caller // requirement, though the operator's egress floor, if configured, is still enforced. +// +// Reaching the peer is retried (see connectMCPSessionWithRetry); tools/call is +// sent once. After it is sent the provider may have run the tool, so whatever +// the error, only the caller knows whether running it again is safe. func (n *AgentMeshNode) CallMCPTool(ctx context.Context, targetPeer peer.ID, toolName string, params any, requiredLabels map[string]string) (*mcp.CallToolResult, error) { - var res *mcp.CallToolResult + targetService, originalToolName, err := api.SplitToolName(toolName) + if err != nil { + return nil, err + } + + session, cleanup, err := n.connectMCPSessionWithRetry(ctx, targetPeer, targetService, requiredLabels) + if err != nil { + return nil, err + } + defer cleanup() + + // We don't need to marshall the params, the SDK takes care of it + // Passing pre-marshaled []byte triggers encoding/json's base64 which gets rejected + callParams := &mcp.CallToolParams{ + Name: originalToolName, + Arguments: params, + } + + res, err := session.CallTool(ctx, callParams) + if err != nil { + return nil, fmt.Errorf("failed to call tool %s: %w", toolName, err) + } + + return res, nil +} + +// connectMCPSessionWithRetry calls ConnectMCPSession up to three times, with +// backoff, for failures to reach the peer: dial, relay and handshake errors. +// Nothing has been sent to the MCP backend yet, so a retry cannot repeat work. +func (n *AgentMeshNode) connectMCPSessionWithRetry(ctx context.Context, targetPeer peer.ID, targetService string, requiredLabels map[string]string) (*mcp.ClientSession, func(), error) { var err error maxRetries := 3 backoff := 1 * time.Second @@ -209,39 +242,43 @@ func (n *AgentMeshNode) CallMCPTool(ctx context.Context, targetPeer peer.ID, too n.preparePeerAddrs(ctx, targetPeer) for i := 0; i < maxRetries; i++ { - res, err = n.callMCPToolOnce(ctx, targetPeer, toolName, params, requiredLabels) - if err == nil { - return res, nil + session, cleanup, connErr := n.ConnectMCPSession(ctx, targetPeer, targetService, requiredLabels) + if connErr == nil { + return session, cleanup, nil } + err = connErr if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || strings.Contains(err.Error(), "context deadline exceeded") { - logger.Warnf("[MCP] Tool call failed with timeout or cancellation, not retrying: %v", err) - return nil, err + logger.Warnf("[MCP] Connecting to %s failed with timeout or cancellation, not retrying: %v", targetPeer, err) + return nil, nil, err } // A denial is an answer, not a failure to reach anyone. Retrying it // just delays the same reply and knocks on the remote peer three times // for one call. That holds for the remote peer refusing the caller // and for the caller refusing a provider not granted the service. if errors.Is(err, ErrAuthRejected) || errors.Is(err, ErrProviderNotGranted) { - logger.Warnf("[MCP] Tool call denied, not retrying: %v", err) - return nil, err + logger.Warnf("[MCP] Connecting to %s denied, not retrying: %v", targetPeer, err) + return nil, nil, err } // An authorizer that stopped at its facts or iterations limit stops // there again on a retry; only its time limit depends on load. if stopped, transient := authorizerLimit(err); stopped && !transient { - logger.Warnf("[MCP] Tool call stopped at an authorizer limit, not retrying: %v", err) - return nil, err + logger.Warnf("[MCP] Connecting to %s stopped at an authorizer limit, not retrying: %v", targetPeer, err) + return nil, nil, err } - logger.Warnf("[MCP] Tool call failed, retrying in %v: %v", backoff, err) + if i == maxRetries-1 { + break + } + logger.Warnf("[MCP] Connecting to %s failed, retrying in %v: %v", targetPeer, backoff, err) timer := time.NewTimer(backoff) select { case <-ctx.Done(): timer.Stop() - return nil, fmt.Errorf("context canceled during retry: %w", ctx.Err()) + return nil, nil, fmt.Errorf("context canceled during retry: %w", ctx.Err()) case <-timer.C: } backoff *= 2 } - return nil, fmt.Errorf("failed after %d retries: %w", maxRetries, err) + return nil, nil, fmt.Errorf("failed after %d retries: %w", maxRetries, err) } // ErrAuthRejected marks a service-level authorization denial from a remote peer, @@ -376,33 +413,6 @@ func (n *AgentMeshNode) ConnectMCPSession(ctx context.Context, targetPeer peer.I return session, fullCleanup, nil } -func (n *AgentMeshNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolName string, params any, requiredLabels map[string]string) (*mcp.CallToolResult, error) { - targetService, originalToolName, err := api.SplitToolName(toolName) - if err != nil { - return nil, err - } - - session, cleanup, err := n.ConnectMCPSession(ctx, targetPeer, targetService, requiredLabels) - if err != nil { - return nil, err - } - defer cleanup() - - // We don't need to marshall the params, the SDK takes care of it - // Passing pre-marshaled []byte triggers encoding/json's base64 which gets rejected - callParams := &mcp.CallToolParams{ - Name: originalToolName, - Arguments: params, - } - - res, err := session.CallTool(ctx, callParams) - if err != nil { - return nil, fmt.Errorf("failed to call tool %s: %w", toolName, err) - } - - return res, nil -} - // fetchRemoteServiceCatalog calls the remote peer's list_local_services // MCP tool with the type filter and returns the parsed catalog. func (n *AgentMeshNode) fetchRemoteServiceCatalog(ctx context.Context, peerID peer.ID, typeStr string) ([]*api.ServiceInfo, error) { diff --git a/internal/node/mcp_handlers_test.go b/internal/node/mcp_handlers_test.go index e5234ab7..015ea9ca 100644 --- a/internal/node/mcp_handlers_test.go +++ b/internal/node/mcp_handlers_test.go @@ -24,6 +24,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -33,6 +34,7 @@ import ( "github.com/google/agentmesh/internal/identity" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + "github.com/modelcontextprotocol/go-sdk/jsonrpc" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -687,6 +689,140 @@ func TestCallMCPTool_ProviderNotGrantedIsNotRetried(t *testing.T) { } } +// tools/call is sent once. Here the backend runs the tool and the connection +// drops before the reply comes back; the error goes to the caller, and the tool +// is not run a second time. +func TestCallMCPTool_ToolCallIsSentOnce(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + nodeA, cleanupA := startBareNode(t, ctx) + defer cleanupA() + nodeB, cleanupB := startBareNode(t, ctx) + defer cleanupB() + + if err := nodeA.Host.Connect(ctx, peer.AddrInfo{ID: nodeB.Host.ID(), Addrs: nodeB.Host.Addrs()}); err != nil { + t.Fatalf("connect: %v", err) + } + _, rootPriv := enrollUnderRoot(t, nodeA, nodeB) + nodeBIdentity, err := identity.MintBootstrapBiscuitToken(rootPriv, nodeB.Host.ID(), api.RoleNode, time.Now().Add(time.Hour), publishEverything, nil) + if err != nil { + t.Fatalf("mint nodeB identity: %v", err) + } + nodeB.SetIdentityCache(nodeBIdentity) + + var calls atomic.Int32 + fake := newFakeMCPHandler(t, []*mcp.Tool{ + {Name: "send_payment", Description: "Send a payment", InputSchema: map[string]any{"type": "object"}}, + }) + hostedSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read request body: %v", err) + return + } + if strings.Contains(string(body), `"tools/call"`) { + calls.Add(1) + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijack: %v", err) + return + } + _ = conn.Close() + return + } + r.Body = io.NopCloser(strings.NewReader(string(body))) + fake.ServeHTTP(w, r) + })) + defer hostedSrv.Close() + if err := nodeB.RegisterService(ctx, &api.RegisterServiceRequest{ + Service: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "payments"}, + Backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: hostedSrv.URL}, + }); err != nil { + t.Fatalf("RegisterService: %v", err) + } + defer func() { _ = nodeB.UnregisterService(ctx, "payments") }() + + _, err = nodeA.CallMCPTool(ctx, nodeB.Host.ID(), "mcp://payments/send_payment", map[string]any{"amount": 10}, nil) + if err == nil { + t.Fatal("CallMCPTool with the reply lost = nil error, want an error") + } + if got := calls.Load(); got != 1 { + t.Fatalf("backend received tools/call %d times, want 1", got) + } + if strings.Contains(err.Error(), "retries") { + t.Fatalf("CallMCPTool with the reply lost = %v, want one attempt with no retry", err) + } +} + +// A provider whose task check denies a tool outside the caller's task answers +// tools/call itself, with api.JSONRPCCodeUnauthorized. That answer goes back +// once, and the same task still reaches the tool it allows. +func TestCallMCPTool_TaskDeniedToolIsNotRetried(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + nodeA, cleanupA := startBareNode(t, ctx) + defer cleanupA() + nodeB, cleanupB := startBareNode(t, ctx) + defer cleanupB() + + if err := nodeA.Host.Connect(ctx, peer.AddrInfo{ID: nodeB.Host.ID(), Addrs: nodeB.Host.Addrs()}); err != nil { + t.Fatalf("connect: %v", err) + } + _, rootPriv := enrollUnderRoot(t, nodeA, nodeB) + nodeBIdentity, err := identity.MintBootstrapBiscuitToken(rootPriv, nodeB.Host.ID(), api.RoleNode, time.Now().Add(time.Hour), publishEverything, nil) + if err != nil { + t.Fatalf("mint nodeB identity: %v", err) + } + nodeB.SetIdentityCache(nodeBIdentity) + + hostedSrv := httptest.NewServer(newFakeMCPHandler(t, []*mcp.Tool{ + {Name: "review_pr", Description: "Run a code review", InputSchema: map[string]any{"type": "object"}}, + {Name: "delete_repo", Description: "Delete a repository", InputSchema: map[string]any{"type": "object"}}, + })) + defer hostedSrv.Close() + if err := nodeB.RegisterService(ctx, &api.RegisterServiceRequest{ + Service: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "code-reviewer"}, + Backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: hostedSrv.URL}, + }); err != nil { + t.Fatalf("RegisterService: %v", err) + } + defer func() { _ = nodeB.UnregisterService(ctx, "code-reviewer") }() + + base, err := nodeA.Store.LoadIdentity() + if err != nil { + t.Fatalf("load nodeA identity: %v", err) + } + tar, err := api.BuildTARFromOAuthParams("oauth-task", `{"name":"review-only","rules":[{"allowed_services":["mcp://code-reviewer"],"operation":{"allowed_tools":["review_pr"]}}]}`, nil, "", nil) + if err != nil { + t.Fatalf("build task rule: %v", err) + } + reviewOnly, err := identity.AttenuateBiscuit(base, tar) + if err != nil { + t.Fatalf("attenuate: %v", err) + } + + _, err = nodeA.CallMCPTool(WithCallerBiscuit(ctx, reviewOnly), nodeB.Host.ID(), "mcp://code-reviewer/delete_repo", map[string]any{}, nil) + if err == nil || !strings.Contains(err.Error(), `task authorization denied tool "delete_repo"`) { + t.Fatalf("CallMCPTool(delete_repo) = %v, want a task authorization denial", err) + } + if strings.Contains(err.Error(), "retries") { + t.Fatalf("CallMCPTool(delete_repo) = %v, want one answer with no retry", err) + } + var rpcErr *jsonrpc.Error + if !errors.As(err, &rpcErr) || rpcErr.Code != api.JSONRPCCodeUnauthorized { + t.Fatalf("CallMCPTool(delete_repo) = %v, want JSON-RPC code %d", err, api.JSONRPCCodeUnauthorized) + } + if !strings.Contains(rpcErr.Message, "does not allow it") { + t.Fatalf("denial message = %q, want it to say the task does not allow the call", rpcErr.Message) + } + + if _, err := nodeA.CallMCPTool(WithCallerBiscuit(ctx, reviewOnly), nodeB.Host.ID(), "mcp://code-reviewer/review_pr", map[string]any{}, nil); err != nil { + t.Fatalf("CallMCPTool(review_pr) within the task = %v, want success", err) + } +} + // The operator's egress floor gates MCP egress even when the caller supplies no // required_labels: silence must not waive the floor (see #385). func TestCallMCPTool_EgressFloorEnforcement(t *testing.T) { diff --git a/internal/node/mcp_service.go b/internal/node/mcp_service.go index 66932285..7a4ffde9 100644 --- a/internal/node/mcp_service.go +++ b/internal/node/mcp_service.go @@ -358,6 +358,17 @@ func (m *MCPService) HandleStreamPassThrough(s network.Stream, reqCtx RequestCon } } +// taskDenied answers a request the caller's task does not allow, with +// api.JSONRPCCodeUnauthorized. A calling model sees only the message (the MCP +// go-sdk builds its error from it), so the message says what was refused, that +// nothing reached the service, and that the same call is refused again. +func taskDenied(id jsonrpc.ID, what string, err error) *jsonrpc.Response { + return &jsonrpc.Response{ID: id, Error: &jsonrpc.Error{ + Code: api.JSONRPCCodeUnauthorized, + Message: fmt.Sprintf("task authorization denied %s: %v. The task this call runs under does not allow it, so it was not sent to the service; the same call under this task is denied again", what, err), + }} +} + // authorizeMCPRequest evaluates reqCtx.TaskRules against an inbound JSON-RPC // request on an MCP stream. Returns a JSON-RPC error response when denied, or // nil when allowed. @@ -374,7 +385,7 @@ func (m *MCPService) authorizeMCPRequest(req *jsonrpc.Request, reqCtx RequestCon AllowMCPStreamInit: true, }, time.Now()); err != nil { logger.Warnf("[MCPService] %s: TAR denied %s from %s: %v", svcName, req.Method, reqCtx.PeerID, err) - return &jsonrpc.Response{ID: req.ID, Error: &jsonrpc.Error{Code: jsonrpc.CodeInvalidRequest, Message: fmt.Sprintf("task authorization denied %s: %v", req.Method, err)}} + return taskDenied(req.ID, req.Method, err) } return nil case "tools/call": @@ -390,7 +401,7 @@ func (m *MCPService) authorizeMCPRequest(req *jsonrpc.Request, reqCtx RequestCon MCPTool: params.Name, }, time.Now()); err != nil { logger.Warnf("[MCPService] %s: TAR denied tools/call %q from %s: %v", svcName, params.Name, reqCtx.PeerID, err) - return &jsonrpc.Response{ID: req.ID, Error: &jsonrpc.Error{Code: jsonrpc.CodeInvalidRequest, Message: fmt.Sprintf("task authorization denied tool %q: %v", params.Name, err)}} + return taskDenied(req.ID, fmt.Sprintf("tool %q", params.Name), err) } return nil default: @@ -399,7 +410,7 @@ func (m *MCPService) authorizeMCPRequest(req *jsonrpc.Request, reqCtx RequestCon ServiceName: svcName, }, time.Now()); err != nil { logger.Warnf("[MCPService] %s: TAR denied %s from %s: %v", svcName, req.Method, reqCtx.PeerID, err) - return &jsonrpc.Response{ID: req.ID, Error: &jsonrpc.Error{Code: jsonrpc.CodeInvalidRequest, Message: fmt.Sprintf("task authorization denied %s: %v", req.Method, err)}} + return taskDenied(req.ID, req.Method, err) } return nil } diff --git a/site/content/docs/use-cases/multi-tier-guardrails.md b/site/content/docs/use-cases/multi-tier-guardrails.md index 3425186b..e0b54c23 100644 --- a/site/content/docs/use-cases/multi-tier-guardrails.md +++ b/site/content/docs/use-cases/multi-tier-guardrails.md @@ -145,7 +145,7 @@ Using that narrowed Task Biscuit: `Order #1042: status=SHIPPED, item='Mechanical Keyboard Pro', total=$129.00, customer=alice@acme.com` 2. If a prompt injection inside the order notes tricks the agent into calling `issue_refund(order_id="1042", amount_dollars=5000)` using the **same** Task Biscuit, the node's authorizer blocks the tool call mid-flight and leaves the SQLite `refunds` table untouched (`COUNT(*) = 0`): ```text - task authorization denied tool "issue_refund": tar_block[1] ("oauth-task") denied request to mcp://orders-db + task authorization denied tool "issue_refund": tar_block[1] ("oauth-task") denied request to mcp://orders-db. The task this call runs under does not allow it, so it was not sent to the service; the same call under this task is denied again ``` --- diff --git a/site/content/docs/use-cases/private-diy-stack.md b/site/content/docs/use-cases/private-diy-stack.md index 2f1ccbe0..9b6b6814 100644 --- a/site/content/docs/use-cases/private-diy-stack.md +++ b/site/content/docs/use-cases/private-diy-stack.md @@ -176,7 +176,7 @@ Act 3 proves that an LLM or compromised harness cannot bypass enterprise guardra --> [3d. Per-Task Sealed Biscuit (tar_block) — Blocking Prompt-Injected Wire Transfers] Minting a sealed Task Biscuit via RFC 8693 POST /oauth/token narrowed to read-only tools: 1. Allowed read tool (lookup_invoice) -> Invoice INV-2026-089: vendor=Acme Robotics GmbH (ACME-01), sku=SKU-GPU-H200, qty=4, amount=$48,500.00, status=PENDING_APPROVAL - 2. Prompt-injected tool (execute_wire_transfer) -> failed after 3 retries: failed to call tool mcp://erp-db/execute_wire_transfer: calling "tools/call": task authorization denied tool "execute_wire_transfer": tar_block[1] ("oauth-task") denied request to mcp://erp-db + 2. Prompt-injected tool (execute_wire_transfer) -> failed to call tool mcp://erp-db/execute_wire_transfer: calling "tools/call": task authorization denied tool "execute_wire_transfer": tar_block[1] ("oauth-task") denied request to mcp://erp-db. The task this call runs under does not allow it, so it was not sent to the service; the same call under this task is denied again 3. SQLite wire_transfers table row count -> 0 (database untouched!) ```