diff --git a/docs-data/personas.yaml b/docs-data/personas.yaml new file mode 100644 index 000000000..46d4e7912 --- /dev/null +++ b/docs-data/personas.yaml @@ -0,0 +1,153 @@ +# Redpanda Cloud Documentation Personas +# +# These personas represent the target audience for Redpanda Cloud documentation. +# Use these when assigning :personas: attributes to documentation pages. + +schema_version: "1.0" +repository: cloud-docs + +personas: + - id: app_developer + name: Application Developer + description: Builds applications that produce and consume data from Redpanda Cloud + experience_level: intermediate + goals: + - Connect applications to Redpanda Cloud clusters + - Produce and consume messages reliably + - Implement proper error handling and retries + - Optimize client performance + pain_points: + - Authentication and connection configuration + - Understanding Kafka client options + - Debugging connectivity issues + - Choosing the right client library + content_preferences: + - Working code examples in multiple languages + - Connection configuration templates + - Client library comparisons + - Performance tuning guides + typical_content_types: + - how-to + - tutorial + - reference + + - id: platform_admin + name: Platform Administrator + description: Manages Redpanda Cloud clusters, users, security, and billing + experience_level: intermediate + goals: + - Provision and configure clusters + - Manage user access and security + - Monitor cluster health and usage + - Control costs and optimize resources + pain_points: + - Complex security configuration + - Understanding billing and usage metrics + - Managing access across teams + - Capacity planning + content_preferences: + - Step-by-step administration guides + - Security best practices + - Billing and cost optimization tips + - Monitoring and alerting setup + typical_content_types: + - how-to + - reference + - best-practices + + - id: data_engineer + name: Data Engineer + description: Builds data pipelines using managed connectors and Redpanda Connect + experience_level: intermediate + goals: + - Set up managed connectors to move data between systems + - Transform and route data reliably + - Monitor connector and pipeline health + - Handle errors and retries + pain_points: + - Connector configuration complexity + - Debugging failed connectors + - Schema management and evolution + - Performance tuning + content_preferences: + - Connector setup guides + - Transformation examples + - Error handling patterns + - Monitoring and troubleshooting + typical_content_types: + - how-to + - cookbook + - troubleshooting + + - id: ai_agent_developer + name: AI Agent Developer + description: Builds AI agents and integrations using MCP tools and LLM frameworks + experience_level: intermediate + goals: + - Create MCP tools that AI assistants can discover and use + - Deploy MCP servers to Redpanda Cloud + - Integrate with AI/LLM applications + - Debug agent-tool interactions + pain_points: + - MCP configuration syntax + - Testing tools before deployment + - Limited AI-specific examples + content_preferences: + - Working code examples with AI context + - Testing and debugging workflows + - Integration patterns + typical_content_types: + - tutorial + - how-to + - cookbook + - best-practices + + - id: streaming_developer + name: Streaming Platform Developer + description: Works with Kafka/Redpanda streaming systems at scale + experience_level: advanced + goals: + - Build event-driven architectures on Redpanda Cloud + - Optimize for high throughput and low latency + - Implement exactly-once semantics + - Migrate from self-hosted Kafka + pain_points: + - Consumer group behavior differences + - Schema registry configuration + - Performance at scale + - Migration complexity + content_preferences: + - Advanced configuration options + - Performance tuning guides + - Migration documentation + - Architecture patterns + typical_content_types: + - concepts + - how-to + - reference + - best-practices + + - id: evaluator + name: Technical Evaluator + description: Assessing Redpanda Cloud for their organization + experience_level: beginner + goals: + - Understand Redpanda Cloud capabilities quickly + - Evaluate fit for specific use cases + - Understand pricing and billing + - Compare with alternatives + pain_points: + - Hard to find high-level overview + - Too much detail too soon + - Unclear pricing structure + - Missing comparison information + content_preferences: + - High-level overviews + - Use case examples + - Quick start guides + - Pricing documentation + typical_content_types: + - overview + - concepts + - tutorial + - get-started diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index 3cd0d159d..b5a655ee8 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -26,7 +26,7 @@ content: branches: main start_paths: [docs,'*/docs'] - url: https://github.com/redpanda-data/rp-connect-docs - branches: main + branches: improve-mcp-devex ui: bundle: url: https://github.com/redpanda-data/docs-ui/releases/latest/download/ui-bundle.zip diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 3cff3873c..72455d4ee 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -78,12 +78,15 @@ ** xref:ai-agents:mcp/remote/index.adoc[Remote MCP] *** xref:ai-agents:mcp/remote/overview.adoc[Overview] *** xref:ai-agents:mcp/remote/quickstart.adoc[Quickstart] -*** xref:ai-agents:mcp/remote/developer-guide.adoc[Developer Guide] +*** xref:ai-agents:mcp/remote/concepts.adoc[Concepts] +*** xref:ai-agents:mcp/remote/create-tool.adoc[Create a Tool] +*** xref:ai-agents:mcp/remote/best-practices.adoc[Best Practices] +*** xref:ai-agents:mcp/remote/tool-patterns.adoc[Tool Patterns] +*** xref:ai-agents:mcp/remote/troubleshooting.adoc[Troubleshooting] *** xref:ai-agents:mcp/remote/admin-guide.adoc[Admin Guide] **** xref:ai-agents:mcp/remote/manage-servers.adoc[Manage Servers] **** xref:ai-agents:mcp/remote/scale-resources.adoc[Scale Resources] **** xref:ai-agents:mcp/remote/monitor-activity.adoc[Monitor Activity] -*** xref:ai-agents:mcp/remote/pipeline-patterns.adoc[MCP Server Patterns] * xref:develop:connect/about.adoc[Redpanda Connect] ** xref:develop:connect/connect-quickstart.adoc[Quickstart] diff --git a/modules/ai-agents/examples/redpanda_output_with_processors.yaml b/modules/ai-agents/examples/redpanda_output_with_processors.yaml index d58997a60..eea4b323f 100644 --- a/modules/ai-agents/examples/redpanda_output_with_processors.yaml +++ b/modules/ai-agents/examples/redpanda_output_with_processors.yaml @@ -1,12 +1,4 @@ -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topic: "llm-responses" - tls: - enabled: true - sasl: - - mechanism: "${REDPANDA_SASL_MECHANISM}" - username: "${REDPANDA_SASL_USERNAME}" - password: "${REDPANDA_SASL_PASSWORD}" +label: summarize_and_publish processors: - openai_chat_completion: @@ -17,3 +9,23 @@ processors: root.question = this.question root.answer = this.content root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") + +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topic: "llm-responses" + tls: + enabled: true + sasl: + - mechanism: SCRAM-SHA-256 + username: "${secrets.MCP_USERNAME}" + password: "${secrets.MCP_PASSWORD}" + +meta: + mcp: + enabled: true + description: "Process a question through an LLM and publish the response to Redpanda" + properties: + - name: question + type: string + description: "The question to send to the LLM" + required: true diff --git a/modules/ai-agents/examples/test-mcp-examples.sh b/modules/ai-agents/examples/test-mcp-examples.sh index 595fcebfe..068f7306a 100755 --- a/modules/ai-agents/examples/test-mcp-examples.sh +++ b/modules/ai-agents/examples/test-mcp-examples.sh @@ -1,9 +1,18 @@ #!/usr/bin/env bash # -# Automated testing script for Redpanda Connect MCP examples +# Test script for Redpanda Cloud MCP examples +# +# This script tests: +# 1. MCP tool definitions using `rpk connect mcp-server lint` +# 2. MCP metadata validation (enabled, description, properties) # # Usage: -# ./test-mcp-examples.sh +# ./test-mcp-examples.sh # Run all tests +# ./test-mcp-examples.sh --lint-only # Only lint, skip metadata validation +# +# Unlike rp-connect-docs, Cloud MCP tools cannot be tested with +# `rpk connect run` because they are standalone tool definitions, not +# full pipelines. End-to-end testing requires the Cloud Console. set -euo pipefail @@ -12,51 +21,95 @@ RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' +CYAN='\033[0;36m' NC='\033[0m' +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + # Counters -TOTAL=0 +TOTAL_TOOLS=0 +PASSED_LINT=0 +PASSED_METADATA=0 +FAILED_LINT=0 +FAILED_METADATA=0 SKIPPED=0 -MCP_FAILS=0 -echo "๐Ÿงช Redpanda Connect MCP Examples Test Suite" +echo "๐Ÿงช Redpanda Cloud MCP Examples - Test Suite" echo "============================================" echo "" -# Run MCP server lint on the directory -echo "Running rpk connect mcp-server lint..." -LINT_OUTPUT=$(rpk connect mcp-server lint --skip-env-var-check --verbose 2>&1) || { - echo -e "${RED}โŒ Linting failed${NC}" - echo "" - echo "$LINT_OUTPUT" - exit 1 -} -echo -e "${GREEN}โœ… Linting passed${NC}" +# Parse arguments +RUN_METADATA=true + +if [[ $# -gt 0 ]]; then + case "$1" in + --lint-only) + RUN_METADATA=false + ;; + esac +fi + +# ============================================================================ +# SECTION 1: MCP Tool Linting +# Validates YAML syntax and component schemas +# ============================================================================ + +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" +echo -e "๐Ÿ“ฆ ${CYAN}SECTION 1: MCP Tool Linting${NC}" +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo "" -# Function to validate MCP metadata -validate_mcp_metadata() { - local file=$1 +# Count YAML files +file_count=$(find . -maxdepth 1 -name "*.yaml" | wc -l | tr -d ' ') +TOTAL_TOOLS=$file_count - echo -n " Validating MCP metadata... " +echo -n -e "${BLUE}๐Ÿ“ examples/${NC} ($file_count files)... " + +if output=$(rpk connect mcp-server lint --skip-env-var-check . 2>&1); then + echo -e "${GREEN}โœ“ PASSED${NC}" + PASSED_LINT=$file_count +else + echo -e "${RED}โœ— FAILED${NC}" + echo "$output" | sed 's/^/ /' | head -20 + FAILED_LINT=$file_count +fi + +# ============================================================================ +# SECTION 2: MCP Metadata Validation +# Validates tool metadata (enabled, description, properties) +# ============================================================================ + +if $RUN_METADATA; then + echo "" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + echo -e "๐Ÿ“ ${CYAN}SECTION 2: MCP Metadata Validation${NC}" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + echo "" # Determine which YAML parser to use - local use_yq=true + use_yq=true if ! command -v yq &> /dev/null; then use_yq=false if ! command -v python3 &> /dev/null; then - echo -e "${RED}FAILED${NC} (neither yq nor python3 available)" - MCP_FAILS=$((MCP_FAILS + 1)) - return 1 + echo -e "${YELLOW}โš  Neither yq nor python3 available - skipping metadata validation${NC}" + RUN_METADATA=false fi fi - # Check if .meta.mcp exists - local mcp_exists - if $use_yq; then - mcp_exists=$(yq eval '.meta.mcp' "$file" 2>/dev/null) - else - mcp_exists=$(python3 -c " + if $RUN_METADATA; then + for file in *.yaml; do + if [[ -f "$file" ]]; then + echo -n -e " ${BLUE}$file${NC}... " + + # Check if .meta.mcp exists + if $use_yq; then + mcp_exists=$(yq eval '.meta.mcp' "$file" 2>/dev/null) + enabled=$(yq eval '.meta.mcp.enabled' "$file" 2>/dev/null) + description=$(yq eval '.meta.mcp.description' "$file" 2>/dev/null) + else + mcp_exists=$(python3 -c " import yaml try: with open('$file') as f: @@ -67,20 +120,7 @@ try: except: print('null') " 2>/dev/null) - fi - - if [[ "$mcp_exists" == "null" || -z "$mcp_exists" ]]; then - echo -e "${YELLOW}SKIPPED${NC} (no MCP metadata)" - SKIPPED=$((SKIPPED + 1)) - return 0 - fi - - # Read .meta.mcp.enabled - local enabled - if $use_yq; then - enabled=$(yq eval '.meta.mcp.enabled' "$file" 2>/dev/null) - else - enabled=$(python3 -c " + enabled=$(python3 -c " import yaml try: with open('$file') as f: @@ -90,19 +130,7 @@ try: except: print('null') " 2>/dev/null) - fi - - if [[ "$enabled" != "true" ]]; then - echo -e "${YELLOW}WARNING${NC} (mcp.enabled not set to true)" - return 0 - fi - - # Read .meta.mcp.description - local description - if $use_yq; then - description=$(yq eval '.meta.mcp.description' "$file" 2>/dev/null) - else - description=$(python3 -c " + description=$(python3 -c " import yaml try: with open('$file') as f: @@ -112,44 +140,79 @@ try: except: print('null') " 2>/dev/null) + fi + + # Validate + if [[ "$mcp_exists" == "null" || -z "$mcp_exists" ]]; then + echo -e "${YELLOW}SKIPPED${NC} (no MCP metadata)" + SKIPPED=$((SKIPPED + 1)) + elif [[ "$enabled" != "true" ]]; then + echo -e "${YELLOW}WARNING${NC} (mcp.enabled not true)" + SKIPPED=$((SKIPPED + 1)) + elif [[ "$description" == "null" || -z "$description" ]]; then + echo -e "${RED}FAILED${NC} (missing description)" + FAILED_METADATA=$((FAILED_METADATA + 1)) + else + echo -e "${GREEN}PASSED${NC}" + PASSED_METADATA=$((PASSED_METADATA + 1)) + fi + fi + done fi +fi - if [[ "$description" == "null" || -z "$description" ]]; then - echo -e "${RED}FAILED${NC} (missing description)" - MCP_FAILS=$((MCP_FAILS + 1)) - return 1 - fi +# ============================================================================ +# SECTION 3: Cloud-Specific Validation +# Validates secrets use Cloud format (${secrets.X}) +# ============================================================================ - echo -e "${GREEN}PASSED${NC}" - return 0 -} +echo "" +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" +echo -e "โ˜๏ธ ${CYAN}SECTION 3: Cloud Secrets Format${NC}" +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" +echo "" -# Validate MCP metadata for each file +secrets_issues=0 for file in *.yaml; do if [[ -f "$file" ]]; then - TOTAL=$((TOTAL + 1)) - echo "" - echo -e "${BLUE}๐Ÿ“„ Validating: $file${NC}" - validate_mcp_metadata "$file" + # Check for non-Cloud secrets patterns (${VAR} without secrets. prefix) + # Exclude: + # - ${! ... } which is Bloblang interpolation + # - ${REDPANDA_BROKERS} which is platform-injected + if grep -E '\$\{[A-Z_]+\}' "$file" | grep -v '\${secrets\.' | grep -v '\${!' | grep -v '\${REDPANDA_BROKERS}' > /dev/null 2>&1; then + echo -e " ${BLUE}$file${NC}... ${YELLOW}WARNING${NC} (uses env vars instead of \${secrets.X})" + secrets_issues=$((secrets_issues + 1)) + fi fi done -# Summary +if [[ $secrets_issues -eq 0 ]]; then + echo -e " ${GREEN}โœ“ All files use Cloud secrets format${NC}" +fi + +# ============================================================================ +# SUMMARY +# ============================================================================ + echo "" echo "============================================" echo "๐Ÿ“Š Test Summary" echo "============================================" -echo "Total configs tested: $TOTAL" -if [[ $MCP_FAILS -gt 0 ]]; then - echo -e "MCP validation failures: ${RED}$MCP_FAILS${NC}" + +echo -e "Lint: ${PASSED_LINT}/${TOTAL_TOOLS} passed" +if $RUN_METADATA; then + METADATA_TOTAL=$((PASSED_METADATA + FAILED_METADATA + SKIPPED)) + echo -e "Metadata: ${PASSED_METADATA}/${METADATA_TOTAL} passed (${SKIPPED} skipped)" fi -if [[ $SKIPPED -gt 0 ]]; then - echo -e "Skipped: ${YELLOW}$SKIPPED${NC}" +if [[ $secrets_issues -gt 0 ]]; then + echo -e "Secrets: ${YELLOW}${secrets_issues} warnings${NC}" fi -echo "" +echo "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + +TOTAL_FAILED=$((FAILED_LINT + FAILED_METADATA)) -if [[ $MCP_FAILS -gt 0 ]]; then - echo -e "${RED}โŒ Some tests failed${NC}" +if [[ $TOTAL_FAILED -gt 0 ]]; then + echo -e "${RED}โŒ Some tests failed ($TOTAL_FAILED failures)${NC}" exit 1 else echo -e "${GREEN}โœ… All tests passed!${NC}" diff --git a/modules/ai-agents/examples/testing.adoc b/modules/ai-agents/examples/testing.adoc index 812400b24..96d8525ab 100644 --- a/modules/ai-agents/examples/testing.adoc +++ b/modules/ai-agents/examples/testing.adoc @@ -58,18 +58,15 @@ MCP metadata validation checks: === Unit testing limitations -[IMPORTANT] -==== -MCP tool examples are standalone component definitions (e.g., `label:`, `processors:`, `meta:`), not full pipelines with `input:`, `pipeline:`, `output:` sections. This means they cannot use inline `tests:` sections like cookbook examples do. +MCP tool examples are standalone component definitions (`label:`, `processors:`, `meta:`), not full pipelines with `input:`, `pipeline:`, `output:` sections. This means they cannot use inline `tests:` sections like cookbook examples do. The `rpk connect test` command requires full pipeline structure with paths like `/pipeline/processors/0`, which don't exist in MCP tool definitions. -==== For testing MCP tools: -* **Linting is the primary validation** - ensures syntax and schema correctness -* **MCP metadata validation** - verifies tool has proper description and properties -* **Manual testing** - uses the Cloud Console MCP Server interface to test tools end-to-end +- Ensure syntax and schema correctness. +- Verify MCP metadata has proper description and properties. +- Perform manual testing using the Cloud Console MCP Server interface to test tools end-to-end. == MCP tool structure @@ -143,13 +140,12 @@ For comprehensive validation, test MCP tools using the Cloud Console: . Navigate to your Cloud cluster's MCP Server configuration . Add or update your MCP tool configuration -. Use the Cloud Console's AI chat interface to invoke the tool +. Use the Cloud Console's MCP Inspector to locate your tool . Verify the tool executes correctly and returns expected results This validates: * Tool loads correctly in the MCP server -* MCP metadata is properly exposed to AI clients * Tool executes with provided parameters * Responses are formatted correctly * Secrets are properly resolved from Cloud Secrets Store diff --git a/modules/ai-agents/examples/weather_service.yaml b/modules/ai-agents/examples/weather_service.yaml index 1a6c56fe4..3ebe920f6 100644 --- a/modules/ai-agents/examples/weather_service.yaml +++ b/modules/ai-agents/examples/weather_service.yaml @@ -1,6 +1,6 @@ label: weather-service processors: - - label: validate_inputs + - label: validate_inputs # <1> mutation: | # Validate and sanitize city input meta city = this.city.string(). @@ -12,15 +12,15 @@ processors: throw("City name cannot be empty") } else { "" } - - label: fetch_weather_data + - label: fetch_weather_data # <2> try: - - http: + - http: # <3> url: "https://wttr.in/${! @city }?format=j1" verb: GET headers: User-Agent: "redpanda-connect-mcp/1.0" timeout: "10s" - - mutation: | + - mutation: | # <4> root = { "city": @city, "temperature_c": this.current_condition.0.temp_C.number(), @@ -35,7 +35,7 @@ processors: message: "Weather data fetched for city: ${! @city }" level: "INFO" - - label: handle_weather_errors + - label: handle_weather_errors # <5> catch: - mutation: | root = { @@ -48,7 +48,7 @@ processors: message: "Weather API error for city ${! @city }: ${! error() }" level: "ERROR" -meta: +meta: # <6> tags: [ weather, example ] mcp: enabled: true diff --git a/modules/ai-agents/pages/index.adoc b/modules/ai-agents/pages/index.adoc index 731129252..c5dd8a04c 100644 --- a/modules/ai-agents/pages/index.adoc +++ b/modules/ai-agents/pages/index.adoc @@ -7,6 +7,8 @@ AI agents are configurable assistants that autonomously perform specialist tasks Redpanda Cloud provides two complementary Model Context Protocol (MCP) options to help you build AI agents: -**Local MCP server for Redpanda Cloud** badge::[label=beta, tooltip={page-beta-text}]: A built-in server that gives your AI assistant direct access to your Redpanda Cloud account and clusters. This runs locally on your computer and lets you quickly perform operations like creating clusters, listing topics, and reading messages. +MCP server for the Redpanda Cloud API badge::[label=beta, tooltip={page-beta-text}]:: +A built-in server that gives your AI agent direct access to your Redpanda Cloud account and clusters. This runs locally on your computer and lets you quickly perform operations like creating clusters, listing topics, and reading messages. -**Remote MCP (managed)**: Your own custom MCP servers built with Redpanda Connect pipelines and hosted inside your Redpanda Cloud cluster. These let you create reusable tools for your team that integrate with knowledge bases, APIs, and web content while running close to your data. \ No newline at end of file +Remote MCP:: +Your own custom MCP servers built with Redpanda Connect pipelines and hosted inside your Redpanda Cloud cluster. These let you create reusable tools for your team that integrate with knowledge bases, APIs, and web content while running close to your data. \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/local/configuration.adoc b/modules/ai-agents/pages/mcp/local/configuration.adoc index 4b3c4cee7..9bcaf55fe 100644 --- a/modules/ai-agents/pages/mcp/local/configuration.adoc +++ b/modules/ai-agents/pages/mcp/local/configuration.adoc @@ -1,8 +1,15 @@ -= Configure the Local MCP server for Redpanda Cloud += Configure the MCP Server for the Redpanda Cloud API :page-beta: true -:description: Learn how to configure the local MCP server for Redpanda Cloud, including auto and manual client setup, enabling deletes, and security considerations. +:description: Learn how to configure the MCP server for the Redpanda Cloud API, including auto and manual client setup, enabling deletes, and security considerations. +:page-topic-type: how-to +:personas: ai_agent_developer, platform_admin +// Reader journey: "I customize and configure" +// Learning objectives - what readers can learn from this page: +:learning-objective-1: Configure MCP clients manually +:learning-objective-2: Enable delete operations safely +:learning-objective-3: Troubleshoot common configuration issues -This page explains how to configure the local MCP server for Redpanda Cloud, including auto and manual client setup, enabling deletes, and security considerations. +This page explains how to configure the MCP server for the Redpanda Cloud API, including auto and manual client setup, enabling deletes, and security considerations. == Prerequisites @@ -31,7 +38,7 @@ If you need to update the integration, re-run the install command for your clien == Manual configuration for other MCP clients -If you're using another MCP-compatible client, you can manually configure it to use the local MCP server for Redpanda Cloud. Follow these steps: +If you're using another MCP-compatible client, you can manually configure it to use the MCP server for the Redpanda Cloud API. Follow these steps: Add an MCP server entry to your client's configuration (example shown in JSON). Adjust paths for your system. diff --git a/modules/ai-agents/pages/mcp/local/index.adoc b/modules/ai-agents/pages/mcp/local/index.adoc index 16429590c..d656760f4 100644 --- a/modules/ai-agents/pages/mcp/local/index.adoc +++ b/modules/ai-agents/pages/mcp/local/index.adoc @@ -1,4 +1,4 @@ -= Local MCP server for Redpanda Cloud += MCP Server for the Redpanda Cloud API :page-beta: true -:description: Find links to information about the Local MCP server for Redpanda Cloud and its features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. +:description: Find links to information about the MCP server for the Redpanda Cloud API and its features for building and managing AI agents that can interact with your Redpanda Cloud account and clusters. :page-layout: index diff --git a/modules/ai-agents/pages/mcp/local/overview.adoc b/modules/ai-agents/pages/mcp/local/overview.adoc index cc624f400..6b799b27a 100644 --- a/modules/ai-agents/pages/mcp/local/overview.adoc +++ b/modules/ai-agents/pages/mcp/local/overview.adoc @@ -1,16 +1,22 @@ - -= About the Local MCP server for Redpanda Cloud += MCP Server for the Redpanda Cloud API :page-beta: true -:description: Learn about the local MCP server for Redpanda Cloud, which lets AI assistants securely access and operate your Redpanda Cloud account and clusters. +:description: Learn about the MCP server for the Redpanda Cloud API, which lets AI agents securely access and operate your Redpanda Cloud account and clusters. +:page-topic-type: overview +:personas: evaluator, ai_agent_developer, platform_admin +// Reader journey: "I'm new" +// Learning objectives - what readers should understand after reading this page: +:learning-objective-1: Explain what the MCP server for the Redpanda Cloud API does +:learning-objective-2: Identify what operations are available through MCP +:learning-objective-3: Identify security considerations for MCP authentication -The local MCP server for Redpanda Cloud lets AI assistants securely access and operate your Redpanda Cloud account and clusters. MCP provides controlled access to: +The MCP server for the Redpanda Cloud API lets AI agents securely access and operate your Redpanda Cloud account and clusters. MCP provides controlled access to: -* link:https://docs.redpanda.com/api/doc/cloud-controlplane/[Control Plane] actions, such as creating a Redpanda Cloud cluster or listing clusters. -* link:https://docs.redpanda.com/api/doc/cloud-dataplane/[Data Plane] actions, such as creating topics or listing topics. +* link:https://docs.redpanda.com/api/doc/cloud-controlplane/[Control Plane] APIs, such as creating a Redpanda Cloud cluster or listing clusters. +* link:https://docs.redpanda.com/api/doc/cloud-dataplane/[Data Plane] APIs, such as creating topics or listing topics. By speaking natural language to your assistant, you can ask it to perform Redpanda operations on your behalf. The MCP server runs locally on your machine and authenticates to Redpanda Cloud using a Redpanda Cloud token. -image::shared:cloud-mcp.gif[A terminal window showing Claude Code invoking the local MCP server for Redpanda Cloud to list topics in a cluster.] +image::shared:cloud-mcp.gif[A terminal window showing Claude Code invoking the MCP server for the Redpanda Cloud API to list topics in a cluster.] == What you can do @@ -32,13 +38,13 @@ NOTE: The MCP server does **not** expose delete endpoints by default. You can en . Authenticate to Redpanda Cloud and receive a token using the xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[`rpk cloud login` command]. . Configure your MCP client using the xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-install.adoc[`rpk cloud mcp install`] command. Your client then starts the server on-demand using xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc[`rpk cloud mcp stdio`], authenticating with the Redpanda Cloud token from `rpk cloud login`. -. Prompt your assistant to perform Redpanda operations. The local MCP server for Redpanda Cloud executes them in your Redpanda Cloud account using your Redpanda Cloud token. +. Prompt your assistant to perform Redpanda operations. The MCP server executes them in your Redpanda Cloud account using your Redpanda Cloud token. === Components * AI client (Claude, Claude Code, or any other MCP client) that connects to the MCP server. -* Redpanda CLI (`rpk`) for obtaining a token and starting the local MCP server. -* Redpanda Cloud account that the local MCP server can connect to and issue API requests. +* Redpanda CLI (`rpk`) for obtaining a token and starting the MCP server. +* Redpanda Cloud account that the MCP server can connect to and issue API requests. == Security considerations @@ -50,9 +56,13 @@ MCP servers authenticate to Redpanda Cloud using your personal or service accoun == Next steps -* xref:ai-agents:mcp/local/quickstart.adoc[] -* xref:ai-agents:mcp/local/configuration.adoc[] +Continue your learning journey with these resources: + +* xref:ai-agents:mcp/local/quickstart.adoc[MCP server for the Redpanda Cloud API quickstart] +* xref:ai-agents:mcp/local/configuration.adoc[Configure the MCP server for the Redpanda Cloud API] == Suggested reading -The Redpanda documentation site has a read-only MCP server that provides access to Redpanda docs and examples. This server has no access to your Redpanda Cloud account or clusters. See xref:home:ROOT:mcp-setup.adoc[]. +For more information, see these resources: + +* xref:home:ROOT:mcp-setup.adoc[Read-only MCP server for Redpanda docs]: Access Redpanda documentation and examples through AI agents diff --git a/modules/ai-agents/pages/mcp/local/quickstart.adoc b/modules/ai-agents/pages/mcp/local/quickstart.adoc index cf5f7be3d..6a67db1ff 100644 --- a/modules/ai-agents/pages/mcp/local/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/local/quickstart.adoc @@ -1,8 +1,15 @@ -= Local MCP server for Redpanda Cloud Quickstart += MCP Server for the Redpanda Cloud API Quickstart :page-beta: true -:description: Connect your Claude AI assistant to your Redpanda Cloud account and clusters using the local MCP server for Redpanda Cloud. +:description: Connect your Claude AI agent to your Redpanda Cloud account and clusters using the MCP server for the Redpanda Cloud API. +:page-topic-type: tutorial +:personas: ai_agent_developer, platform_admin +// Reader journey: "I'm new" โ†’ first hands-on experience +// Learning objectives - what readers will achieve by completing this quickstart: +:learning-objective-1: Authenticate to Redpanda Cloud with rpk +:learning-objective-2: Install the MCP integration for Claude +:learning-objective-3: Issue natural language commands to manage clusters -In this quickstart, you'll get your Claude AI assistant talking to Redpanda Cloud using the xref:ai-agents:mcp/local/overview.adoc[local MCP server for Redpanda Cloud]. +In this quickstart, you'll get your Claude AI agent talking to Redpanda Cloud using the xref:ai-agents:mcp/local/overview.adoc[MCP server for the Redpanda Cloud API]. If you're trying to deploy your own MCP server as a managed service inside your cluster, see xref:ai-agents:mcp/remote/quickstart.adoc[]. @@ -30,7 +37,7 @@ Ensure the version is at least 25.2.3. rpk cloud login ``` + -A browser window opens. Sign in to grant access. After you sign in, `rpk` stores a token locally. This token is not shared with your AI assistant. It is used by the MCP server to authenticate requests to your Redpanda Cloud account. +A browser window opens. Sign in to grant access. After you sign in, `rpk` stores a token locally. This token is not shared with your AI agent. It is used by the MCP server to authenticate requests to your Redpanda Cloud account. . Install the MCP integration. Choose one client: + @@ -60,4 +67,6 @@ NOTE: The MCP server does *not* expose API endpoints that result in delete opera == Next steps -* xref:ai-agents:mcp/local/configuration.adoc[] +Continue your learning journey with this resource: + +* xref:ai-agents:mcp/local/configuration.adoc[]: Configure the MCP server for advanced use cases diff --git a/modules/ai-agents/pages/mcp/overview.adoc b/modules/ai-agents/pages/mcp/overview.adoc index 05bcd3f79..257435679 100644 --- a/modules/ai-agents/pages/mcp/overview.adoc +++ b/modules/ai-agents/pages/mcp/overview.adoc @@ -1,49 +1,93 @@ -= MCP in Redpanda Cloud -:description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: Redpanda Cloud MCP Server and Remote MCP (managed). += MCP Overview +:description: Learn about Model Context Protocol (MCP) in Redpanda Cloud, including the two complementary options: the MCP server for the Redpanda Cloud API and Remote MCP. +:page-topic-type: overview +:personas: evaluator, ai_agent_developer +// Reader journey: "I'm new" - understanding the landscape +// Learning objectives - what readers should understand after reading this page: +:learning-objective-1: Describe what MCP enables for AI agents +:learning-objective-2: Distinguish between local and remote MCP options +:learning-objective-3: Choose the right MCP option for your use case +This page introduces MCP in Redpanda Cloud and helps you choose the right option for your use case. -Model Context Protocol (MCP) is a way to give AI assistants like Claude access to tools and data they wouldn't normally have. Instead of just chatting with your AI assistant, you can ask it to perform real tasks on your behalf. +After reading this page, you will be able to: -== Why use MCP with Redpanda Cloud? +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} -With MCP, you can have natural conversations with your AI assistant about your streaming data infrastructure. Rather than remembering command-line syntax or navigating through web interfaces, you describe what you want in plain language and your AI assistant handles the technical details. +== What is MCP? + +MCP (Model Context Protocol) is an open standard that lets AI agents use tools. Think of it like a universal adapter: instead of building custom integrations for every AI system, you define your tools once using MCP, and any MCP-compatible AI client can discover and use them. + +Without MCP, connecting AI to your business systems requires custom API code, authentication handling, and response formatting for each AI platform. With MCP, you describe what a tool does and what inputs it needs, and the protocol handles the rest. == MCP options in Redpanda Cloud -In Redpanda Cloud, there are two complementary options to use MCP: +Redpanda Cloud offers two complementary MCP options: + +MCP server for the Redpanda Cloud API badge::[label=beta, tooltip={page-beta-text}]:: +A pre-built server that gives AI agents access to Redpanda Cloud APIs. It runs on your computer and lets you manage clusters, topics, and other resources through natural language. ++ +Example: "Create a cluster called `dev-analytics` with 3 brokers." + +Remote MCP:: +Your own MCP server built with Redpanda Connect and hosted inside your Redpanda Cloud cluster. You define custom tools that access your data, call external APIs, or trigger workflows. ++ +Example: "Analyze the last 100 orders and show me the top product categories." -[cols="1s,2a,2a"] +=== Comparison + +[cols="1h,2,2"] |=== -| |Redpanda Cloud MCP Server badge::[label=beta, tooltip={page-beta-text}] | Remote MCP +| | MCP server for the Redpanda Cloud API | Remote MCP -| Description -| A server that gives your AI assistant access to your Redpanda Cloud account and clusters. -| Your own MCP server built with Redpanda Connect and hosted inside your Redpanda Cloud cluster. +| Purpose +| Operate your Redpanda Cloud account +| Build custom tools for your data and workflows | Where it runs | Your computer -| In your Redpanda Cloud cluster (managed by Redpanda) +| Redpanda Cloud (managed) | Who builds it | Redpanda -| You, with Redpanda Connect +| You, using Redpanda Connect -| Example use cases (prompts you might ask) -| - Create a new cluster called `dev-analytics` for testing. -- List all topics in my production cluster. -| - Analyze the last 100 orders and show me the top product categories. -- Generate a summary of user activity from the events stream. -|=== +| Access to +| Redpanda Cloud APIs (clusters, topics, users, ACLs) +| Anything you configure (databases, APIs, Redpanda topics, LLMs) -TIP: The Redpanda documentation site includes a read-only MCP server that gives you access to Redpanda documentation and code examples through AI assistants like Claude Desktop. This server only provides access to public documentation content. It cannot access your Redpanda Cloud account, clusters, or any private data. This is useful for getting help with Redpanda concepts and examples without connecting your actual Redpanda environment. For setup instructions, see xref:home:ROOT:mcp-setup.adoc[]. +| Best for +| Platform operations, quick admin tasks, incident response +| Data analysis, workflow automation, team-shared tools +|=== == Which should I use? -Use the **Redpanda Cloud MCP Server** to operate your Redpanda Cloud account quickly from your laptop. +Choose based on what you want to accomplish: -Use the **Remote MCP** to provide reusable tools for your team, running inside the cluster, authored with Redpanda Connect. +**Use the MCP server for the Redpanda Cloud API** when you want to: + +* Manage Redpanda Cloud resources without memorizing CLI commands +* Quickly create test clusters, topics, or users +* Inspect cluster health during incidents +* Onboard team members who prefer natural language over APIs + +**Use Remote MCP** when you want to: + +* Build tools that access your business data in Redpanda topics +* Create reusable tools shared across your team +* Connect AI agents to external systems (databases, APIs, LLMs) +* Run tools close to your data with managed infrastructure + +You can use both options together. For example, use the MCP server for the Redpanda Cloud API to create a cluster, then deploy Remote MCP tools to analyze data in that cluster. == Get started -* xref:ai-agents:mcp/local/quickstart.adoc[] -* xref:ai-agents:mcp/remote/quickstart.adoc[] +* xref:ai-agents:mcp/local/quickstart.adoc[]: Connect Claude to your Redpanda Cloud account +* xref:ai-agents:mcp/remote/quickstart.adoc[]: Build and deploy custom MCP tools + +== Suggested reading + +* xref:home:ROOT:mcp-setup.adoc[]: Access Redpanda documentation through AI agents (read-only, no Cloud access required) diff --git a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc index 8df0ea460..d4e47e3e1 100644 --- a/modules/ai-agents/pages/mcp/remote/admin-guide.adoc +++ b/modules/ai-agents/pages/mcp/remote/admin-guide.adoc @@ -1,5 +1,12 @@ = Remote MCP Server Administration Guide :description: Overview of administrative tasks for managing MCP servers in Redpanda Cloud. +:page-topic-type: overview +:personas: platform_admin, ai_agent_developer +// Reader journey: "I operate and maintain" +// Learning objectives - what readers can learn from this page: +:learning-objective-1: Identify available MCP server administrative tasks +:learning-objective-2: Describe how to use the Data Plane API for programmatic management +:learning-objective-3: Describe the server lifecycle stages This guide provides an overview of administrative tasks for MCP servers. Use these resources to manage your MCP servers throughout their lifecycle, from creation to deletion. @@ -51,6 +58,6 @@ Access tokens expire after 1 hour. To refresh, make the authentication request a == Next steps -* xref:ai-agents:mcp/remote/developer-guide.adoc[Build custom tools] for your specific use cases. +* xref:ai-agents:mcp/remote/best-practices.adoc[Learn best practices] for building robust tools. * xref:develop:connect/configuration/secret-management.adoc[Manage secrets] used by MCP server tools. -* xref:billing:billing.adoc#remote-mcp-billing-metrics[Understand MCP billing] to optimize costs. +* xref:billing:billing.adoc#remote-mcp-billing-metrics[Review MCP billing] to optimize costs. diff --git a/modules/ai-agents/pages/mcp/remote/best-practices.adoc b/modules/ai-agents/pages/mcp/remote/best-practices.adoc new file mode 100644 index 000000000..28df4084d --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/best-practices.adoc @@ -0,0 +1,42 @@ += MCP Tool Design +:description: Design effective MCP tool interfaces with clear names, descriptions, and input properties. +:page-topic-type: best-practices +:personas: ai_agent_developer +// Reader journey: "I want AI clients to discover and use my tools effectively" +// Learning objectives - what readers should be able to do after reading this page: +:learning-objective-1: Write tool names and descriptions that help AI clients select the right tool +:learning-objective-2: Define input properties with appropriate types and constraints +:learning-objective-3: Design focused tools that complete quickly + +After xref:ai-agents:mcp/remote/create-tool.adoc[creating your first tool], apply these design guidelines so AI clients can discover, understand, and invoke your tools correctly. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +[[tool-discovery]] +== Tool discovery + +include::redpanda-connect:ai-agents:partial$mcp/best-practices/mcp-metadata-design.adoc[] + +[[tool-execution]] +== Tool execution + +include::redpanda-connect:ai-agents:partial$mcp/best-practices/tool-implementation-practices.adoc[] + +== Complete example + +This example combines all the best practices: + +[source,yaml] +---- +include::redpanda-connect:ai-agents:example$best-practices/mcp-metadata/search-customer-orders.yaml[tags=complete-example] +---- + +== Next steps + +* xref:ai-agents:mcp/remote/create-tool.adoc#secrets[Use secrets]: Store credentials securely in the Secrets Store +* xref:ai-agents:mcp/remote/tool-patterns.adoc[]: Find reusable patterns including validation, error handling, and response formatting +* xref:ai-agents:mcp/remote/troubleshooting.adoc[]: Diagnose common issues diff --git a/modules/ai-agents/pages/mcp/remote/concepts.adoc b/modules/ai-agents/pages/mcp/remote/concepts.adoc new file mode 100644 index 000000000..16fe9644c --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/concepts.adoc @@ -0,0 +1,165 @@ += MCP Tool Execution and Components +:description: Understand the MCP execution model, choose the right component type, and use traces for observability. +:page-aliases: ai-agents:mcp/remote/understanding-mcp-tools.adoc +:page-topic-type: concepts +:personas: ai_agent_developer, streaming_developer +// Reader journey: "I want to understand how it works" +// Learning objectives - what readers should know after reading this page: +:learning-objective-1: Describe the request/response execution model +:learning-objective-2: Choose the right component type for a use case +:learning-objective-3: Interpret MCP server traces for debugging and monitoring + +This page explains how MCP tools execute and how to choose the right component type for your use case. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +// Component mapping - single-sourced from partial +include::redpanda-connect:ai-agents:partial$mcp/concepts/component-mapping.adoc[] + +// Execution model - single-sourced from partial +include::redpanda-connect:ai-agents:partial$mcp/concepts/execution-model.adoc[] + +[[component-selection]] +== Choose the right component type + +// Component selection guide - single-sourced from partial +include::redpanda-connect:ai-agents:partial$mcp/concepts/component-selection.adoc[] + +[[execution-log]] +== Execution log and observability + +Every MCP server automatically emits OpenTelemetry traces to a topic called `redpanda.otel_traces`. These traces provide detailed observability into your MCP server's operations, creating a complete execution log. + +=== Traces and spans + +OpenTelemetry traces provide a complete picture of how a request flows through your system: + +* A _trace_ represents the entire lifecycle of a request (for example, a tool invocation from start to finish). +* A _span_ represents a single unit of work within that trace (such as a data processing operation or an external API call). +* A trace contains one or more spans organized hierarchically, showing how operations relate to each other. + +With 100% sampling, every operation is captured, creating a complete execution log that you can use for debugging, monitoring, and performance analysis. + +=== How Redpanda stores traces + +The `redpanda.otel_traces` topic stores OpenTelemetry spans in JSON format, following the https://opentelemetry.io/docs/specs/otel/protocol/[OpenTelemetry Protocol (OTLP)^] specification. A Protobuf schema named `redpanda.otel_traces-value` is also automatically registered with the topic, enabling clients to deserialize trace data correctly. + +The `redpanda.otel_traces` topic and its schema are managed automatically by Redpanda. If you delete either the topic or the schema, they are recreated automatically. However, deleting the topic permanently deletes all trace data, and the topic comes back empty. Do not produce your own data to this topic. It is reserved for OpenTelemetry traces. + +Each span in the execution log represents a specific operation performed by your MCP server, such as: + +* Tool invocation requests +* Data processing operations +* External API calls +* Error conditions +* Performance metrics + +=== Topic configuration and lifecycle + +The `redpanda.otel_traces` topic has a predefined retention policy. Configuration changes to this topic are not supported. If you modify settings, Redpanda reverts them to the default values. + +The topic persists in your cluster even after all MCP servers are deleted, allowing you to retain historical trace data for analysis. + +Trace data may contain sensitive information from your tool inputs and outputs. Consider implementing appropriate glossterm:ACL[,access control lists (ACLs)] for the `redpanda.otel_traces` topic, and review the data in traces before sharing or exporting to external systems. + +=== Understand the trace structure + +Each span captures a unit of work. Here's what a typical MCP tool invocation looks like: + +[,json] +---- +{ + "traceId": "71cad555b35602fbb35f035d6114db54", + "spanId": "43ad6bc31a826afd", + "name": "http_processor", + "attributes": [ + {"key": "city_name", "value": {"stringValue": "london"}}, + {"key": "result_length", "value": {"intValue": "198"}} + ], + "startTimeUnixNano": "1765198415253280028", + "endTimeUnixNano": "1765198424660663434", + "instrumentationScope": {"name": "rpcn-mcp"}, + "status": {"code": 0, "message": ""} +} +---- + +Key elements to understand: + +* **`traceId`**: Links all spans belonging to the same request. Use this to follow a tool invocation through its entire lifecycle. +* **`name`**: The tool name (`http_processor` in this example). This tells you which tool was invoked. +* **`instrumentationScope.name`**: When this is `rpcn-mcp`, the span represents an MCP tool. When it's `redpanda-connect`, it's internal processing. +* **`attributes`**: Context about the operation, like input parameters or result metadata. +* **`status.code`**: `0` means success, `2` means error. + +=== Parent-child relationships + +Traces show how operations relate. A tool invocation (parent) may trigger internal operations (children): + +[,json] +---- +{ + "traceId": "71cad555b35602fbb35f035d6114db54", + "spanId": "ed45544a7d7b08d4", + "parentSpanId": "43ad6bc31a826afd", + "name": "http", + "instrumentationScope": {"name": "redpanda-connect"}, + "status": {"code": 0, "message": ""} +} +---- + +The `parentSpanId` links this child span to the parent tool invocation. Both share the same `traceId`, so you can reconstruct the complete operation. + +=== Error events in traces + +When something goes wrong, traces capture error details: + +[,json] +---- +{ + "traceId": "71cad555b35602fbb35f035d6114db54", + "spanId": "ba332199f3af6d7f", + "parentSpanId": "43ad6bc31a826afd", + "name": "http_request", + "events": [ + { + "name": "event", + "timeUnixNano": "1765198420254169629", + "attributes": [{"key": "error", "value": {"stringValue": "type"}}] + } + ], + "status": {"code": 0, "message": ""} +} +---- + +The `events` array captures what happened and when. Use `timeUnixNano` to see exactly when the error occurred within the operation. + +=== Traces compared to audit logs + +OpenTelemetry traces are designed for observability and debugging, not audit logging or compliance. + +Traces provide: + +* Hierarchical view of request flow through your system (parent-child span relationships) +* Detailed timing information for performance analysis +* Ability to reconstruct execution paths and identify bottlenecks +* Insights into how operations flow through distributed systems + +Traces are not: + +* Immutable audit records for compliance purposes +* Designed for "who did what" accountability tracking + +For monitoring tasks like consuming traces, debugging failures, and measuring performance, see xref:ai-agents:mcp/remote/monitor-activity.adoc[]. + +== Next steps + +Continue your learning journey with these resources: + +* xref:ai-agents:mcp/remote/create-tool.adoc[]: Create custom tools step by step +* xref:ai-agents:mcp/remote/best-practices.adoc[]: Apply naming and design guidelines +* xref:ai-agents:mcp/remote/tool-patterns.adoc[]: Find reusable patterns +* xref:ai-agents:mcp/remote/troubleshooting.adoc[]: Diagnose common issues diff --git a/modules/ai-agents/pages/mcp/remote/create-tool.adoc b/modules/ai-agents/pages/mcp/remote/create-tool.adoc new file mode 100644 index 000000000..73a080526 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/create-tool.adoc @@ -0,0 +1,423 @@ += Create an MCP Tool +:description: Create an MCP tool with the correct YAML structure, metadata, and parameter mapping. +:page-topic-type: how-to +:personas: ai_agent_developer, streaming_developer, data_engineer +// Reader journey: "I want to create a tool for my AI agent" +// Learning objectives - what readers can do after reading this page: +:learning-objective-1: Create a tool with the correct structure and MCP metadata +:learning-objective-2: Map MCP parameters to component configuration fields using Bloblang +:learning-objective-3: Test tools using the MCP Inspector + +After xref:ai-agents:mcp/remote/quickstart.adoc[deploying your first MCP server], create custom tools that AI clients can discover and invoke. This guide walks you through the process using any Redpanda Connect component. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +== Prerequisites + +* A Redpanda Cloud cluster with *Remote MCP* enabled. +* You can describe the MCP execution model (see xref:ai-agents:mcp/remote/concepts.adoc#execution-model[The MCP execution model]) +* You have chosen the right component type for your use case (see xref:ai-agents:mcp/remote/concepts.adoc#component-selection[Choose the right component type]) + +== Create the tool + +In Redpanda Cloud, you create tools directly in the Cloud Console or using the Data Plane API. + +[tabs] +====== +Cloud Console:: ++ +-- +. Log in to the link:https://cloud.redpanda.com/[Redpanda Cloud Console^]. + +. Navigate to *Remote MCP* and either create a new MCP server or edit an existing one. + +. In the *Tools* section, click *Add Tool*. + +. Enter the YAML configuration for your tool. + +. Click *Lint* to validate your configuration. + +. Click *Save* to deploy the tool. +-- + +Data Plane API:: ++ +-- +Use the link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_createmcpserver[Create MCP Server] or link:/api/doc/cloud-dataplane/operation/operation-mcpserverservice_updatemcpserver[Update MCP Server] endpoints to add tools programmatically. +-- +====== + +[[yaml-structure]] +== Add the tool structure + +An MCP tool wraps a xref:develop:connect/components/about.adoc[Redpanda Connect component] and exposes it to AI clients. Each tool has three parts: + +* **Label**: The tool name AI clients see +* **Component configuration**: A Redpanda Connect component (processor, input, output, or cache) that does the work +* **MCP metadata**: Describes the tool's purpose and parameters for AI clients + +Here's an example using the xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor]: + +[source,yaml] +---- +label: lookup-customer # <1> + +sql_select: # <2> + driver: postgres + dsn: "${secrets.DATABASE_URL}" + table: customers + columns: ["id", "name", "email", "plan"] + where: id = ? + args_mapping: '[this.customer_id]' + +meta: # <3> + mcp: + enabled: true + description: "Look up a customer by ID and return their profile." + properties: + - name: customer_id + type: string + description: "The customer's unique identifier" + required: true +---- + +<1> **Label**: Becomes the tool name. +<2> **Component**: The `sql_select` processor configured to query a database. +<3> **MCP metadata**: Tells AI clients what this tool does and what parameters it accepts. + +Each tool configuration must contain exactly one component. The component type is inferred from the type you select when creating or editing the MCP server. The component can be a processor, input, output, or cache. + +The following sections show how to structure tools for each component type. + +=== Label naming rules + +include::redpanda-connect:ai-agents:partial$mcp/create-tool/label-naming-rules.adoc[] + +=== Component types + +xref:develop:connect/components/processors/about.adoc[Processors] transform, filter, or enrich data. Use a `processors:` array with one or more processors: + +.Processor tool +[source,yaml] +---- +label: enrich-order + +processors: + - http: + url: "https://api.example.com/lookup" + verb: GET + +meta: + mcp: + enabled: true + description: "Enrich order with customer data" +---- + +xref:develop:connect/components/inputs/about.adoc[Inputs] read data from sources, xref:develop:connect/components/outputs/about.adoc[outputs] write data to destinations, and xref:develop:connect/components/caches/about.adoc[caches] store and retrieve data. Define these components directly at the top level: + +.Input tool +[source,yaml] +---- +label: read-events + +redpanda: # <1> + seed_brokers: ["${REDPANDA_BROKERS}"] + topics: ["events"] + consumer_group: "mcp-reader" + tls: + enabled: true + sasl: + - mechanism: SCRAM-SHA-256 + username: "${secrets.MCP_USERNAME}" + password: "${secrets.MCP_PASSWORD}" + +meta: + mcp: + enabled: true + description: "Read events from Redpanda" +---- +<1> The component name (`redpanda`) is at the top level, not wrapped in `input:`. + +.Output tool +[source,yaml] +---- +label: publish-event + +redpanda: + seed_brokers: ["${REDPANDA_BROKERS}"] + topic: "processed-events" + tls: + enabled: true + sasl: + - mechanism: SCRAM-SHA-256 + username: "${secrets.MCP_USERNAME}" + password: "${secrets.MCP_PASSWORD}" + +meta: + mcp: + enabled: true + description: "Publish event to Redpanda" +---- + +.Cache tool +[source,yaml] +---- +label: session-cache + +memory: + default_ttl: 300s + +meta: + mcp: + enabled: true + description: "In-memory cache for session data" +---- + +Outputs can include a `processors:` section to transform data before publishing: + +.Output tool with processors +[source,yaml] +---- +label: publish-with-timestamp + +processors: + - mutation: | + root = this + root.published_at = now() + +redpanda: + seed_brokers: ["${REDPANDA_BROKERS}"] + topic: "processed-events" + tls: + enabled: true + sasl: + - mechanism: SCRAM-SHA-256 + username: "${secrets.MCP_USERNAME}" + password: "${secrets.MCP_PASSWORD}" + +meta: + mcp: + enabled: true + description: "Add timestamp and publish to Redpanda" +---- + +See xref:ai-agents:mcp/remote/tool-patterns.adoc#outputs-with-processors[outputs with processors] for more examples. + +Do not wrap components in `input:`, `output:`, or `cache:` blocks. This syntax is for pipelines, not MCP tools. + +[[mcp-metadata]] +=== MCP metadata fields + +The `meta.mcp` block defines how AI clients discover and interact with your tool. These fields control tool visibility, naming, and input parameters. + +include::redpanda-connect:ai-agents:partial$mcp/create-tool/mcp-metadata-fields-table.adoc[] + +==== Property fields + +include::redpanda-connect:ai-agents:partial$mcp/create-tool/property-fields-table.adoc[] + +[[property-restrictions]] +==== Property restrictions by component type + +include::redpanda-connect:ai-agents:partial$mcp/create-tool/property-restrictions-table.adoc[] + +[[parameter-mapping]] +== Map parameters to component fields + +When an AI client calls your tool, the `arguments` object becomes the message body. You can access these arguments using xref:develop:connect/guides/bloblang/about.adoc[Bloblang], but the syntax depends on where you're using it: + +* **Inside Bloblang contexts** (mutation, mapping, args_mapping): Use `this.field_name` +* **Inside string fields** (URLs, topics, headers): Use interpolation `${! json("field_name") }` + +=== In Bloblang contexts + +Use `this` to access message fields directly in processors like `mutation`, `mapping`, or in `args_mapping` fields: + +[source,yaml] +---- +mutation: | + root.search_query = this.query.lowercase() + root.max_results = this.limit.or(10) +---- + +[source,yaml] +---- +sql_select: + table: orders + where: customer_id = ? AND status = ? + args_mapping: '[this.customer_id, this.status.or("active")]' +---- + +=== In string fields (interpolation) + +Use `${! ... }` interpolation to embed Bloblang expressions inside string values like URLs or topic names: + +[source,yaml] +---- +http: + url: 'https://api.weather.com/v1/current?city=${! json("city") }&units=${! json("units").or("metric") }' +---- + +[source,yaml] +---- +redpanda: + seed_brokers: ["${REDPANDA_BROKERS}"] # <1> + topic: '${! json("topic_name") }' # <2> +---- +<1> `$\{VAR}` without `!` is environment variable substitution, not Bloblang. +<2> `${! ... }` with `!` is Bloblang interpolation that accesses message data. + +TIP: For more on Bloblang syntax, see xref:develop:connect/guides/bloblang/about.adoc[]. For interpolation details, see xref:develop:connect/configuration/interpolation.adoc[]. + +=== Provide defaults for optional parameters + +Use `.or(default)` to handle missing optional parameters: + +[source,yaml] +---- +mutation: | + root.city = this.city # Required - will error if missing + root.units = this.units.or("metric") # Optional with default + root.limit = this.limit.or(10).number() # Optional, converted to number +---- + +Declare which parameters are required in your `meta.mcp.properties`: + +[source,yaml] +---- +properties: + - name: city + type: string + description: "City name to look up" + required: true + - name: units + type: string + description: "Temperature units: 'metric' or 'imperial' (default: metric)" + required: false +---- + +[[secrets]] +== Use the Secrets Store + +Never hardcode credentials, API keys, or connection strings in your tool configurations. Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] to securely manage sensitive values. + +Reference secrets using `${secrets.SECRET_NAME}` syntax: + +[source,yaml] +---- +http: + url: "https://api.example.com/data" + headers: + Authorization: "Bearer ${secrets.API_TOKEN}" + +sql_select: + driver: postgres + dsn: "${secrets.DATABASE_URL}" + table: customers +---- + +When you add secret references to your tool configuration, the Cloud Console automatically detects them and provides an interface to create the required secrets. + +=== Secrets best practices + +* Use uppercase snake_case for secret names (for example, `DATAPLANE_TOKEN`, `API_KEY`). +* Rotate secrets periodically. +* Follow the principle of least privilege. Only request the scopes and roles your tool actually needs. + +See xref:ai-agents:mcp/remote/best-practices.adoc#secrets[Secrets management] for more guidance. + +== Test the tool + +. Click *Lint* to validate your configuration. + +. Deploy the MCP server. + +. Use the *MCP Inspector* tab to test tool calls: ++ +* Select the tool from the list +* Enter test parameter values +* Click *Run Tool* to execute +* Review the response + +. Connect an AI client and verify the tool appears: ++ +[,bash] +---- +rpk cloud mcp proxy \ + --cluster-id \ + --mcp-server-id \ + --install --client claude-code +---- + +. Test end-to-end with realistic prompts to verify the AI client uses your tool correctly. + +== Complete example + +Here's a complete tool that wraps the `http` processor to fetch weather data: + +[source,yaml] +---- +label: get-weather + +processors: + # Validate and sanitize input + - label: validate_city + mutation: | + root.city = if this.city.or("").trim() == "" { + throw("city is required") + } else { + this.city.trim().lowercase().re_replace_all("[^a-z\\s\\-]", "") + } + root.units = this.units.or("metric") + + # Fetch weather data + - label: fetch_weather + try: + - http: + url: 'https://wttr.in/${! json("city") }?format=j1' + verb: GET + timeout: 10s + + - mutation: | + root.weather = { + "location": this.nearest_area.0.areaName.0.value, + "country": this.nearest_area.0.country.0.value, + "temperature_c": this.current_condition.0.temp_C, + "temperature_f": this.current_condition.0.temp_F, + "condition": this.current_condition.0.weatherDesc.0.value, + "humidity": this.current_condition.0.humidity, + "wind_kph": this.current_condition.0.windspeedKmph + } + + # Handle errors gracefully + - label: handle_errors + catch: + - mutation: | + root.error = true + root.message = "Failed to fetch weather: " + error() + +meta: + mcp: + enabled: true + description: "Get current weather for a city. Returns temperature, conditions, humidity, and wind speed." + properties: + - name: city + type: string + description: "City name (e.g., 'London', 'New York', 'Tokyo')" + required: true + - name: units + type: string + description: "Temperature units: 'metric' or 'imperial' (default: metric)" + required: false +---- + +== Next steps + +* xref:ai-agents:mcp/remote/best-practices.adoc[]: Apply naming and design guidelines. +* xref:ai-agents:mcp/remote/tool-patterns.adoc[]: Find patterns for databases, APIs, and Redpanda. +* xref:ai-agents:mcp/remote/troubleshooting.adoc[]: Diagnose common issues. +* xref:develop:connect/components/about.adoc[]: Browse all available components. diff --git a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc b/modules/ai-agents/pages/mcp/remote/developer-guide.adoc deleted file mode 100644 index 0db151426..000000000 --- a/modules/ai-agents/pages/mcp/remote/developer-guide.adoc +++ /dev/null @@ -1,407 +0,0 @@ -= Build Remote MCP Servers in Redpanda Cloud -:description: Learn how to write and deploy MCP servers in Redpanda Cloud. This guide covers concepts, patterns, and best practices. - - -This guide teaches you how to build xref:ai-agents:mcp/remote/overview.adoc[MCP servers] that are managed in Redpanda Cloud. MCP servers run inside your Redpanda Cloud cluster and expose tools that AI clients can call using MCP. - -== Prerequisites - -* Access to a Redpanda Cloud cluster. -* Ability to manage the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] entries. -* At least version 25.2.5 of xref:manage:rpk/rpk-install.adoc[`rpk` installed on your computer]. -* (Optional) An AI assistant like Claude or Claude Code for testing. - -TIP: For a quickstart, see xref:ai-agents:mcp/remote/quickstart.adoc[]. - -You should also be familiar with YAML, HTTP APIs, and basic event-stream processing concepts. - -== Concepts and architecture - -Remote MCP server:: A managed, hosted MCP service deployed to your cluster. You write each tool's logic using Redpanda Connect configuration and annotate them with MCP metadata so clients can discover and invoke them. - -Redpanda Connect:: A framework for building event-driven data pipelines. Tools inside your MCP servers are implemented as Redpanda Connect configurations that run inside your Redpanda Cloud cluster. - -Tool:: A single request/response operation exposed to MCP. - -Secrets:: Credentials and tokens that you want to use in your tools. These values must be stored in the Secrets Store and referenced as `${secrets.NAME}`. Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). - -== Development workflow - -. <>: Navigate to Remote MCP in Redpanda Cloud Console, create a new server, and configure server metadata (display name, description, tags, resources, service account). -. <>: Annotate MCP metadata with `meta.mcp.enabled: true`, provide a concise description, and declare parameters. -. <>: Create secrets in the Secrets Store for any credentials. Never inline secrets in YAML. -. <>: Build your MCP tool using Redpanda Connect components. -. <>: Validate your configuration, catch schema errors, then deploy and test with the MCP Inspector. -. <>: Install a proxy entry with `rpk cloud mcp proxy` in your AI assistant MCP configuration, or connect with your own MCP client. -. <>: Add temporary `log` processors, adjust timeouts/retries, and right-size resources. - -[[access]] -== Create MCP server and configure metadata - -. Log into your link:https://cloud.redpanda.com[Redpanda Cloud Console^]. -. Select your cluster from the cluster list. -. In the left sidebar, navigate to *Remote MCP*. -. If this is your first MCP server, you'll see an empty list with a *Create MCP Server* button. -. Click *Create MCP Server* and configure the *Server Metadata*: -+ -* *Display Name*: A human-friendly name for your server (for example `weather-tool`, `customer-lookup`). -* *Description*: Brief description of what your server does. -* *Tags*: Key-value pairs for organizing and categorizing your MCP servers. See <> for details. -* *Resources*: Choose the compute resources (XSmall through XLarge) based on your expected workload. -* *Service Account*: A service account name is pre-filled and will be created automatically. -+ -. Click *Next* to define your tools using YAML configuration. See <> to design your tool contract. - -[[tags]] -== Organize servers with tags - -Tags are key-value metadata that help you organize, categorize, and filter your MCP servers. Use tags to track ownership, environment, cost allocation, or any other organizational dimension relevant to your team. - -=== Common tagging strategies - -[cols="1,2,2"] -|=== -| Strategy | Description | Examples - -| Ownership -| Track which team or individual owns the MCP server. -| `owner=platform-team`, `team=data-engineering` - -| Environment -| Identify the deployment environment. -| `env=production`, `env=staging`, `env=development` - -| Cost allocation -| Allocate costs to specific departments or projects. -| `cost-center=marketing`, `project=analytics` - -| Purpose -| Describe the use case or data domain. -| `purpose=customer-data`, `use-case=real-time-alerts` -|=== - -=== Reserved tag names - -The following tag names are reserved and cannot be used. These tags are system-managed: - -* `service_account_id`: Automatically assigned to track the service account associated with the MCP server. -* `secret_id`: Automatically assigned to track the service account secret used by the MCP server. - -=== Best practices - -* Use consistent naming conventions across your organization (for example, always lowercase with hyphens or underscores). -* Document your tagging strategy so all team members use tags consistently. -* Use tags for filtering and searching MCP servers in the Cloud Console and API. -* Review and update tags periodically to keep them accurate as servers evolve. - -[[contract]] -== Design the tool contract and MCP metadata - -Each MCP tool must declare its interface using `meta.mcp` metadata. This metadata allows AI clients to discover and invoke the tool correctly. - -Define a clear, stable interface for each tool. Keep the description task-oriented and keep properties to a minimum. - -[source,yaml] ----- -meta: - mcp: - enabled: true <1> - description: "Publishes a message to a specified Redpanda topic" <2> - properties: <3> - - name: message - type: string - description: "The message content to publish" - required: true - - name: topic_name - type: string - description: "The Redpanda topic to publish to" - required: true ----- - -<1> Set `meta.mcp.enabled: true` to expose the tool using MCP. -<2> A concise description that explains what the tool does. This should be understandable by an AI model. -<3> Define the properties that AI clients can provide when calling this tool. -+ -Properties define the inputs that AI clients can provide when calling your tool. Each property has: - -* **name**: The parameter name AI clients will use -* **type**: Data type (`string`, `number`, or `boolean`) -* **description**: Human-readable explanation of what this property does -* **required**: Whether the property is mandatory (`true`) or optional (`false`) - -Property guidance: - -* Use descriptive names that clearly indicate the property's purpose. -* Provide clear descriptions that explain both the purpose and any constraints. -* Mark only mandatory fields as required. Prefer optional properties with sensible defaults. -* Document default values in the description and implement them in your tool logic. -* Consider component type restrictions: `input` components only support a `count` property, `cache` components have no custom properties, and `output` components receive properties as arrays for batch operations. - -After defining your tool contract, implement the logic to handle input validation, defaults, and the main processing steps. - -[[provision-secrets]] -== Provision secrets - -All credentials and sensitive values must be stored in the Redpanda Cloud xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. - -=== Creating secrets in the Cloud Console - -When you reference secrets in your tool's YAML configuration using `${secrets.SECRET_NAME}`, the Cloud Console automatically detects these references and provides an interface next to the configuration editor where you can: - -. Enter the secret values directly -. Create the secrets in the Secrets Store with a single click - -This inline secret creation workflow simplifies the development process by letting you configure secrets as you build your tools, without navigating away to the Secrets Store. - -Alternatively, you can manually create and manage secrets directly in the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. - -=== Best practices - -- Reference secrets as `${secrets.NAME}` in your YAML configuration. -- Never commit secrets to Git or reference them directly inline in configuration files. -- Use uppercase snakecase for secret names (such as `DATAPLANE_TOKEN`). -- Rotate secrets in the Secrets Store as needed. -- Only request the scopes/roles required by your tool (principle of least privilege). - -See an example of using secrets in xref:ai-agents:mcp/remote/pipeline-patterns.adoc#secrets[MCP patterns]. - -[[pipeline-patterns]] -== Build your MCP tool - -Use Redpanda Connect components to implement the logic of your MCP tool. Here are some best practices: - -* **Single responsibility**: Each tool should do one thing well. -* **Descriptive naming**: Use clear, specific labels like `fetch-user-profile` instead of generic names like `get-data`. The top-level label becomes the tool's name in the MCP server. -* **Input validation**: Always validate and sanitize user inputs using xref:develop:connect/guides/bloblang/about.adoc[Bloblang]. -* **Error handling**: Provide meaningful error messages. -* **Documentation**: Write clear descriptions that explain what the tool does and what it returns. - -For common patterns and examples, see xref:ai-agents:mcp/remote/pipeline-patterns.adoc[]. - -Here's a complete example that demonstrates best practices: - -[source,yaml] ----- -include::ai-agents:example$weather_service.yaml[] ----- - -=== YAML configuration rules - -Each YAML configuration (tool) should contain exactly one component type. The component type is inferred from the type you choose in the dropdown when creating or editing the MCP server. Valid component types are: - -* xref:develop:connect/components/inputs/about.adoc[`input`] (for data sources) -* xref:develop:connect/components/outputs/about.adoc[`output`] (for data sinks) -* xref:develop:connect/components/processors/about.adoc[`processor`] (for data transformations and data access) -* xref:develop:connect/components/caches/about.adoc[`cache`] (for caching intermediate results) - -=== Property restrictions by component type - -Different component types have different property capabilities when exposed as MCP tools: - -[cols="1,2,2"] -|=== -| Component Type | Property Support | Details - -| `input` -| Only supports `count` property -| AI clients can specify how many messages to read, but you cannot define custom properties. - -| `cache` -| No custom properties -| Properties are hardcoded to `key` and `value` for cache operations. - -| `output` -| Custom properties supported -| AI sees properties as an array for batch operations: `[{prop1, prop2}, {prop1, prop2}]`. - -| `processor` -| Custom properties supported -| You can define any properties needed for data processing operations. -|=== - -=== Configuration examples - -.Correct example -[source,yaml] ----- -label: event-reader -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "events" ] - consumer_group: "mcp-reader" - -meta: - mcp: - enabled: true - description: "Consume events from Redpanda" ----- - -.Correct example -[source,yaml] ----- -label: fetch-example-data -processors: - - label: safe_operation - try: - - http: - url: "https://api.example.com/data" - timeout: "10s" - - mutation: | - root = this.merge({"processed": true}) - - - label: handle_errors - catch: - - mutation: | - root = { - "error": "Operation failed", - "details": error() - } ----- - -.Incorrect (do not include the input wrapper) -[source,yaml] ----- -label: incorrect-example -input: - redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "events" ] ----- - -.Incorrect (multiple component types in one file) -[source,yaml] ----- -label: incorrect-example -input: - redpanda: { ... } -processors: - - mutation: { ... } -output: - redpanda: { ... } ----- - -[[lint]] -== Lint and deploy your MCP server - -=== Lint your configuration - -Before deploying, validate your YAML configuration to catch syntax errors and schema violations: - -. In the Redpanda Cloud Console, after entering your YAML configuration, click *Lint*. -. The linter checks for: -+ -* Valid YAML syntax -* Correct Redpanda Connect component schemas -* Required MCP metadata fields -* Secret reference formatting -* Component configuration completeness - -. Fix any errors shown in the linting output before proceeding to deployment. -. A successful lint shows no errors and confirms your configuration is ready to deploy. - -TIP: Always lint your configuration after making changes. This catches issues early and prevents deployment failures. - -=== Deploy your MCP server - -. Click *Create MCP Server* to deploy the server. -. Monitor the *Status* column. It will show one of: -+ -* *Starting*: Server is being deployed. -* *Running*: Server is active and ready to receive requests. -* *Error*: Check logs for deployment issues. - -=== Test with MCP Inspector - -. When your server status shows *Running*, click on the server name. -. Navigate to the *MCP Inspector* tab. -. This built-in tool lets you: -+ -* View all available tools your server exposes. -* Test tool calls with sample parameters. -* See request/response data. -* Debug issues without setting up external clients. - -=== Manage your MCP server - -From the MCP Servers list, you can: - -* *View logs*: Click the server name, then go to the *Logs* tab to see execution logs and errors. -* *Update configuration*: Edit the YAML and redeploy. -* *Start/Stop*: Control server lifecycle. -* *Delete*: Remove the server and clean up resources. - -See xref:ai-agents:mcp/remote/admin-guide.adoc[]. - -[[test]] -== Authenticate and connect your MCP client - -To connect your local MCP client to your MCP server: - -. First, authenticate to Redpanda Cloud: -+ -[source,bash] ----- -rpk cloud login ----- -+ -This opens a browser window for sign-in and stores your authentication token locally. - -. Install the MCP proxy for your client (Claude/Claude Code). -+ -For BYOC and Dedicated clusters, use: -+ -[,bash] ----- -rpk cloud mcp proxy \ ---cluster-id \ <1> ---mcp-server-id \ <2> ---install --client claude-code ----- -+ -For Serverless clusters, use: -+ -[,bash] ----- -rpk cloud mcp proxy \ ---serverless-cluster-id \ <1> ---mcp-server-id \ <2> ---install --client claude-code ----- -+ --- -<1> The target Redpanda Cloud cluster ID where your MCP server is hosted. -<2> The unique ID of your deployed MCP server --- -+ -TIP: You can find this command and the IDs in the *Connection* tab of your MCP server in Redpanda Cloud. -+ -The proxy acts as a bridge between your local client and the MCP server running in your cluster. It: -+ -* Connects to your MCP server using your authentication token -* Discovers and registers all tools from the Remote server locally -* Proxies MCP requests from your client to the Remote server -* Handles authentication by injecting your token into each request - -. Restart your client and invoke your tool. - -:note-caption: Building your own agent? - -[NOTE] -==== -You can implement the auth workflow directly against Redpanda Cloud APIs and skip the use of `rpk`. The proxy is a convenience, not a requirement for MCP servers. For code examples, see the *Connection* tab in your MCP server in Redpanda Cloud. -==== - -:note-caption: Note - -[[observe]] -== Observe and debug your MCP tools - -You can view execution logs in the Redpanda Cloud Console under your MCP server's *Logs* tab. - -== Suggested reading - -* xref:develop:connect/components/about.adoc[Redpanda Connect components reference] -* xref:develop:connect/guides/bloblang/about.adoc[Bloblang language guide] -* xref:develop:connect/configuration/secret-management.adoc[Secret management in Redpanda Connect] -* xref:reference:rpk/rpk-cloud/rpk-cloud-login.adoc[`rpk cloud login` command reference] -* xref:reference:rpk/rpk-cloud/rpk-cloud-mcp-proxy.adoc[`rpk cloud mcp proxy` command reference] diff --git a/modules/ai-agents/pages/mcp/remote/index.adoc b/modules/ai-agents/pages/mcp/remote/index.adoc index 39771ad81..1c77473f2 100644 --- a/modules/ai-agents/pages/mcp/remote/index.adoc +++ b/modules/ai-agents/pages/mcp/remote/index.adoc @@ -1,3 +1,3 @@ = Remote MCP Servers for Redpanda Cloud -:description: Enable AI assistants to directly interact with your Redpanda Cloud clusters and streaming data. +:description: Enable AI agents to directly interact with your Redpanda Cloud clusters and streaming data. :page-layout: index diff --git a/modules/ai-agents/pages/mcp/remote/manage-servers.adoc b/modules/ai-agents/pages/mcp/remote/manage-servers.adoc index e0aeb6a29..d09c0ab24 100644 --- a/modules/ai-agents/pages/mcp/remote/manage-servers.adoc +++ b/modules/ai-agents/pages/mcp/remote/manage-servers.adoc @@ -1,5 +1,12 @@ = Manage Remote MCP Servers :description: Learn how to edit, stop, start, and delete MCP servers in Redpanda Cloud. +:page-topic-type: how-to +:personas: platform_admin, ai_agent_developer +// Reader journey: "I operate and maintain" +// Learning objectives - what readers can accomplish from this page: +:learning-objective-1: Edit MCP server configurations +:learning-objective-2: Stop and start MCP servers +:learning-objective-3: Delete MCP servers safely This guide shows you how to manage the lifecycle of MCP servers, including editing configurations, pausing servers to save costs, and permanent deletion. @@ -13,7 +20,7 @@ You can update the configuration, resources, or metadata of an MCP server at any [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . In the Redpanda Cloud Console, navigate to *Remote MCP*. @@ -56,7 +63,7 @@ Stopping a server pauses all tool execution and releases compute resources, but [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . In the Redpanda Cloud Console, navigate to *Remote MCP*. @@ -88,7 +95,7 @@ Resume a stopped server to restore its functionality. [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . In the Redpanda Cloud Console, navigate to *Remote MCP*. @@ -119,7 +126,7 @@ Deleting a server is permanent and cannot be undone. All configuration, tools, a [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . In the Redpanda Cloud Console, navigate to *Remote MCP*. @@ -151,4 +158,4 @@ Deletion is immediate and permanent. Make sure you have backed up any important * xref:ai-agents:mcp/remote/scale-resources.adoc[Scale MCP server resources] to optimize performance and costs. * xref:ai-agents:mcp/remote/monitor-activity.adoc[Monitor MCP server activity] using OpenTelemetry traces. -* xref:ai-agents:mcp/remote/developer-guide.adoc[Build custom tools] for your specific use cases. +* xref:ai-agents:mcp/remote/best-practices.adoc[Learn best practices] for building robust tools. diff --git a/modules/ai-agents/pages/mcp/remote/monitor-activity.adoc b/modules/ai-agents/pages/mcp/remote/monitor-activity.adoc index fca49c323..11e67fcf1 100644 --- a/modules/ai-agents/pages/mcp/remote/monitor-activity.adoc +++ b/modules/ai-agents/pages/mcp/remote/monitor-activity.adoc @@ -1,73 +1,34 @@ -= Monitor Remote MCP Server Activity -:description: Learn how to monitor MCP servers using OpenTelemetry traces, track tool invocations, debug failures, and integrate with observability platforms. += Monitor MCP Server Activity +:description: How to consume traces, track tool invocations, measure performance, and debug failures in MCP servers. +:page-topic-type: how-to +:personas: platform_admin, ai_agent_developer, data_engineer +// Reader journey: "I need to accomplish X" +// Learning objectives - what readers can DO with this guide: +:learning-objective-1: Consume traces from the execution log +:learning-objective-2: Track tool invocations and measure performance +:learning-objective-3: Debug tool failures using trace data -Every MCP server automatically emits OpenTelemetry traces to a topic called `redpanda.otel_traces`. These traces provide detailed observability into your MCP server's operations, including tool invocations, processing times, errors, and resource usage. +This guide shows you how to monitor your MCP servers using the execution log. -== Prerequisites - -You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. - -== Execution log - -When you create an MCP server, Redpanda captures a complete execution log by recording all operations as OpenTelemetry traces with 100% sampling. This execution log is stored in the `redpanda.otel_traces` topic. - -=== Traces and spans - -OpenTelemetry traces provide a complete picture of how a request flows through your system: - -* A *trace* represents the entire lifecycle of a request (for example, a tool invocation from start to finish). -* A *span* represents a single unit of work within that trace (such as a data processing operation or an external API call). -* A trace contains one or more spans organized hierarchically, showing how operations relate to each other. - -With 100% sampling, every operation is captured, creating a complete execution log that you can use for debugging, monitoring, and performance analysis. - -=== How Redpanda stores traces - -The `redpanda.otel_traces` topic stores OpenTelemetry spans in JSON format, following the https://opentelemetry.io/docs/specs/otel/protocol/[OpenTelemetry Protocol (OTLP)^] specification. A Protobuf schema named `redpanda.otel_traces-value` is also automatically registered with the topic, enabling clients to deserialize trace data correctly. - -The `redpanda.otel_traces` topic and its schema are managed automatically by Redpanda. If you delete either the topic or the schema, they are recreated automatically. However, deleting the topic permanently deletes all trace data, and the topic comes back empty. Do not produce your own data to this topic. It is reserved for OpenTelemetry traces. - -Each span in the execution log represents a specific operation performed by your MCP server, such as: - -* Tool invocation requests -* Data processing operations -* External API calls -* Error conditions -* Performance metrics +Use this guide to: -=== Topic configuration and lifecycle +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} -The `redpanda.otel_traces` topic has a predefined retention policy. Configuration changes to this topic are not supported. If you modify settings, Redpanda reverts them to the default values. +For conceptual background on traces, spans, and the trace data structure, see xref:ai-agents:mcp/remote/concepts.adoc#execution-log[Execution log and observability]. -The topic persists in your cluster even after all MCP servers are deleted, allowing you to retain historical trace data for analysis. - -You can create glossterm:ACL[,access control lists (ACLs)] to control access to the `redpanda.otel_traces` topic using standard Redpanda ACL operations. - -Trace data may contain sensitive information from your tool inputs and outputs. Consider implementing appropriate access controls and review the data in traces before sharing or exporting to external systems. - -=== Traces compared to audit logs - -OpenTelemetry traces are designed for observability and debugging, not audit logging or compliance: - -Traces provide: - -* Hierarchical view of request flow through your system (parent-child span relationships) -* Detailed timing information for performance analysis -* Ability to reconstruct execution paths and identify bottlenecks -* Insights into how operations flow through distributed systems - -Traces are not: +== Prerequisites -* Immutable audit records for compliance purposes -* Designed for "who did what" accountability tracking +You must have an existing MCP server. If you do not have one, see xref:ai-agents:mcp/remote/quickstart.adoc[]. == Consume traces from the execution log -You can consume traces from the `redpanda.otel_traces` topic using any Kafka-compatible client or the Redpanda Console. +MCP servers emit OpenTelemetry traces to the `redpanda.otel_traces` topic. You can consume these traces using any Kafka-compatible client or the Redpanda Cloud Console. [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . In the Redpanda Cloud Console, navigate to *Topics*. @@ -96,190 +57,41 @@ Use the link:/api/doc/cloud-dataplane/topic/topic-quickstart[Data Plane API] to -- ===== -== Trace data structure - -Each trace message contains an OpenTelemetry span with the following structure: - -[,json] ----- -{ - "traceId": "71cad555b35602fbb35f035d6114db54", - "spanId": "43ad6bc31a826afd", - "parentSpanId": "", - "name": "http_processor", - "kind": 1, - "startTimeUnixNano": "1765198415253280028", - "endTimeUnixNano": "1765198424660663434", - "attributes": [ - { - "key": "city_name", - "value": {"stringValue": "london"} - }, - { - "key": "result_length", - "value": {"intValue": "198"} - } - ], - "status": { - "code": 0, - "message": "" - }, - "instrumentationScope": { - "name": "rpcn-mcp" - }, - "flags": 1 -} ----- - -* `traceId`: Unique identifier for the entire trace (hex-encoded string). -* `spanId`: Unique identifier for this specific span (hex-encoded string). -* `parentSpanId`: Links child spans to parent operations (empty string if root span). -* `name`: Descriptive name of the operation (typically the tool name in the case of MCP servers). -* `kind`: Span kind as integer (see the `redpanda.otel_traces-value` schema for details). -* `startTimeUnixNano` / `endTimeUnixNano`: Timing information in nanoseconds since Unix epoch. -* `attributes`: Array of key-value pairs with contextual information about the operation. -* `status`: Operation status with `code` (0 = OK, 2 = ERROR) and optional `message`. -* `instrumentationScope`: Identifies the library or component that created the span. - -=== Identifying MCP servers and tools - -All MCP servers in a cluster write traces to the same `redpanda.otel_traces` topic. Each trace has a unique `traceId`, and you can identify which tool created the trace by examining the span fields: - -* When `instrumentationScope.name` is `rpcn-mcp`, the span represents an MCP tool invocation. -* The `name` field contains the MCP tool name (for example, `weather`, `http_processor`, `generate_input`). -* Child spans reference their parent using `parentSpanId` and share the same `traceId` - -For example, a span with `instrumentationScope.name: "rpcn-mcp"` and `name: "weather"` indicates the `weather` tool was invoked. Child spans created by Redpanda Connect components show `instrumentationScope.name: "redpanda-connect"` and represent internal processing steps. - -=== Real trace examples - -Here are actual traces from MCP tool executions showing the parent-child span relationships and detailed attributes. - -**Successful tool execution:** - -This trace shows an `http_processor` tool that fetched weather data for London. The parent span includes child spans for mutations, HTTP requests, and processing: - -[source,json] ----- -{ - "attributes": [ - { - "key": "city_name", - "value": {"stringValue": "london"} - }, - { - "key": "result_prefix", - "value": {"stringValue": "{\"city\":\"london\",\"description\":\"Partly cloudy\",\"feels_like\":12,\"humidity\":88,\"metadata\":{\"fetched_at\":\"2025-12-08T12:53:44.660Z\""} - }, - { - "key": "result_length", - "value": {"intValue": "198"} - } - ], - "endTimeUnixNano": "1765198424660663434", - "flags": 1, - "instrumentationScope": {"name": "rpcn-mcp"}, - "kind": 1, - "name": "http_processor", - "spanId": "43ad6bc31a826afd", - "startTimeUnixNano": "1765198415253280028", - "status": {"code": 0, "message": ""}, - "traceId": "71cad555b35602fbb35f035d6114db54" -} ----- - -The parent span (`http_processor`) has child spans showing the processing pipeline: - -[,json] ----- -{ - "attributes": [], - "endTimeUnixNano": "1765198424659619511", - "flags": 1, - "instrumentationScope": {"name": "redpanda-connect"}, - "kind": 1, - "name": "http", - "parentSpanId": "43ad6bc31a826afd", - "spanId": "ed45544a7d7b08d4", - "startTimeUnixNano": "1765198415253319496", - "status": {"code": 0, "message": ""}, - "traceId": "71cad555b35602fbb35f035d6114db54" -} ----- - -* The parent span (`http_processor`) includes custom attributes like `city_name` and `result_length` that help debug and monitor tool behavior. -* Child spans use the same `traceId` but different `spanId` values. -* The `parentSpanId` in child spans matches the parent's `spanId`, creating the hierarchy. -* Duration can be calculated: `(1765198424660663434 - 1765198415253280028) / 1000000 = 9407ms`. - -**Trace with error event:** - -This trace shows an HTTP request that encountered an error during execution: - -[,json] ----- -{ - "attributes": [], - "endTimeUnixNano": "1765198423080903265", - "events": [ - { - "attributes": [ - { - "key": "error", - "value": {"stringValue": "type"} - } - ], - "name": "event", - "timeUnixNano": "1765198420254169629" - } - ], - "flags": 1, - "instrumentationScope": {"name": "redpanda-connect"}, - "kind": 1, - "name": "http_request", - "parentSpanId": "43ad6bc31a826afd", - "spanId": "ba332199f3af6d7f", - "startTimeUnixNano": "1765198415253325782", - "status": {"code": 0, "message": ""}, - "traceId": "71cad555b35602fbb35f035d6114db54" -} ----- - -* The span still shows `status.code: 0` (success) overall. -* However, the `events` array contains error information with `error: type`. -* This indicates a recoverable error or warning during execution. -* The event's `timeUnixNano` shows exactly when the error occurred. - -== Common monitoring scenarios - -=== Track tool invocations +== Track tool invocations Monitor which tools are being called and how often: . Consume traces from `redpanda.otel_traces`. -. Filter spans where `name` contains `tool_invocation`. -. Examine the `mcp.tool.name` attribute to see which tools are being used. +. Filter spans where `instrumentationScope.name` is `rpcn-mcp`. +. Examine the `name` field to see which tools are being invoked. . Calculate frequency by counting spans per tool name over time windows. -=== Measure performance +Example: To find all invocations of a specific tool, filter for spans where `name` matches your tool name (for example, `weather`, `http_processor`). + +== Measure performance Analyze tool execution times: -. Find spans with `mcp.tool.name` attributes. +. Find spans with `instrumentationScope.name` set to `rpcn-mcp`. . Calculate duration: `(endTimeUnixNano - startTimeUnixNano) / 1000000` (milliseconds). . Track percentiles (p50, p95, p99) to identify performance issues. . Set alerts for durations exceeding acceptable thresholds. -=== Debug failures +Example: A span with `startTimeUnixNano: "1765198415253280028"` and `endTimeUnixNano: "1765198424660663434"` has a duration of 9407ms. + +== Debug failures Investigate errors and failures: -. Filter spans where `status.code` is not `STATUS_CODE_OK`. -. Look for `STATUS_CODE_ERROR` in the status field. +. Filter spans where `status.code` is `2` (error). . Examine `status.message` for error details. +. Check the `events` array for error events with timestamps. . Use `traceId` to correlate related spans and understand the full error context. +. Follow `parentSpanId` relationships to trace the error back to the originating tool. + +Example: A span with `status.code: 2` and `status.message: "connection timeout"` indicates the operation failed due to a timeout. -=== Correlate distributed operations +== Correlate distributed operations Link MCP server activity to downstream effects: @@ -288,9 +100,14 @@ Link MCP server activity to downstream effects: . Follow `parentSpanId` relationships to build complete operation timelines. . Identify bottlenecks across your entire system. -== Integration with observability platforms +== Integrate with observability platforms The `redpanda.otel_traces` topic stores trace data in OpenTelemetry format. Direct export to platforms like Grafana Cloud and Datadog is not supported due to format compatibility limitations. Redpanda produces one span per topic message, whereas these platforms expect traces in batch format. You can consume traces directly from the `redpanda.otel_traces` topic using any Kafka-compatible consumer for custom analysis and processing. +== Next steps + +* xref:ai-agents:mcp/remote/concepts.adoc#execution-log[]: Learn how traces and spans work +* xref:ai-agents:mcp/remote/troubleshooting.adoc[]: Diagnose and fix common issues +* xref:ai-agents:mcp/remote/manage-servers.adoc[]: Manage MCP server lifecycle diff --git a/modules/ai-agents/pages/mcp/remote/overview.adoc b/modules/ai-agents/pages/mcp/remote/overview.adoc index 2b66d6135..462b40353 100644 --- a/modules/ai-agents/pages/mcp/remote/overview.adoc +++ b/modules/ai-agents/pages/mcp/remote/overview.adoc @@ -1,51 +1,86 @@ -= About Remote MCP Servers for Redpanda Cloud -:description: Discover how AI assistants can interact with your streaming data and how to connect them to Redpanda Cloud. += Remote MCP Server Overview +:description: Discover how AI agents can interact with your streaming data and how to connect them to Redpanda Cloud. +:page-topic-type: overview +:personas: evaluator, ai_agent_developer +// Reader journey: "I'm evaluating this" +// Learning objectives - what readers should understand after reading this page: +:learning-objective-1: Explain what a Remote MCP server is and how tools differ from pipelines +:learning-objective-2: Identify use cases where Remote MCP provides business value +:learning-objective-3: Describe how MCP tools expose Redpanda Connect components to AI -Remote MCP (Model Context Protocol) allows you to build and host MCP servers in your Redpanda Cloud clusters. They can expose your data, analytics, and automation pipelines as AI-consumable tools with no custom API code. MCP servers bridge your business logic to AI agents using the open MCP standard, so you can: +This page introduces Remote MCP servers and helps you decide if they're right for your use case. -* Make your data and workflows accessible to AI assistants and LLMs, safely and on your terms. -* Accelerate automation and insight by letting AI trigger real business actions. -* Control exposure so that only the tools you tag are visible to AI. +After reading this page, you will be able to: -== Why use Remote MCP? +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} -* *Always-on and centrally managed*: No need to run a local process. Your tools live with your data, managed by Redpanda. -* *Proximity to data*: Execute tools next to your cluster for lower latency and simpler networking. -* *Secure by design*: Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store]. Never hardcode secrets in your YAML configuration. -* *Fast iteration*: Define tools as YAML configuration, deploy your MCP server, and your AI agents can use the new logic instantly. +== What is MCP? + +// What is MCP - single-sourced from rp-connect-docs partial +include::redpanda-connect:ai-agents:partial$mcp/overview/what-is-mcp.adoc[] + +== What is Remote MCP? + +Remote MCP lets you build and host MCP servers in your Redpanda Cloud clusters. Your tools run next to your data, managed by Redpanda, so you get: + +* *Always-on availability:* No local process to run. Your tools are hosted and managed by Redpanda Cloud. +* *Proximity to data:* Tools execute next to your cluster for lower latency and simpler networking. +* *Secure secrets management:* Use the xref:develop:connect/configuration/secret-management.adoc[Secrets Store] instead of hardcoding credentials. +* *Fast iteration:* Define tools as YAML, deploy, and your AI agents can use them immediately. + +// MCP tools vs pipelines - single-sourced from rp-connect-docs partial +include::redpanda-connect:ai-agents:partial$mcp/overview/mcp-vs-pipelines.adoc[] // Use cases table - single-sourced from rp-connect-docs partial -include::redpanda-connect:ai-agents:partial$mcp/use-cases-table.adoc[] +include::redpanda-connect:ai-agents:partial$mcp/overview/use-cases-table.adoc[] == How it works -. Build your tools as Redpanda Connect configurations with `meta.mcp` metadata (description, properties, and logic). -. Deploy to your cluster. Redpanda Cloud hosts your MCP server with secure access. -. Connect your AI client. AI assistants authenticate, discover available tools, and invoke them using natural language prompts. +Remote MCP servers sit between AI clients and your data: + +. Your AI agent connects to your MCP server using `rpk cloud mcp proxy` or direct authentication +. A user asks their AI agent something like "What's the weather in London?" +. The server finds the matching tool and runs your Redpanda Connect configuration +. Your configuration fetches data, transforms it, and returns a structured response +. The AI agent gets the data and can use it to answer the user + +=== What a tool looks like -// Customer analytics example - single-sourced from rp-connect-docs partial -include::redpanda-connect:ai-agents:partial$mcp/customer-analytics-example.adoc[] +A tool is a YAML configuration with two parts: the logic (what the tool does) and the metadata (how AI understands it). -== Authentication options +Here's a minimal example that returns weather data: -- **MCP proxy (recommended):** For tools with built-in MCP clients like Claude Code: -+ -[,bash] +[source,yaml] ---- -rpk cloud login -rpk cloud mcp proxy --install --client claude-code --cluster-id --mcp-server-id +http: + url: "https://wttr.in/${! this.city }?format=j1" + verb: GET + +meta: + mcp: + enabled: true + name: get_weather + description: "Get current weather for a city" + properties: + - name: city + type: string + description: "City name" + required: true ---- -- **Direct connection:** If building your own AI agent, implement the MCP client flow yourself and connect directly to the MCP server. +When an AI client asks about weather, it calls this tool with the city name. The tool fetches data from the weather API and returns it. // MCP specification support - single-sourced from rp-connect-docs partial -include::redpanda-connect:ai-agents:partial$mcp/specification-support.adoc[] +include::redpanda-connect:ai-agents:partial$mcp/overview/specification-support.adoc[] -== Get started +== Next steps -* xref:ai-agents:mcp/remote/quickstart.adoc[] +Continue your learning journey with these resources: -== Suggested reading +* xref:ai-agents:mcp/remote/quickstart.adoc[]: Build your first MCP server +* xref:ai-agents:mcp/remote/concepts.adoc[]: Learn about execution and component types +* xref:ai-agents:mcp/remote/create-tool.adoc[]: Create custom tools step by step +* link:https://modelcontextprotocol.io/[Model Context Protocol documentation^] -* xref:ai-agents:mcp/remote/developer-guide.adoc[] -* Learn more about MCP in the link:https://docs.anthropic.com/en/docs/mcp[official documentation^]. diff --git a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc b/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc deleted file mode 100644 index 72b7a9d1b..000000000 --- a/modules/ai-agents/pages/mcp/remote/pipeline-patterns.adoc +++ /dev/null @@ -1,233 +0,0 @@ -= Patterns for Remote MCP Servers -:description: Catalog of patterns for MCP server tools in Redpanda Cloud. - -This page provides a reference catalog of patterns designed for use with MCP servers in Redpanda Cloud. Use these patterns as building blocks for your own MCP tools. For step-by-step instructions on building, deploying, and testing MCP servers, see xref:ai-agents:mcp/remote/developer-guide.adoc[]. - -Each pattern is a reusable example for a common MCP tool scenario. Patterns are grouped by use case. All YAML is ready to use in your MCP server project. - -For a high-level overview of MCP servers, see xref:ai-agents:mcp/overview.adoc[]. - -== Pattern selection guide - -Choose the right pattern for your use case: - -* **Generate test data or synthetic events**: <> -* **Call external REST APIs**: <> -* **Query databases**: <> -* **Read from Redpanda topics**: <> -* **Transform or validate data**: <> -* **Publish to Redpanda topics**: <> -* **Integrate with AI/LLM services**: <> -* **Cache frequently accessed data**: <> - -== Data generators - -Use xref:develop:connect/components/inputs/about.adoc[`inputs`] to create tools that read data from internal or external systems or generate sample data for testing and development. - -**When to use:** Development and testing environments where you need synthetic data, load testing scenarios, or demonstrating data flows without real data sources. - -**Example use cases:** Mock user events, test order data, synthetic sensor readings, demo data for presentations. - -This example generates a realistic user event message: - -[source,yaml] ----- -include::ai-agents:example$generate_input.yaml[] ----- - -See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input component] - -== External API calls - -Use xref:develop:connect/components/processors/about.adoc[`processors`] to fetch data from external APIs, databases, or services and return formatted results. This is one of the most common patterns for MCP tools. - -**When to use:** Integrating with third-party services, fetching real-time data, calling internal microservices, or enriching event data with external information. - -**Example use cases:** Fetch user profile from CRM, get product pricing from inventory API, validate addresses with geocoding service, retrieve weather data. - -[source,yaml] ----- -include::ai-agents:example$http_processor.yaml[] ----- - -See also: xref:develop:connect/components/processors/http.adoc[`http` processor], xref:develop:connect/components/processors/mutation.adoc[`mutation` processor] - -== Database queries - -Query external databases and return structured results. This pattern is essential for tools that need to access business data. - -**When to use:** Retrieving customer records, querying analytics data, looking up configuration values, or joining streaming data with dimensional data from data warehouses. - -**Example use cases:** Fetch customer details from PostgreSQL, query sales data from BigQuery, retrieve product catalog from MongoDB, look up reference data. - -[source,yaml] ----- -include::ai-agents:example$gcp_bigquery_select_processor.yaml[] ----- - -See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] (for other databases) - -== Data consumers - -Read messages from Redpanda topics to process streaming data in your MCP tools. This pattern enables event-driven workflows and real-time data processing. - -**When to use:** Processing events from Redpanda topics, building event-driven AI agents, consuming audit logs, or subscribing to data change streams. - -**Example use cases:** Monitor order events, process user activity streams, consume IoT sensor data, react to system notifications. - -[source,yaml] ----- -redpanda: - seed_brokers: [ "${REDPANDA_BROKERS}" ] - topics: [ "user-events" ] - consumer_group: "mcp-event-processor" - start_from_oldest: true - tls: - enabled: true - sasl: - - mechanism: "${REDPANDA_SASL_MECHANISM}" - username: "${REDPANDA_SASL_USERNAME}" - password: "${REDPANDA_SASL_PASSWORD}" ----- - -See also: xref:develop:connect/components/inputs/redpanda.adoc[`redpanda` input] - -== Data transformation - -Transform, validate, and enrich data as it flows through your MCP tools. Use Bloblang mapping language for powerful data manipulation. - -**When to use:** Converting data formats, validating schemas, filtering events, enriching messages with computed fields, or normalizing data structures. - -**Example use cases:** Parse JSON payloads, validate required fields, add timestamps, convert units, mask sensitive data, aggregate nested objects. - -[source,yaml] ----- -mapping: | - # Parse and validate incoming data - root.user_id = this.user_id.or(throw("user_id is required")) - root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") - - # Transform and enrich - root.email_domain = this.email.split("@").index(1) - root.is_premium = this.subscription_tier == "premium" - - # Filter sensitive data - root.profile = this.profile.without("ssn", "credit_card") ----- - -See also: xref:develop:connect/components/processors/mapping.adoc[`mapping` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang guide] - -== AI/LLM integration - -Integrate AI and LLM services into your MCP tools for intelligent data processing, embeddings generation, and natural language understanding. - -**When to use:** Generating embeddings for semantic search, calling LLM APIs for text generation, building RAG (Retrieval Augmented Generation) pipelines, or analyzing sentiment. - -**Example use cases:** Generate embeddings for documents, classify customer feedback, summarize long text, extract entities, answer questions with context. - -=== OpenAI integration - -[source,yaml] ----- -openai_chat_completion: - api_key: "${secrets.OPENAI_API_KEY}" - model: "gpt-4" - prompt: | - Analyze this customer feedback and provide: - 1. Sentiment (positive/negative/neutral) - 2. Key themes - 3. Actionable insights - - Feedback: ${! json("feedback_text") } - max_tokens: 500 ----- - -See also: xref:develop:connect/components/processors/openai_chat_completion.adoc[`openai_chat_completion`], xref:develop:connect/components/processors/openai_embeddings.adoc[`openai_embeddings`] - -=== Embeddings generation - -[source,yaml] ----- -openai_embeddings: - api_key: "${secrets.OPENAI_API_KEY}" - model: "text-embedding-3-small" - text: ${! json("content") } ----- - -See also: xref:develop:connect/components/processors/cohere_embeddings.adoc[`cohere_embeddings`], xref:develop:connect/components/processors/gcp_vertex_ai_embeddings.adoc[`gcp_vertex_ai_embeddings`] - -== Redpanda integration - -Publish data to Redpanda topics and implement caching systems backed by Redpanda. - -Use these patterns when you need to integrate with Redpanda infrastructure, publish data to topics, or implement caching systems. - -=== Data publishers - -Use xref:develop:connect/components/outputs/about.adoc[`outputs`] to send data to external systems, publish events to Redpanda topics, or integrate with downstream services. - -**When to use:** Publishing events to Redpanda for consumption by other services, creating event sourcing patterns, building audit trails, or triggering downstream workflows. - -**Example use cases:** Publish order confirmations, emit audit events, trigger notifications, create event-driven workflows. - -This example publishes a message to a Redpanda topic: - -[source,yaml] ----- -include::ai-agents:example$redpanda_output.yaml[] ----- - -See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] - -==== Outputs with processors - -Output tools can include processors to transform data before publishing. This pattern is useful when you need to process data and save the result to a destination in a single tool. - -**When to use:** Processing user input with an LLM and saving the response, transforming data before publishing to a topic, enriching events before writing to external systems. - -**Example use cases:** Send a prompt to an LLM, then save the answer to a topic in Redpanda. - -[source,yaml] ----- -include::ai-agents:example$redpanda_output_with_processors.yaml[] ----- - -You can use an output component type with processors, but you cannot use a processor component type with outputs. The `processors` field is available in all output components. - -=== Caching systems - -Use caching to store frequently accessed data, reduce latency, and minimize external API calls. You can implement caching using either Redpanda topics or in-memory stores. - -**When to use:** Reducing repeated API calls, storing lookup tables, caching database query results, or maintaining session state across tool invocations. - -**Example use cases:** Cache user profiles, store API rate limit counters, maintain configuration values, cache product catalogs. - -.Redpanda-backed cache -[source,yaml] ----- -include::ai-agents:example$redpanda_cache.yaml[] ----- - -.In-memory cache for low-latency access to small datasets -[source,yaml] ----- -include::ai-agents:example$memory_cache.yaml[] ----- - -See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], Redpanda-backed cache using xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] - -// Production workflows section - single-sourced from rp-connect-docs partials -// Part 1: Parameter validation, dynamic config, error handling, conditional processing -include::redpanda-connect:ai-agents:partial$mcp/production-workflows-before-secrets.adoc[] - -// Secrets section - deployment-specific for Cloud (uses Secrets Store) -include::redpanda-connect:ai-agents:partial$mcp/secrets-cloud.adoc[] - -// Part 2: Monitoring, multi-step enrichment, workflow orchestration -include::redpanda-connect:ai-agents:partial$mcp/production-workflows-after-secrets.adoc[] - -== Suggested reading - -* xref:develop:connect/components/about.adoc[Components overview] -* xref:develop:connect/guides/bloblang/about.adoc[Bloblang guide] -* xref:develop:connect/configuration/secret-management.adoc[Secret management] \ No newline at end of file diff --git a/modules/ai-agents/pages/mcp/remote/quickstart.adoc b/modules/ai-agents/pages/mcp/remote/quickstart.adoc index 734f8e8a4..a778df103 100644 --- a/modules/ai-agents/pages/mcp/remote/quickstart.adoc +++ b/modules/ai-agents/pages/mcp/remote/quickstart.adoc @@ -1,31 +1,24 @@ = Remote MCP Server Quickstart :description: Learn how to extend AI agents with custom tools that interact with your Redpanda data using the Model Context Protocol (MCP). +:page-topic-type: tutorial +:personas: ai_agent_developer, streaming_developer, evaluator +// Reader journey: "I want to try it now" +// Learning objectives - what readers will achieve by completing this quickstart: +:learning-objective-1: Create an MCP server in Redpanda Cloud +:learning-objective-2: Define tools that generate and publish data +:learning-objective-3: Connect Claude Code to your MCP server and invoke tools -Model Context Protocol (MCP) enables AI assistants like Claude to interact with external systems through custom tools. This quickstart teaches you how to create an MCP server in Redpanda Cloud with tools that generate and publish event data, then connect Claude Code to use those tools through natural language commands. +This quickstart builds an MCP server in Redpanda Cloud that exposes tools for generating and publishing event data. You'll create two tools, then ask Claude Code to use them through natural language: -== What you'll learn +> Generate 10 user events and publish them to the events topic. -By completing this quickstart, you'll understand: +By completing this quickstart, you will be able to: -* What MCP is and how it extends AI capabilities -* How MCP servers, tools, and clients work together -* How to create and deploy custom tools in Redpanda Cloud -* How to connect AI assistants to your Redpanda data infrastructure +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} -== How MCP works - -The Model Context Protocol defines a standard way for AI assistants to interact with external tools and data sources: - -MCP Server:: Hosts a collection of tools that perform specific actions. MCP servers run as managed services in your Redpanda Cloud cluster. -Tools:: Individual capabilities your AI can use, built with Redpanda Connect components. Tools can read from sources (inputs), transform data (processors), write to destinations (outputs), or cache results. -Client:: An AI assistant (like Claude Code) that discovers available tools and calls them based on user requests. - -In this quickstart, you'll create a server with two tools: - -. `generate_input`: Creates fake user event data with randomized fields -. `redpanda_output`: Publishes generated events to a Redpanda topic - -Once connected, you can ask Claude in natural language to "generate 10 events and publish them to Redpanda," and Claude will automatically call your tools to complete the task. +TIP: For background on how MCP tools work and when to use each component type, see xref:ai-agents:mcp/remote/concepts.adoc[]. == Prerequisites @@ -183,7 +176,7 @@ curl -X POST "https:///v1/acls" \ [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . Log in to the link:https://cloud.redpanda.com/[Redpanda Cloud Console^]. @@ -382,47 +375,23 @@ rpk topic consume events --num 10 + You should see the generated event data in JSON format, confirming that Claude successfully used your custom tools to generate data and publish it to Redpanda. -== Summary - -You extended Claude's capabilities with custom tools that interact with your Redpanda cluster. Claude can now generate and publish event data through natural language commands without manual scripting. This same pattern works for any workflow: reading from topics, transforming data, querying databases, or calling external APIs. - == Troubleshoot -=== MCP server not starting - -If your server is not starting or shows an error state: +If you encounter issues during this quickstart: -- Check the *Logs* tab for specific error messages. -- Verify your YAML syntax by clicking *Lint*. +- **MCP server not starting**: Check the *Logs* tab and verify your YAML syntax by clicking *Lint*. +- **Connection issues**: Verify you're logged in with `rpk cloud login` and that your server status shows *Running*. +- **Publishing failures**: Verify the `events` topic exists with `rpk topic list`. -=== Connection issues - -If Claude Code can't connect to your server: - -- Verify you're logged in with `rpk cloud login`. -- Check that your rpk version is 25.2.5 or later: `rpk version`. -- Ensure your server status shows *Running* in Redpanda Cloud. -- Try restarting Claude Code after running the proxy command. - -=== Publishing failures - -If the publish tool returns errors: - -- Verify the `events` topic exists: -+ -```bash -rpk topic list -``` -- Use the MCP Inspector in Redpanda Cloud to test with different event data. -- Check the MCP server logs for authentication or authorization errors. +For detailed solutions, see xref:ai-agents:mcp/remote/troubleshooting.adoc[]. == Next steps -Now that you understand how MCP works, you can create custom tools for your specific use cases: - -* **Data ingestion**: Build tools that read from external APIs, databases, or file systems and publish to Redpanda topics. -* **Data processing**: Create tools that consume from topics, transform data, and write results back to Redpanda. -* **Monitoring and observability**: Add tools that query metrics, check system health, or analyze streaming data. -* **Integration workflows**: Connect Redpanda to other systems through custom tools that Claude can orchestrate. +You've deployed an MCP server and connected Claude Code to your Redpanda cluster. Here's where to go next: -See xref:ai-agents:mcp/remote/developer-guide.adoc[] to learn how to build custom tools using Redpanda Connect components. +* xref:ai-agents:mcp/remote/concepts.adoc[]: Understand how MCP tools differ from pipelines +* xref:ai-agents:mcp/remote/create-tool.adoc[]: Build production-quality tools with validation +* xref:ai-agents:mcp/remote/best-practices.adoc[]: Apply naming and design guidelines +* xref:ai-agents:mcp/remote/tool-patterns.adoc[]: Find reusable patterns +* xref:ai-agents:mcp/remote/troubleshooting.adoc[]: Diagnose common issues +* xref:ai-agents:mcp/remote/admin-guide.adoc[]: Scale resources, monitor activity, and administer your MCP servers diff --git a/modules/ai-agents/pages/mcp/remote/scale-resources.adoc b/modules/ai-agents/pages/mcp/remote/scale-resources.adoc index bb63ee827..139fa979f 100644 --- a/modules/ai-agents/pages/mcp/remote/scale-resources.adoc +++ b/modules/ai-agents/pages/mcp/remote/scale-resources.adoc @@ -1,5 +1,12 @@ = Scale Remote MCP Server Resources :description: Learn how to scale MCP server resources up or down to match workload demands and optimize costs. +:page-topic-type: how-to +:personas: platform_admin +// Reader journey: "I optimize costs and performance" +// Learning objectives - what readers can accomplish from this page: +:learning-objective-1: Scale MCP server resources up or down +:learning-objective-2: Choose appropriate resource sizes for workloads +:learning-objective-3: Optimize costs through resource management You can scale your MCP server's resources up or down to match your workload needs. Resource allocation affects your xref:billing:billing.adoc#remote-mcp-billing-metrics[billing costs], which are charged per compute unit hour. @@ -11,7 +18,7 @@ You must have an existing MCP server. If you do not have one, see xref:ai-agents [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . In the Redpanda Cloud Console, navigate to *Remote MCP*. diff --git a/modules/ai-agents/pages/mcp/remote/tool-patterns.adoc b/modules/ai-agents/pages/mcp/remote/tool-patterns.adoc new file mode 100644 index 000000000..1348419f1 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/tool-patterns.adoc @@ -0,0 +1,315 @@ += MCP Tool Patterns +:page-aliases: ai-agents:mcp/remote/pipeline-patterns.adoc +:description: Catalog of patterns for MCP server tools in Redpanda Cloud. +:page-topic-type: cookbook +:personas: ai_agent_developer, data_engineer +// Reader journey: "I need an example for X" +:learning-objective-1: Find reusable patterns for common MCP tool scenarios +:learning-objective-2: Apply validation and error handling patterns for production robustness +:learning-objective-3: Format responses consistently for AI client consumption + +When building tools, use these patterns as starting points for common scenarios. For step-by-step instructions, see xref:ai-agents:mcp/remote/create-tool.adoc[]. For design guidelines, see xref:ai-agents:mcp/remote/best-practices.adoc[]. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + + + +[[read-data]] +== Read data + +Use xref:develop:connect/components/inputs/about.adoc[inputs] to create tools that read from data sources or generate sample data. + +[[data-generators]] +=== Generate test data + +*When to use:* Development and testing environments where you need synthetic data, load testing scenarios, or demonstrating data flows without real data sources. + +*Example use cases:* Mock user events, test order data, synthetic sensor readings, demo data for presentations. + +[source,yaml] +---- +include::ai-agents:example$generate_input.yaml[] +---- + +See also: xref:develop:connect/components/inputs/generate.adoc[`generate` input component] + +[[consume-from-redpanda]] +=== Consume from Redpanda topics + +*When to use:* Processing events from Redpanda topics, building event-driven AI agents, consuming audit logs, or subscribing to data change streams. + +*Example use cases:* Monitor order events, process user activity streams, consume IoT sensor data, react to system notifications. + +[source,yaml] +---- +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "user-events" ] + consumer_group: "mcp-event-processor" + start_from_oldest: true + tls: + enabled: true + sasl: + - mechanism: "${REDPANDA_SASL_MECHANISM}" + username: "${REDPANDA_SASL_USERNAME}" + password: "${REDPANDA_SASL_PASSWORD}" +---- + +See also: xref:develop:connect/components/inputs/redpanda.adoc[`redpanda` input] + +[[stream-processing]] +=== Process streaming data + +*When to use:* Real-time analytics, windowed aggregations, computing metrics over time, or building streaming dashboards. + +*Example use cases:* Calculate rolling averages, count events per time window, detect anomalies in streams, aggregate metrics. + +[source,yaml] +---- +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "sensor-readings" ] + consumer_group: "analytics-processor" + tls: + enabled: true + sasl: + - mechanism: "${REDPANDA_SASL_MECHANISM}" + username: "${REDPANDA_SASL_USERNAME}" + password: "${REDPANDA_SASL_PASSWORD}" + processors: + - mapping: | + root.sensor_id = this.sensor_id + root.avg_temperature = this.readings.map_each(r -> r.temperature).mean() + root.max_temperature = this.readings.map_each(r -> r.temperature).max() + root.reading_count = this.readings.length() + root.window_end = now() +---- + +See also: xref:develop:connect/components/inputs/redpanda.adoc[`redpanda` input] + +[[call-external-services]] +== Call external services + +Use xref:develop:connect/components/processors/about.adoc[processors] to fetch data from external APIs, databases, or AI services. + +[[external-api-calls]] +=== Call REST APIs + +*When to use:* Integrating with third-party services, fetching real-time data, calling internal microservices, or enriching event data with external information. + +*Example use cases:* Fetch user profile from CRM, get product pricing from inventory API, validate addresses with geocoding service, retrieve weather data. + +[source,yaml] +---- +include::ai-agents:example$http_processor.yaml[] +---- + +See also: xref:develop:connect/components/processors/http.adoc[`http` processor], xref:develop:connect/components/processors/mutation.adoc[`mutation` processor] + +[[database-queries]] +=== Query databases + +*When to use:* Retrieving customer records, querying analytics data, looking up configuration values, or joining streaming data with dimensional data from data warehouses. + +*Example use cases:* Fetch customer details from PostgreSQL, query sales data from BigQuery, retrieve product catalog from MongoDB, look up reference data. + +[source,yaml] +---- +include::ai-agents:example$gcp_bigquery_select_processor.yaml[] +---- + +See also: xref:develop:connect/components/processors/gcp_bigquery_select.adoc[`gcp_bigquery_select` processor], xref:develop:connect/components/processors/sql_select.adoc[`sql_select` processor] + +[[ai-llm-integration]] +=== Integrate with AI/LLM services + +*When to use:* Generating embeddings for semantic search, calling LLM APIs for text generation, building RAG pipelines, or analyzing sentiment. + +*Example use cases:* Generate embeddings for documents, classify customer feedback, summarize long text, extract entities, answer questions with context. + +==== OpenAI chat completion + +[source,yaml] +---- +openai_chat_completion: + api_key: "${secrets.OPENAI_API_KEY}" + model: "gpt-4" + prompt: | + Analyze this customer feedback and provide: + 1. Sentiment (positive/negative/neutral) + 2. Key themes + 3. Actionable insights + + Feedback: ${! json("feedback_text") } + max_tokens: 500 +---- + +See also: xref:develop:connect/components/processors/openai_chat_completion.adoc[`openai_chat_completion`], xref:develop:connect/components/processors/openai_embeddings.adoc[`openai_embeddings`] + +==== Generate embeddings + +[source,yaml] +---- +openai_embeddings: + api_key: "${secrets.OPENAI_API_KEY}" + model: "text-embedding-3-small" + text: ${! json("content") } +---- + +See also: xref:develop:connect/components/processors/cohere_embeddings.adoc[`cohere_embeddings`], xref:develop:connect/components/processors/gcp_vertex_ai_embeddings.adoc[`gcp_vertex_ai_embeddings`] + +[[write-data]] +== Write data + +Use xref:develop:connect/components/outputs/about.adoc[outputs] to write data to Redpanda topics or cache stores. + +[[publish-to-redpanda]] +=== Publish to Redpanda topics + +*When to use:* Publishing events to Redpanda for consumption by other services, creating event sourcing patterns, building audit trails, or triggering downstream workflows. + +*Example use cases:* Publish order confirmations, emit audit events, trigger notifications, create event-driven workflows. + +[source,yaml] +---- +include::ai-agents:example$redpanda_output.yaml[] +---- + +See also: xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +==== Outputs with processors + +Output tools can include processors to transform data before publishing. This pattern is useful when you need to process data and save the result to a destination in a single tool. + +*When to use:* Processing user input with an LLM and saving the response, transforming data before publishing to a topic, enriching events before writing to external systems. + +[source,yaml] +---- +include::ai-agents:example$redpanda_output_with_processors.yaml[] +---- + +[[caching]] +=== Cache data + +*When to use:* Reducing repeated API calls, storing lookup tables, caching database query results, or maintaining session state across tool invocations. + +*Example use cases:* Cache user profiles, store API rate limit counters, maintain configuration values, cache product catalogs. + +.Redpanda-backed cache +[source,yaml] +---- +include::ai-agents:example$redpanda_cache.yaml[] +---- + +.In-memory cache +[source,yaml] +---- +include::ai-agents:example$memory_cache.yaml[] +---- + +See also: xref:develop:connect/components/caches/memory.adoc[`memory` cache], xref:develop:connect/components/outputs/redpanda.adoc[`redpanda` output] + +[[transform-data]] +== Transform data + +Use Bloblang and processors to transform, validate, and route data. + +[[data-transformation]] +=== Transform and validate + +*When to use:* Converting data formats, validating schemas, filtering events, enriching messages with computed fields, or normalizing data structures. + +*Example use cases:* Parse JSON payloads, validate required fields, add timestamps, convert units, mask sensitive data, aggregate nested objects. + +[source,yaml] +---- +mapping: | + # Parse and validate incoming data + root.user_id = this.user_id.or(throw("user_id is required")) + root.timestamp = now().ts_format("2006-01-02T15:04:05Z07:00") + + # Transform and enrich + root.email_domain = this.email.split("@").index(1) + root.is_premium = this.subscription_tier == "premium" + + # Filter sensitive data + root.profile = this.profile.without("ssn", "credit_card") +---- + +See also: xref:develop:connect/components/processors/mapping.adoc[`mapping` processor], xref:develop:connect/guides/bloblang/about.adoc[Bloblang guide] + +[[event-driven-workflows]] +=== Build event-driven workflows + +*When to use:* Orchestrating multi-step processes, responding to business events, implementing saga patterns, or coordinating microservices. + +*Example use cases:* Order fulfillment workflows, approval processes, notification cascades, data pipeline orchestration. + +[source,yaml] +---- +redpanda: + seed_brokers: [ "${REDPANDA_BROKERS}" ] + topics: [ "order-events" ] + consumer_group: "workflow-orchestrator" + tls: + enabled: true + sasl: + - mechanism: "${REDPANDA_SASL_MECHANISM}" + username: "${REDPANDA_SASL_USERNAME}" + password: "${REDPANDA_SASL_PASSWORD}" + processors: + - switch: + - check: this.event_type == "order_created" + processors: + - http: + url: "${secrets.INVENTORY_API}/reserve" + verb: POST + body: '{"order_id": "${! this.order_id }", "items": ${! json("items") }}' + - check: this.event_type == "payment_confirmed" + processors: + - http: + url: "${secrets.FULFILLMENT_API}/ship" + verb: POST + body: '{"order_id": "${! this.order_id }"}' +---- + +See also: xref:develop:connect/components/inputs/redpanda.adoc[`redpanda` input] + +[[production-readiness]] +== Production readiness + +Build production-ready tools with proper input validation, error handling, and response formatting. + +[[input-validation]] +=== Validate input + +AI clients may send unexpected or malformed input. Validate early to return helpful error messages instead of cryptic failures from downstream components. + +include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/input-validation.adoc[leveloffset=+1] + +[[error-handling]] +=== Handle errors + +External services fail. Databases go down. APIs return unexpected responses. Wrap risky operations in error handling so your tool returns useful error messages instead of crashing. + +include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/error-handling.adoc[leveloffset=+1] + +[[response-formatting]] +=== Format responses + +AI clients work best with clean, predictable response structures. Transform raw component output into consistent formats. + +include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/response-formatting.adoc[leveloffset=+1] + +// Production workflows section - single-sourced from rp-connect-docs +include::redpanda-connect:ai-agents:partial$mcp/tool-patterns/production-workflows.adoc[] + +== Next steps + +* xref:ai-agents:mcp/remote/create-tool.adoc[]: Step-by-step tool creation guide +* xref:ai-agents:mcp/remote/best-practices.adoc[]: Apply naming and design guidelines +* xref:ai-agents:mcp/remote/troubleshooting.adoc[]: Diagnose and fix common issues diff --git a/modules/ai-agents/pages/mcp/remote/troubleshooting.adoc b/modules/ai-agents/pages/mcp/remote/troubleshooting.adoc new file mode 100644 index 000000000..9c0dc41e6 --- /dev/null +++ b/modules/ai-agents/pages/mcp/remote/troubleshooting.adoc @@ -0,0 +1,46 @@ += Troubleshoot Remote MCP Servers +:description: Diagnose and fix common issues when building and running Remote MCP servers in Redpanda Cloud. +:page-topic-type: troubleshooting +:personas: ai_agent_developer, streaming_developer, platform_admin +// Reader journey: "Something went wrong" +// Learning objectives - what readers can do with this page: +:learning-objective-1: Diagnose and fix lint and YAML configuration errors +:learning-objective-2: Resolve runtime issues when tools don't appear or return unexpected results +:learning-objective-3: Debug client connection problems + +This page helps you diagnose and fix common issues when building and running Remote MCP servers. + +Use this page to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +== Lint errors + +include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/troubleshooting-lint.adoc[] + +== Runtime issues + +include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/troubleshooting-runtime.adoc[] + +== Connection issues + +include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/troubleshooting-connection.adoc[] + +[[debugging]] +== Debugging techniques + +Use these techniques to systematically isolate and fix issues with your MCP tools. + +include::redpanda-connect:ai-agents:partial$mcp/troubleshooting/debugging-techniques.adoc[] + +== Next steps + +If you're still experiencing issues: + +* xref:ai-agents:mcp/remote/create-tool.adoc[]: Review YAML structure rules and metadata fields +* xref:ai-agents:mcp/remote/best-practices.adoc[]: Review naming and metadata design +* xref:ai-agents:mcp/remote/concepts.adoc[]: Review component type selection + +For protocol-level troubleshooting, see the link:https://modelcontextprotocol.io/[MCP documentation^]. diff --git a/modules/develop/pages/connect/configuration/resource-management.adoc b/modules/develop/pages/connect/configuration/resource-management.adoc index c48a3e58e..802e1d8fb 100644 --- a/modules/develop/pages/connect/configuration/resource-management.adoc +++ b/modules/develop/pages/connect/configuration/resource-management.adoc @@ -125,7 +125,7 @@ To view resources already allocated to a data pipeline: [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^]. @@ -152,7 +152,7 @@ To scale the resources for a pipeline: [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^]. diff --git a/modules/develop/pages/connect/configuration/secret-management.adoc b/modules/develop/pages/connect/configuration/secret-management.adoc index ac0fee981..026616216 100644 --- a/modules/develop/pages/connect/configuration/secret-management.adoc +++ b/modules/develop/pages/connect/configuration/secret-management.adoc @@ -15,7 +15,7 @@ You can create a secret and reference it in multiple data pipelines on the same [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^]. @@ -71,7 +71,7 @@ NOTE: Changes to secret values do not take effect until a pipeline is restarted. [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^]. @@ -122,7 +122,7 @@ NOTE: Changes do not affect pipelines that are already running. [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . Log in to https://cloud.redpanda.com[Redpanda Cloud^]. @@ -158,7 +158,7 @@ You must include the following values: [tabs] ===== -Cloud UI:: +Cloud Console:: + -- . Go to the **Connect** page, and create a pipeline (or open an existing pipeline to edit). diff --git a/modules/get-started/pages/whats-new-cloud.adoc b/modules/get-started/pages/whats-new-cloud.adoc index 998af9d38..ab3c6600f 100644 --- a/modules/get-started/pages/whats-new-cloud.adoc +++ b/modules/get-started/pages/whats-new-cloud.adoc @@ -77,7 +77,7 @@ Deploy managed MCP servers directly inside your Redpanda Cloud cluster with xref Create custom AI tools using templates or write your own Redpanda Connect configurations to build event-driven workflows. Remote MCP servers provide AI assistants with streaming data capabilities, enabling use cases like real-time data generation, stream processing, and event publishing. -Get started with the xref:ai-agents:mcp/remote/quickstart.adoc[quickstart guide] or explore advanced patterns in the xref:ai-agents:mcp/remote/developer-guide.adoc[developer guide]. +Get started with the xref:ai-agents:mcp/remote/quickstart.adoc[quickstart guide] or learn xref:ai-agents:mcp/remote/best-practices.adoc[best practices] for building robust tools. === API Gateway access diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc index 286dc30dc..d4cb4a5e5 100644 --- a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-install.adoc @@ -1,4 +1,4 @@ = rpk cloud mcp install -:description: Install the local MCP server for Redpanda Cloud configuration. +:description: Install the MCP server for the Redpanda Cloud API. include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-install.adoc[tag=single-source] \ No newline at end of file diff --git a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc index aa1f11f42..2b51cff50 100644 --- a/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc +++ b/modules/reference/pages/rpk/rpk-cloud/rpk-cloud-mcp-stdio.adoc @@ -1,4 +1,4 @@ = rpk cloud mcp stdio -:description: Communicate with the local MCP server for Redpanda Cloud using the stdio protocol. +:description: Communicate with the MCP server for the Redpanda Cloud API using the stdio protocol. include::ROOT:reference:partial$rpk-cloud/rpk-cloud-mcp-stdio.adoc[tag=single-source] \ No newline at end of file