diff --git a/app/en/guides/create-tools/evaluate-tools/_meta.tsx b/app/en/guides/create-tools/evaluate-tools/_meta.tsx index 39a1896e7..2bb89e837 100644 --- a/app/en/guides/create-tools/evaluate-tools/_meta.tsx +++ b/app/en/guides/create-tools/evaluate-tools/_meta.tsx @@ -1,5 +1,28 @@ -export default { - "why-evaluate": "Why evaluate tools?", - "create-evaluation-suite": "Create an evaluation suite", - "run-evaluations": "Run evaluations", +import type { MetaRecord } from "nextra"; + +const meta: MetaRecord = { + "*": { + theme: { + breadcrumb: true, + toc: true, + copyPage: true, + }, + }, + "why-evaluate": { + title: "Why evaluate tools?", + }, + "create-evaluation-suite": { + title: "Create an evaluation suite", + }, + "run-evaluations": { + title: "Run evaluations", + }, + "capture-mode": { + title: "Capture mode", + }, + "comparative-evaluations": { + title: "Comparative evaluations", + }, }; + +export default meta; diff --git a/app/en/guides/create-tools/evaluate-tools/capture-mode/page.mdx b/app/en/guides/create-tools/evaluate-tools/capture-mode/page.mdx new file mode 100644 index 000000000..6f386aa79 --- /dev/null +++ b/app/en/guides/create-tools/evaluate-tools/capture-mode/page.mdx @@ -0,0 +1,414 @@ +--- +title: "Capture mode" +description: "Record tool calls without scoring to bootstrap test expectations" +--- + +import { Callout, Steps } from "nextra/components"; + +# Capture mode + +Capture mode records tool calls without evaluating them. Use it to bootstrap test expectations or debug model behavior. + + + **Backward compatibility**: Capture mode works with existing evaluation + suites. Simply add the `--capture` flag to any `arcade evals` command. No code + changes needed. + + +## When to use capture mode + +**Bootstrapping test expectations**: When you don't know what tool calls to expect, run capture mode to see what the model actually calls. + +**Debugging model behavior**: When evaluations fail unexpectedly, capture mode shows exactly what the model is doing. + +**Exploring new tools**: When adding new tools, capture mode helps you understand how models interpret them. + +**Documenting tool usage**: Create examples of how models use your tools in different scenarios. + +### Typical workflow + +``` +1. Create suite with empty expected_tool_calls + ↓ +2. Run: arcade evals . --capture --format json + ↓ +3. Review captured tool calls in output file + ↓ +4. Copy tool calls into expected_tool_calls + ↓ +5. Add critics for validation + ↓ +6. Run: arcade evals . --details +``` + +## Basic usage + + + +### Create an evaluation suite without expectations + +Create a suite with test cases but empty `expected_tool_calls`: + +```python +from arcade_evals import EvalSuite, tool_eval + +@tool_eval() +async def capture_weather_suite(): + suite = EvalSuite( + name="Weather Capture", + system_message="You are a weather assistant.", + ) + + await suite.add_mcp_stdio_server(["python", "weather_server.py"]) + + # Add cases without expected tool calls + suite.add_case( + name="Simple weather query", + user_message="What's the weather in Seattle?", + expected_tool_calls=[], # Empty for capture + ) + + suite.add_case( + name="Multi-city comparison", + user_message="Compare the weather in Seattle and Portland", + expected_tool_calls=[], + ) + + return suite +``` + +### Run in capture mode + +Run evaluations with the `--capture` flag: + +```bash +arcade evals . --capture --file captures/weather --format json +``` + +This creates `captures/weather.json` with all tool calls. + +### Review captured output + +Open the JSON file to see what the model called: + +```json +{ + "suite_name": "Weather Capture", + "model": "gpt-4o", + "provider": "openai", + "captured_cases": [ + { + "case_name": "Simple weather query", + "user_message": "What's the weather in Seattle?", + "tool_calls": [ + { + "name": "Weather_GetCurrent", + "args": { + "location": "Seattle", + "units": "fahrenheit" + } + } + ] + } + ] +} +``` + +### Convert to test expectations + +Copy the captured calls into your evaluation suite: + +```python +from arcade_evals import ExpectedMCPToolCall, BinaryCritic + +suite.add_case( + name="Simple weather query", + user_message="What's the weather in Seattle?", + expected_tool_calls=[ + ExpectedMCPToolCall( + "Weather_GetCurrent", + {"location": "Seattle", "units": "fahrenheit"} + ) + ], + critics=[ + BinaryCritic(critic_field="location", weight=0.7), + BinaryCritic(critic_field="units", weight=0.3), + ], +) +``` + + + +## CLI options + +### Basic capture + +Record tool calls to JSON: + +```bash +arcade evals . --capture --file captures/baseline --format json +``` + +### Include conversation context + +Capture system messages and conversation history: + +```bash +arcade evals . --capture --include-context --file captures/detailed --format json +``` + +Output includes: + +```json +{ + "case_name": "Weather with context", + "user_message": "What about the weather there?", + "system_message": "You are a weather assistant.", + "additional_messages": [ + {"role": "user", "content": "I'm traveling to Tokyo"}, + {"role": "assistant", "content": "Tokyo is a great city!"} + ], + "tool_calls": [...] +} +``` + +### Multiple formats + +Save captures in multiple formats: + +```bash +arcade evals . --capture --file captures/out --format json,md +``` + +Markdown format is more readable for quick review: + +```markdown +## Weather Capture + +### Model: gpt-4o + +#### Case: Simple weather query + +**Input:** What's the weather in Seattle? + +**Tool Calls:** + +- `Weather_GetCurrent` + - location: Seattle + - units: fahrenheit +``` + +### Multiple providers + +Capture from multiple providers to compare behavior: + +```bash +arcade evals . --capture \ + --use-provider openai:gpt-4o \ + --use-provider anthropic:claude-sonnet-4-5-20250929 \ + --file captures/comparison --format json +``` + +## Capture with comparative tracks + +Capture from multiple tool sources to see how different implementations behave: + +```python +@tool_eval() +async def capture_comparative(): + suite = EvalSuite( + name="Weather Comparison", + system_message="You are a weather assistant.", + ) + + # Register different tool sources + await suite.add_mcp_server( + "http://weather-api-1.example/mcp", + track="Weather API v1" + ) + + await suite.add_mcp_server( + "http://weather-api-2.example/mcp", + track="Weather API v2" + ) + + # Capture will run against each track + suite.add_case( + name="get_weather", + user_message="What's the weather in Seattle?", + expected_tool_calls=[], + ) + + return suite +``` + +Run capture: + +```bash +arcade evals . --capture --file captures/apis --format json +``` + +Output shows captures per track: + +```json +{ + "captured_cases": [ + { + "case_name": "get_weather", + "track_name": "Weather API v1", + "tool_calls": [ + {"name": "GetCurrentWeather", "args": {...}} + ] + }, + { + "case_name": "get_weather", + "track_name": "Weather API v2", + "tool_calls": [ + {"name": "Weather_Current", "args": {...}} + ] + } + ] +} +``` + +## Best practices + +### Start with broad queries + +Begin with open-ended prompts to see natural model behavior: + +```python +suite.add_case( + name="explore_weather_tools", + user_message="Show me everything you can do with weather", + expected_tool_calls=[], +) +``` + +### Capture edge cases + +Record model behavior on unusual inputs: + +```python +suite.add_case( + name="ambiguous_location", + user_message="What's the weather in Portland?", # OR or ME? + expected_tool_calls=[], +) +``` + +### Include context variations + +Capture with different conversation contexts: + +```python +suite.add_case( + name="weather_from_context", + user_message="How about the weather there?", + additional_messages=[ + {"role": "user", "content": "I'm going to Seattle"}, + ], + expected_tool_calls=[], +) +``` + +### Capture multiple providers + +Compare how different models interpret your tools: + +```bash +arcade evals . --capture \ + --use-provider openai:gpt-4o,gpt-4o-mini \ + --use-provider anthropic:claude-sonnet-4-5-20250929 \ + --file captures/models --format json,md +``` + +## Converting captures to tests + +### Step 1: Identify patterns + +Review captured tool calls to find patterns: + +```json +// Most queries use "fahrenheit" +{"location": "Seattle", "units": "fahrenheit"} +{"location": "Portland", "units": "fahrenheit"} + +// Some use "celsius" +{"location": "Tokyo", "units": "celsius"} +``` + +### Step 2: Create base expectations + +Create expected tool calls based on patterns: + +```python +# Default to fahrenheit for US cities +ExpectedMCPToolCall("GetWeather", {"location": "Seattle", "units": "fahrenheit"}) + +# Use celsius for international cities +ExpectedMCPToolCall("GetWeather", {"location": "Tokyo", "units": "celsius"}) +``` + +### Step 3: Add critics + +Add critics to validate parameters. See [Critics](/guides/create-tools/evaluate-tools/create-evaluation-suite#critics) for options. + +### Step 4: Run evaluations + +Test with real evaluations: + +```bash +arcade evals . --details +``` + +### Step 5: Iterate + +Use failures to refine: + +- Adjust expected values +- Change critic weights +- Modify tool descriptions +- Add more test cases + +## Troubleshooting + +### No tool calls captured + +**Symptom:** Empty `tool_calls` arrays + +**Possible causes:** + +1. Model didn't call any tools +2. Tools not properly registered +3. System message doesn't encourage tool use + +**Solution:** + +```python +suite = EvalSuite( + name="Weather", + system_message="You are a weather assistant. Use the available weather tools to answer questions.", +) +``` + +### Missing parameters + +**Symptom:** Some parameters are missing from captured calls + +**Explanation:** Models may omit optional parameters. + +**Solution:** Check if parameters have defaults in your schema. The evaluation framework applies defaults automatically. + +### Different results per provider + +**Symptom:** OpenAI and Anthropic capture different tool calls + +**Explanation:** Providers interpret tool descriptions differently. + +**Solution:** This is expected. Use captures to understand provider-specific behavior, then create provider-agnostic tests. + +## Next steps + +- Learn about [comparative evaluations](/guides/create-tools/evaluate-tools/comparative-evaluations) to compare tool sources +- [Create evaluation suites](/guides/create-tools/evaluate-tools/create-evaluation-suite) with expectations diff --git a/app/en/guides/create-tools/evaluate-tools/comparative-evaluations/page.mdx b/app/en/guides/create-tools/evaluate-tools/comparative-evaluations/page.mdx new file mode 100644 index 000000000..581c68cc1 --- /dev/null +++ b/app/en/guides/create-tools/evaluate-tools/comparative-evaluations/page.mdx @@ -0,0 +1,757 @@ +--- +title: "Comparative evaluations" +description: "Compare different tool implementations with the same test cases" +--- + +# Comparative evaluations + +Comparative evaluations let you test how well AI models select and use tools from different, isolated tool sources. Each "track" represents a separate tool registry, allowing you to compare implementations side-by-side. + +import { Callout, Steps } from "nextra/components"; + +## What are tracks? + +**Tracks are isolated tool registries** within a single evaluation suite. Each track has its own set of tools that are **not shared** with other tracks. This isolation lets you test how models perform when given different tool options for the same task. + +**Key concept**: Comparative evaluations test tool **selection** across different tool sets. Each track provides a different context (set of tools) to the model. + +**Common use cases:** + +- **Compare tool providers**: Test Google Weather vs OpenWeather API +- **Implementation comparison**: Test different MCP servers offering similar functionality +- **A/B testing**: Evaluate alternative tool designs + +### When to use comparative evaluations + +Use **comparative evaluations** when: + +- ✅ Testing multiple implementations of the same functionality +- ✅ Comparing different tool providers +- ✅ Evaluating how models choose between different tool sets + +Use **regular evaluations** when: + +- ✅ Testing a single tool implementation +- ✅ Testing mixed tools from multiple sources in the same context +- ✅ Regression testing + +### Testing mixed tool sources + +To test how multiple MCP servers work **together** in the same context (not isolated), use a regular evaluation and load multiple sources: + +```python +@tool_eval() +async def mixed_tools_eval(): + suite = EvalSuite(name="Mixed Tools", system_message="You are helpful.") + + # All tools available to the model in the same context + await suite.add_mcp_server("http://server1.example") + await suite.add_mcp_server("http://server2.example") + suite.add_tool_definitions([{"name": "CustomTool", ...}]) + + # Model can use any tool from any source + suite.add_case(...) + return suite +``` + +Alternatively, use an Arcade Gateway which aggregates tools from multiple sources. + +## Basic comparative evaluation + + + +### Register tools per track + +Create a suite and register tools for each track: + +```python +from arcade_evals import EvalSuite, tool_eval, ExpectedMCPToolCall, BinaryCritic + +@tool_eval() +async def weather_comparison(): + suite = EvalSuite( + name="Weather API Comparison", + system_message="You are a weather assistant.", + ) + + # Track A: Weather API v1 + await suite.add_mcp_server( + "http://weather-v1.example/mcp", + track="Weather v1" + ) + + # Track B: Weather API v2 + await suite.add_mcp_server( + "http://weather-v2.example/mcp", + track="Weather v2" + ) + + return suite +``` + +### Create comparative test case + +Add a test case with track-specific expectations: + +```python +suite.add_comparative_case( + name="get_current_weather", + user_message="What's the weather in Seattle?", +).for_track( + "Weather v1", + expected_tool_calls=[ + ExpectedMCPToolCall( + "GetWeather", + {"city": "Seattle", "type": "current"} + ) + ], + critics=[ + BinaryCritic(critic_field="city", weight=0.7), + BinaryCritic(critic_field="type", weight=0.3), + ], +).for_track( + "Weather v2", + expected_tool_calls=[ + ExpectedMCPToolCall( + "Weather_GetCurrent", + {"location": "Seattle"} + ) + ], + critics=[ + BinaryCritic(critic_field="location", weight=1.0), + ], +) +``` + +### Run comparative evaluation + +```bash +arcade evals . +``` + +Results show per-track scores: + +``` +Suite: Weather API Comparison + Case: get_current_weather + Track: Weather v1 -- Score: 1.00 -- PASSED + Track: Weather v2 -- Score: 1.00 -- PASSED +``` + + + +## Track registration + +### From MCP HTTP server + +```python +await suite.add_mcp_server( + url="http://localhost:8000", + headers={"Authorization": "Bearer token"}, + track="Production API", +) +``` + +### From MCP stdio server + +```python +await suite.add_mcp_stdio_server( + command=["python", "server_v2.py"], + env={"API_KEY": "secret"}, + track="Version 2", +) +``` + +### From Arcade Gateway + +```python +await suite.add_arcade_gateway( + gateway_slug="weather-gateway", + track="Arcade Gateway", +) +``` + +### Manual tool definitions + +```python +suite.add_tool_definitions( + tools=[ + { + "name": "GetWeather", + "description": "Get weather for a location", + "inputSchema": {...}, + } + ], + track="Custom Tools", +) +``` + + + Tools must be registered before creating comparative cases that reference + their tracks. + + +## Comparative case builder + +The `add_comparative_case()` method returns a builder for defining track-specific expectations. + +### Basic structure + +```python +suite.add_comparative_case( + name="test_case", + user_message="Do something", +).for_track( + "Track A", + expected_tool_calls=[...], + critics=[...], +).for_track( + "Track B", + expected_tool_calls=[...], + critics=[...], +) +``` + +### Optional parameters + +Add conversation context to comparative cases: + +```python +suite.add_comparative_case( + name="weather_with_context", + user_message="What about the weather there?", + system_message="You are helpful.", # Optional override + additional_messages=[ + {"role": "user", "content": "I'm going to Seattle"}, + ], +).for_track("Weather v1", ...).for_track("Weather v2", ...) +``` + +**Bias-aware message design:** + +Design `additional_messages` to avoid leading the model. Keep them neutral so you measure tool behavior, not prompt hints: + +```python +# ✅ Good - Neutral +additional_messages=[ + {"role": "user", "content": "I need weather information"}, + {"role": "assistant", "content": "I can help with that. Which location?"}, +] + +# ❌ Avoid - Tells the model which tool to call +additional_messages=[ + {"role": "user", "content": "Use the GetWeather tool for Seattle"}, +] +``` + +Keep messages generic so the model chooses tools naturally based on what is available in the track. + +### Different expectations per track + +Tracks can expose different tools and schemas. Because of that, you may need different critics per track: + +```python +suite.add_comparative_case( + name="search_query", + user_message="Search for Python tutorials", +).for_track( + "Google Search", + expected_tool_calls=[ + ExpectedMCPToolCall("Google_Search", {"query": "Python tutorials"}) + ], + critics=[BinaryCritic(critic_field="query", weight=1.0)], +).for_track( + "Bing Search", + expected_tool_calls=[ + ExpectedMCPToolCall("Bing_WebSearch", {"q": "Python tutorials"}) + ], + # Different schema, so validate the matching field for this track + critics=[BinaryCritic(critic_field="q", weight=1.0)], +) +``` + +## Complete example + +Here's a full comparative evaluation: + +```python +from arcade_evals import ( + EvalSuite, + tool_eval, + ExpectedMCPToolCall, + BinaryCritic, + SimilarityCritic, +) + +@tool_eval() +async def search_comparison(): + """Compare different search APIs.""" + suite = EvalSuite( + name="Search API Comparison", + system_message="You are a search assistant. Use the available tools to search for information.", + ) + + # Register search providers (MCP servers) + await suite.add_mcp_server( + "http://google-search.example/mcp", + track="Google", + ) + + await suite.add_mcp_server( + "http://bing-search.example/mcp", + track="Bing", + ) + + # Mix with manual tool definitions + suite.add_tool_definitions( + tools=[{ + "name": "DDG_Search", + "description": "Search using DuckDuckGo", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string"} + }, + "required": ["query"] + } + }], + track="DuckDuckGo", + ) + + # Simple query + suite.add_comparative_case( + name="basic_search", + user_message="Search for Python tutorials", + ).for_track( + "Google", + expected_tool_calls=[ + ExpectedMCPToolCall("Search", {"query": "Python tutorials"}) + ], + critics=[BinaryCritic(critic_field="query", weight=1.0)], + ).for_track( + "Bing", + expected_tool_calls=[ + ExpectedMCPToolCall("WebSearch", {"q": "Python tutorials"}) + ], + critics=[BinaryCritic(critic_field="q", weight=1.0)], + ).for_track( + "DuckDuckGo", + expected_tool_calls=[ + ExpectedMCPToolCall("DDG_Search", {"query": "Python tutorials"}) + ], + critics=[BinaryCritic(critic_field="query", weight=1.0)], + ) + + # Query with filters + suite.add_comparative_case( + name="search_with_filters", + user_message="Search for Python tutorials from the last month", + ).for_track( + "Google", + expected_tool_calls=[ + ExpectedMCPToolCall( + "Search", + {"query": "Python tutorials", "time_range": "month"} + ) + ], + critics=[ + SimilarityCritic(critic_field="query", weight=0.7), + BinaryCritic(critic_field="time_range", weight=0.3), + ], + ).for_track( + "Bing", + expected_tool_calls=[ + ExpectedMCPToolCall( + "WebSearch", + {"q": "Python tutorials", "freshness": "Month"} + ) + ], + critics=[ + SimilarityCritic(critic_field="q", weight=0.7), + BinaryCritic(critic_field="freshness", weight=0.3), + ], + ).for_track( + "DuckDuckGo", + expected_tool_calls=[ + ExpectedMCPToolCall( + "DDG_Search", + {"query": "Python tutorials"} + ) + ], + critics=[ + SimilarityCritic(critic_field="query", weight=1.0), + ], + ) + + return suite +``` + +Run the comparison: + +```bash +arcade evals . --details +``` + +Output shows side-by-side results: + +``` +Suite: Search API Comparison + +Case: basic_search + Track: Google -- Score: 1.00 -- PASSED + Track: Bing -- Score: 1.00 -- PASSED + Track: DuckDuckGo -- Score: 1.00 -- PASSED + +Case: search_with_filters + Track: Google -- Score: 1.00 -- PASSED + Track: Bing -- Score: 0.85 -- WARNED + Track: DuckDuckGo -- Score: 0.90 -- WARNED +``` + +## Result structure + +Comparative results are organized by track: + +```python +{ + "Google": { + "model": "gpt-4o", + "suite_name": "Search API Comparison", + "track_name": "Google", + "rubric": {...}, + "cases": [ + { + "name": "basic_search", + "track": "Google", + "input": "Search for Python tutorials", + "expected_tool_calls": [...], + "predicted_tool_calls": [...], + "evaluation": { + "score": 1.0, + "result": "passed", + ... + } + } + ] + }, + "Bing": {...}, + "DuckDuckGo": {...} +} +``` + +## Mixing regular and comparative cases + +A suite can have both regular and comparative cases: + +```python +@tool_eval() +async def mixed_suite(): + suite = EvalSuite( + name="Mixed Evaluation", + system_message="You are helpful.", + ) + + # Register default tools + await suite.add_mcp_stdio_server(["python", "server.py"]) + + # Regular case (uses default tools) + suite.add_case( + name="regular_test", + user_message="Do something", + expected_tool_calls=[...], + ) + + # Register track-specific tools + await suite.add_mcp_server("http://api-v2.example", track="v2") + + # Comparative case + suite.add_comparative_case( + name="compare_versions", + user_message="Do something else", + ).for_track( + "default", # Uses default tools + expected_tool_calls=[...], + ).for_track( + "v2", # Uses v2 tools + expected_tool_calls=[...], + ) + + return suite +``` + + + Use track name `"default"` to reference tools registered without a track. + + +## Capture mode with tracks + +Capture tool calls from each track separately: + +```bash +arcade evals . --capture --file captures/comparison --format json +``` + +Output includes track names: + +```json +{ + "captured_cases": [ + { + "case_name": "get_weather", + "track_name": "Weather v1", + "tool_calls": [ + {"name": "GetWeather", "args": {...}} + ] + }, + { + "case_name": "get_weather", + "track_name": "Weather v2", + "tool_calls": [ + {"name": "Weather_GetCurrent", "args": {...}} + ] + } + ] +} +``` + +## Multi-model comparative evaluations + +Combine comparative tracks with multiple models: + +```bash +arcade evals . \ + --use-provider openai:gpt-4o,gpt-4o-mini \ + --use-provider anthropic:claude-sonnet-4-5-20250929 +``` + +Results show: + +- Per-track scores for each model +- Cross-track comparisons for each model +- Cross-model comparisons for each track + +Example output: + +``` +Suite: Weather API Comparison + +Model: gpt-4o + Case: get_weather + Track: Weather v1 -- Score: 1.00 -- PASSED + Track: Weather v2 -- Score: 1.00 -- PASSED + +Model: gpt-4o-mini + Case: get_weather + Track: Weather v1 -- Score: 0.90 -- WARNED + Track: Weather v2 -- Score: 0.95 -- PASSED + +Model: claude-sonnet-4-5-20250929 + Case: get_weather + Track: Weather v1 -- Score: 1.00 -- PASSED + Track: Weather v2 -- Score: 0.85 -- WARNED +``` + +## Best practices + +### Use descriptive track names + +Choose clear names that indicate what's being compared: + +```python +# ✅ Good +track="Weather API v1" +track="OpenWeather Production" +track="Google Weather (Staging)" + +# ❌ Avoid +track="A" +track="Test1" +track="Track2" +``` + +### Keep test cases consistent + +Use the same user message and context across tracks: + +```python +suite.add_comparative_case( + name="get_weather", + user_message="What's the weather in Seattle?", # Same for all tracks +).for_track("v1", ...).for_track("v2", ...) +``` + +### Adjust critics to track differences + +Different tools may have different parameter names or types: + +```python +.for_track( + "Weather v1", + expected_tool_calls=[ + ExpectedMCPToolCall("GetWeather", {"city": "Seattle"}) + ], + critics=[ + BinaryCritic(critic_field="city", weight=1.0), # v1 uses "city" + ], +).for_track( + "Weather v2", + expected_tool_calls=[ + ExpectedMCPToolCall("GetWeather", {"location": "Seattle"}) + ], + critics=[ + BinaryCritic(critic_field="location", weight=1.0), # v2 uses "location" + ], +) +``` + +### Start with capture mode + +Use capture mode to discover track-specific tool signatures: + +```bash +arcade evals . --capture +``` + +Then create expectations based on captured calls. + +### Test edge cases per track + +Different implementations may handle edge cases differently: + +```python +suite.add_comparative_case( + name="ambiguous_location", + user_message="What's the weather in Portland?", # OR or ME? +).for_track( + "Weather v1", + # v1 defaults to most populous + expected_tool_calls=[ + ExpectedMCPToolCall("GetWeather", {"city": "Portland", "state": "OR"}) + ], +).for_track( + "Weather v2", + # v2 requires disambiguation + expected_tool_calls=[ + ExpectedMCPToolCall("DisambiguateLocation", {"city": "Portland"}), + ExpectedMCPToolCall("GetWeather", {"city": "Portland", "state": "OR"}), + ], +) +``` + +## Troubleshooting + +### Track not found + +**Symptom:** `ValueError: Track 'TrackName' not registered` + +**Solution:** Register the track before adding comparative cases: + +```python +# ✅ Correct order +await suite.add_mcp_server(url, track="TrackName") +suite.add_comparative_case(...).for_track("TrackName", ...) + +# ❌ Wrong order - will fail +suite.add_comparative_case(...).for_track("TrackName", ...) +await suite.add_mcp_server(url, track="TrackName") +``` + +### Missing track expectations + +**Symptom:** Case runs against some tracks but not others + +**Explanation:** Comparative cases only run against tracks with `.for_track()` defined. + +**Solution:** Add expectations for all registered tracks: + +```python +suite.add_comparative_case( + name="test", + user_message="...", +).for_track("Track A", ...).for_track("Track B", ...) +``` + +### Tool name mismatches + +**Symptom:** "Tool not found" errors in specific tracks + +**Solution:** Check tool names in each track: + +```python +# List tools per track +print(suite.list_tool_names(track="Track A")) +print(suite.list_tool_names(track="Track B")) +``` + +Use the exact tool names from the output. + +### Inconsistent results across tracks + +**Symptom:** Same user message produces different scores across tracks + +**Explanation:** This is expected. Different tool implementations may work differently. + +**Solution:** Adjust expectations and critics per track to account for implementation differences. + +## Advanced patterns + +### Baseline comparison + +Compare new implementations against a baseline: + +```python +await suite.add_mcp_server( + "http://production.example/mcp", + track="Production (Baseline)" +) + +await suite.add_mcp_server( + "http://staging.example/mcp", + track="Staging (New)" +) +``` + +Results show deviations from baseline. + +### Progressive feature testing + +Test feature support across versions: + +```python +suite.add_comparative_case( + name="advanced_filters", + user_message="Search with advanced filters", +).for_track( + "v1", + expected_tool_calls=[], # Not supported +).for_track( + "v2", + expected_tool_calls=[ + ExpectedMCPToolCall("SearchWithFilters", {...}) + ], +) +``` + +### Tool catalog comparison + +Compare Arcade tool catalogs: + +```python +from arcade_core import ToolCatalog +from my_tools import weather_v1, weather_v2 + +catalog_v1 = ToolCatalog() +catalog_v1.add_tool(weather_v1, "Weather") + +catalog_v2 = ToolCatalog() +catalog_v2.add_tool(weather_v2, "Weather") + +suite.add_tool_catalog(catalog_v1, track="Python v1") +suite.add_tool_catalog(catalog_v2, track="Python v2") +``` + +## Next steps + +- [Create an evaluation suite](/guides/create-tools/evaluate-tools/create-evaluation-suite) with tracks +- Use [capture mode](/guides/create-tools/evaluate-tools/capture-mode) to discover track-specific tool calls +- [Run evaluations](/guides/create-tools/evaluate-tools/run-evaluations) with multiple models and tracks diff --git a/app/en/guides/create-tools/evaluate-tools/create-evaluation-suite/page.mdx b/app/en/guides/create-tools/evaluate-tools/create-evaluation-suite/page.mdx index 894638a7d..2cf1d2668 100644 --- a/app/en/guides/create-tools/evaluate-tools/create-evaluation-suite/page.mdx +++ b/app/en/guides/create-tools/evaluate-tools/create-evaluation-suite/page.mdx @@ -3,11 +3,9 @@ title: "Create an evaluation suite" description: "Learn how to evaluate your tools using Arcade" --- -# Evaluate tools +# Create an evaluation suite -In this guide, you'll learn how to evaluate your tools to ensure they are selected and used correctly by an AI model. You'll define evaluation cases and use different critics to assess the outcome of your evaluations. - -We'll create evaluation cases to test the `greet` tool and measure its performance. +Evaluation suites help you test whether AI models use your tools correctly. This guide shows you how to create test cases that measure tool selection and parameter accuracy. import { Steps, Tabs, Callout } from "nextra/components"; @@ -18,85 +16,76 @@ import { Steps, Tabs, Callout } from "nextra/components"; - [Create an MCP Server](/guides/create-tools/tool-basics/build-mcp-server) - Install the evaluation dependencies: - + - ```bash - uv tool install 'arcade-mcp[evals]' - ``` +```bash +uv tool install 'arcade-mcp[evals]' +``` - ```bash - pip install 'arcade-mcp[evals]' - ``` +```bash +pip install 'arcade-mcp[evals]' +``` -### Create an evaluation suite +### Create an evaluation file -Navigate to your MCP Server's directory +Navigate to your MCP server directory and create a file starting with `eval_`: ```bash cd my_server +touch eval_server.py ``` -Create a new Python file for your evaluations, e.g., `eval_server.py`. - - For evals, the file name should start with `eval_` and be a Python script - (using the `.py` extension). + Evaluation files must start with `eval_` and use the `.py` extension. The CLI + automatically discovers these files. -### Define your evaluation cases +### Define your evaluation suite -Open `eval_server.py` and add the following code: +Create an evaluation suite that loads tools from your MCP server and defines test cases: ```python from arcade_evals import ( - EvalSuite, tool_eval, EvalRubric, - ExpectedToolCall, BinaryCritic -) -from arcade_core import ToolCatalog - -from server import greet - -# Create a catalog of tools to include in the evaluation -catalog = ToolCatalog() -catalog.add_tool(greet, "Greet") - -# Create rubric with tool calls -rubric = EvalRubric( - fail_threshold=0.8, - warn_threshold=0.9, + EvalSuite, + tool_eval, + ExpectedMCPToolCall, + BinaryCritic, ) @tool_eval() -def hello_eval_suite() -> EvalSuite: - """Create an evaluation suite for the hello tool.""" +async def weather_eval_suite() -> EvalSuite: + """Evaluate weather tool usage.""" suite = EvalSuite( - name="MCP Server Evaluation", - catalog=catalog, - system_message="You are a helpful assistant.", - rubric=rubric, + name="Weather Tools", + system_message="You are a helpful weather assistant.", + ) + + # Load tools from your MCP server + await suite.add_mcp_stdio_server( + command=["python", "server.py"], ) + # Add a test case suite.add_case( - name="Simple Greeting", - user_message="Greet Alice", + name="Get weather for city", + user_message="What's the weather in Seattle?", expected_tool_calls=[ - ExpectedToolCall( - func=greet, - args={ - "name": "Alice", - }, + ExpectedMCPToolCall( + "Weather_GetCurrent", + {"location": "Seattle", "units": "celsius"} ) ], critics=[ - BinaryCritic(critic_field="name", weight=1.0), + BinaryCritic(critic_field="location", weight=0.7), + BinaryCritic(critic_field="units", weight=0.3), ], ) @@ -105,215 +94,248 @@ def hello_eval_suite() -> EvalSuite: ### Run the evaluation -From the server directory, ensure you have an OpenAI API key set in the `OPENAI_API_KEY` environment variable. Then run: +Set your OpenAI API key and run the evaluation: ```bash -export OPENAI_API_KEY= +export OPENAI_API_KEY= arcade evals . ``` -This command executes your evaluation suite and provides a report. +The command discovers all `eval_*.py` files and executes them using OpenAI's `gpt-4o` model by default. - - By default, the evaluation suite will use the `gpt-4o` model. You can specify - a different model and provider using the `--models` and `--provider` options. - If you are using a different provider, you will need to set the appropriate - API key in an environment variable, or use the `--provider-api-key` option. - For more information, see the [Run - evaluations](/guides/create-tools/evaluate-tools/run-evaluations) guide. - +**Using different providers:** -### How it works +```bash +# Anthropic +export ANTHROPIC_API_KEY= +arcade evals . -p anthropic -The evaluation framework in Arcade allows you to define test cases (`EvalCase`) with expected tool calls and use critics to assess an AI model's performance. +# Or specify API key directly +arcade evals . -p anthropic -k anthropic: -Similar to how a unit test suite measures the validity and performance of a function, an eval suite measures how well an AI model understands and uses your tools. +# Multiple models +arcade evals . -p "openai:gpt-4o,gpt-4o-mini" -### Next steps +# Multiple providers (space-separated) +arcade evals . -p "openai anthropic" -k openai:sk-... -k anthropic:sk-ant-... +``` -- Explore [different types of critics](#critic-classes) and [more complex evaluation cases](#advanced-evaluation-cases) to thoroughly test your tools. -- Understand [how to specify options for your evaluation runs](/guides/create-tools/evaluate-tools/run-evaluations). +See [Run evaluations](/guides/create-tools/evaluate-tools/run-evaluations) for all available options. - +### Understand the results -## Critic classes +Evaluation results show: -Critics are used to evaluate the correctness of tool calls. For simple tools, "correct" might be binary: is it exactly what we expected? For more complex tools, we might need to evaluate the similarity between expected and actual values, or measure numeric values within an acceptable range. +- **Passed**: Score meets or exceeds the fail threshold (default: 0.8) +- **Failed**: Score falls below the fail threshold +- **Warned**: Score is between warn and fail thresholds (default: 0.9) -Arcade's evaluation framework provides several critic classes to help you evaluate both exact and "fuzzy" matches between expected and actual values when a model predicts the parameters of a tool call. +Example output: -### BinaryCritic +``` +Suite: Weather Tools + Model: gpt-4o + PASSED Get weather for city -- Score: 1.00 -Checks if a parameter value matches exactly. +Summary -- Total: 1 -- Passed: 1 -- Failed: 0 +``` -```python -BinaryCritic(critic_field="name", weight=1.0) +Use `--details` to see critic feedback: + +```bash +arcade evals . --details +``` + +Detailed output includes per-critic scores: + +``` +PASSED Get weather for city -- Score: 1.00 + Details: + location: + Match: True, Score: 0.70/0.70 + units: + Match: True, Score: 0.30/0.30 ``` -### SimilarityCritic + + +## Loading tools + +You can load tools from different sources. All methods are async and must be awaited in your `@tool_eval()` decorated function. -Evaluates the similarity between expected and actual values. +### From MCP HTTP server + +Load tools from an HTTP or SSE MCP server: ```python -from arcade_evals import SimilarityCritic +await suite.add_mcp_server( + url="http://localhost:8000", + headers={"Authorization": "Bearer token"}, # Optional + timeout=10, # Optional: Connection timeout (default: 10) + use_sse=False, # Optional: Use SSE transport (default: False) +) +``` + +The URL is automatically normalized (appends `/mcp` if not present). + +### From MCP stdio server + +Load tools from a stdio MCP server process: -SimilarityCritic(critic_field="message", weight=1.0) +```python +await suite.add_mcp_stdio_server( + command=["python", "server.py"], + env={"API_KEY": "secret"}, # Optional: Environment variables + timeout=10, # Optional: Connection timeout (default: 10) +) ``` -### NumericCritic +### From Arcade Gateway -Assesses numeric values within a specified tolerance. +Load tools from an Arcade MCP Gateway: ```python -from arcade_evals import NumericCritic +await suite.add_arcade_gateway( + gateway_slug="my-gateway", + arcade_api_key="your-api-key", # Optional: Defaults to ARCADE_API_KEY env var + arcade_user_id="user-id", # Optional: Defaults to ARCADE_USER_ID env var + base_url=None, # Optional: Override gateway URL + timeout=10, # Optional: Connection timeout (default: 10) +) +``` + +### Manual tool definitions -NumericCritic(critic_field="score", tolerance=0.1, weight=1.0) +Define tools manually using MCP format: + +```python +suite.add_tool_definitions([ + { + "name": "Weather.GetCurrent", + "description": "Get current weather for a location", + "inputSchema": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "units": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "default": "celsius" + }, + }, + "required": ["location"], + }, + } +]) ``` -### DatetimeCritic +### Mixing tool sources -Evaluates the closeness of datetime values within a specified tolerance. +You can load tools from multiple sources into the same suite: ```python -from datetime import timedelta -from arcade_evals import DatetimeCritic +# Load from multiple MCP servers +await suite.add_mcp_server("http://server1.example") +await suite.add_mcp_server("http://server2.example") -DatetimeCritic(critic_field="start_time", tolerance=timedelta(seconds=10), weight=1.0) +# Mix with manual definitions +suite.add_tool_definitions([{"name": "CustomTool", ...}]) ``` -## Advanced evaluation cases +All tools are accumulated in the suite's registry and available to the model. -You can add more evaluation cases to test different scenarios. +## Expected tool calls - - Ensure that your `greet` tool and evaluation cases are updated accordingly and - that you rerun `arcade evals .` to test your changes. +Expected tool calls define what the model should predict. Use `ExpectedMCPToolCall` with MCP-style tool names: - If your evals fail, use `--details` to see the detailed feedback from each critic. See [Run evaluations](/guides/create-tools/evaluate-tools/run-evaluations) to understand the options available in `arcade evals`. +```python +ExpectedMCPToolCall( + "Weather_GetCurrent", + {"location": "Seattle", "units": "celsius"} +) +``` + + + Tool names are normalized for compatibility with model tool calling. Dots + (`.`) become underscores (`_`). For example, `Weather.GetCurrent` becomes + `Weather_GetCurrent`. +## Critics -### Example: Greeting with emotion +Critics validate tool call parameters. Each critic type handles different validation needs: -Modify your `hello` tool to accept an `emotion` parameter: +| Critic | Use case | Example | +| ------------------ | --------------- | ------------------------------------------------------------------ | +| `BinaryCritic` | Exact match | `BinaryCritic(critic_field="user_id", weight=1.0)` | +| `SimilarityCritic` | Text similarity | `SimilarityCritic(critic_field="message", weight=0.8)` | +| `NumericCritic` | Numeric range | `NumericCritic(critic_field="temp", tolerance=2.0)` | +| `DatetimeCritic` | Time window | `DatetimeCritic(critic_field="due", tolerance=timedelta(hours=1))` | ```python -from enum import Enum - -class Emotion(str, Enum): - HAPPY = "happy" - SLIGHTLY_HAPPY = "slightly happy" - SAD = "sad" - SLIGHTLY_SAD = "slightly sad" - -@app.tool -def greet( - name: Annotated[str, "The name of the person to greet"], - emotion: Annotated[ - Emotion, "The emotion to convey. Defaults to happy if omitted." - ] = Emotion.HAPPY, -) -> Annotated[str, "A greeting to the user"]: - """ - Greet a person by name, optionally with a specific emotion. - """ - return f"Hello {name}! I'm feeling {emotion.value} today." +from arcade_evals import BinaryCritic, SimilarityCritic + +critics=[ + BinaryCritic(critic_field="location", weight=0.7), + SimilarityCritic(critic_field="message", weight=0.3), +] ``` -Add an evaluation case for this new parameter: +All weights are normalized proportionally to sum to 1.0. Use numeric values or `FuzzyWeight` (`CRITICAL`, `HIGH`, `MEDIUM`, `LOW`). -```python -# At the top of the file: -from server import Emotion -from arcade_evals import SimilarityCritic +## Multiple tool calls + +Test cases can include multiple expected tool calls: -# Inside hello_eval_suite(): +```python suite.add_case( - name="Greeting with Emotion", - user_message="Say hello to Bob sadly", + name="Check weather in multiple cities", + user_message="What's the weather in Seattle and Portland?", expected_tool_calls=[ - ExpectedToolCall( - func=greet, - args={ - "name": "Bob", - "emotion": Emotion.SAD, - }, - ) - ], - critics=[ - BinaryCritic(critic_field="name", weight=0.5), - SimilarityCritic(critic_field="emotion", weight=0.5), + ExpectedMCPToolCall("Weather_GetCurrent", {"location": "Seattle"}), + ExpectedMCPToolCall("Weather_GetCurrent", {"location": "Portland"}), ], ) ``` -Add an evaluation case with additional conversation context: +## Conversation context + +Add conversation history to test cases that require context: ```python suite.add_case( - name="Greeting with Emotion from Context", - user_message="Say hello to Bob based on my current mood.", + name="Weather based on previous location", + user_message="What about the weather there?", expected_tool_calls=[ - ExpectedToolCall( - func=greet, - args={ - "name": "Bob", - "emotion": Emotion.HAPPY, - }, - ) + ExpectedMCPToolCall("Weather_GetCurrent", {"location": "Tokyo"}), ], - critics=[ - BinaryCritic(critic_field="name", weight=0.5), - SimilarityCritic(critic_field="emotion", weight=0.5), + additional_messages=[ + {"role": "user", "content": "I'm planning to visit Tokyo next week."}, + {"role": "assistant", "content": "That sounds exciting! What would you like to know about Tokyo?"}, ], - # Add some context to the evaluation case - additional_messages= [ - {"role": "user", "content": "Hi, I'm so happy!"}, - { - "role": "assistant", - "content": "That's awesome! What's got you feeling so happy today?", - }, - ] ) ``` -Add an evaluation case with multiple expected tool calls: +Use OpenAI message format for `additional_messages`. Arcade converts it automatically for Anthropic. + +## Rubrics + +Customize pass/fail thresholds with `EvalRubric`. Default: fail at 0.8, warn at 0.9. ```python -suite.add_case( - name="Multiple Greetings with Emotion from Context", - user_message="Say hello to Bob based on my current mood. And then say hello to Alice with slightly less of that emotion.", - expected_tool_calls=[ - ExpectedToolCall( - func=greet, - args={ - "name": "Bob", - "emotion": Emotion.HAPPY, - }, - ), - ExpectedToolCall( - func=greet, - args={ - "name": "Alice", - "emotion": Emotion.SLIGHTLY_HAPPY, - }, - ) - ], - critics=[ - BinaryCritic(critic_field="name", weight=0.5), - SimilarityCritic(critic_field="emotion", weight=0.5), - ], - # Add some context to the evaluation case - additional_messages= [ - {"role": "user", "content": "Hi, I'm so happy!"}, - { - "role": "assistant", - "content": "That's awesome! What's got you feeling so happy today?", - }, - ] +from arcade_evals import EvalRubric + +suite = EvalSuite( + name="Strict Evaluation", + system_message="You are helpful.", + rubric=EvalRubric(fail_threshold=0.85, warn_threshold=0.95), ) ``` +If you want stricter suites, increase thresholds (for example `fail_threshold=0.95`). For exploratory testing, lower them (for example `fail_threshold=0.6`). + ## Next steps -- **See an example MCP server with evaluations**: [Source code of a server with evaluations](https://github.com/ArcadeAI/arcade-mcp/tree/139cc2e54db0e5815f1c79dbe9e3285b4fe2bd66/examples/mcp_servers/server_with_evaluations) -- **Learn how to run evaluations**: [Run evaluations](/guides/create-tools/evaluate-tools/run-evaluations) +- Learn how to [run evaluations with different providers](/guides/create-tools/evaluate-tools/run-evaluations) +- Explore [capture mode](/guides/create-tools/evaluate-tools/capture-mode) to record tool calls +- Compare tool sources with [comparative evaluations](/guides/create-tools/evaluate-tools/comparative-evaluations) diff --git a/app/en/guides/create-tools/evaluate-tools/run-evaluations/page.mdx b/app/en/guides/create-tools/evaluate-tools/run-evaluations/page.mdx index d0fa3ffa7..31d2b9f47 100644 --- a/app/en/guides/create-tools/evaluate-tools/run-evaluations/page.mdx +++ b/app/en/guides/create-tools/evaluate-tools/run-evaluations/page.mdx @@ -3,215 +3,398 @@ title: "Run evaluations" description: "Learn how to run evaluations using Arcade" --- -# Run evaluations with the Arcade CLI +# Run evaluations -The Arcade Evaluation Framework allows you to run evaluations of your tool-enabled language models conveniently using the Arcade CLI. This enables you to execute your evaluation suites, gather results, and analyze the performance of your models in an efficient and streamlined manner. +The `arcade evals` command discovers and executes evaluation suites with support for multiple providers, models, and output formats. - - +import { Callout } from "nextra/components"; -Run evaluations of your tool-enabled language models using the Arcade CLI. + + **Backward compatibility**: All new features (multi-provider support, capture + mode, output formats) work with existing evaluation suites. No code changes + required. + - +## Basic usage - +Run all evaluations in the current directory: -- [Arcade CLI](/references/arcade-cli) -- [An MCP Server](/guides/create-tools/tool-basics/build-mcp-server) -- [Create an evaluation suite](/guides/create-tools/evaluate-tools/create-evaluation-suite) +```bash +arcade evals . +``` + +The command searches for files starting with `eval_` and ending with `.py`. + +Show detailed results with critic feedback: + +```bash +arcade evals . --details +``` + +Filter to show only failures: + +```bash +arcade evals . --only-failed +``` + +## Multi-provider support - +### Single provider with default model - +Use OpenAI with default model (`gpt-4o`): -- How to use the `arcade evals` CLI command to run evaluations. +```bash +export OPENAI_API_KEY=sk-... +arcade evals . +``` - - +Use Anthropic with default model (`claude-sonnet-4-5-20250929`): -### Using the `arcade evals` Command +```bash +export ANTHROPIC_API_KEY=sk-ant-... +arcade evals . --use-provider anthropic +``` -To run evaluations, use the `arcade evals` command provided by the Arcade CLI. This command searches for evaluation files in the specified directory, executes any functions decorated with `@tool_eval`, and displays the results. +### Specific models -#### Basic Usage +Specify one or more models for a provider: ```bash -arcade evals +arcade evals . --use-provider "openai:gpt-4o,gpt-4o-mini" ``` -- ``: The directory containing your evaluation files. By default, it searches the current directory (`.`). +### Multiple providers -For example, to run evaluations in the current directory: +Compare performance across providers (space-separated): ```bash -arcade evals +arcade evals . \ + --use-provider "openai:gpt-4o anthropic:claude-sonnet-4-5-20250929" \ + --api-key openai:sk-... \ + --api-key anthropic:sk-ant-... ``` -#### Evaluation File Naming Convention +When you specify multiple models, results show side-by-side comparisons. + +## API keys + +API keys are resolved in the following order: -The `arcade evals` command looks for Python files that start with `eval_` and end with `.py` (e.g., `eval_math_tools.py`, `eval_slack_messaging.py`). These files should contain your evaluation suites. +| Priority | Format | +|----------|--------| +| 1. Explicit flag | `--api-key provider:key` (can repeat) | +| 2. Environment | `OPENAI_API_KEY`, `ANTHROPIC_API_KEY` | +| 3. `.env` file | `OPENAI_API_KEY=...`, `ANTHROPIC_API_KEY=...` | -#### Command Options + + Create a `.env` file in your project directory to avoid setting keys in every terminal session. + -The `arcade evals` command supports several options to customize the evaluation process: +**Examples:** -- `--details`, `-d`: Show detailed results for each evaluation case, including critic feedback. +```bash +# Single provider +arcade evals . --api-key openai:sk-... - Example: +# Multiple providers +arcade evals . \ + --api-key openai:sk-... \ + --api-key anthropic:sk-ant-... +``` - ```bash - arcade evals --details . - ``` +## Capture mode -- `--models`, `-m`: Specify the models to use for evaluation. Provide a comma-separated list of model names. +Record tool calls without scoring to bootstrap test expectations: - Example: +```bash +arcade evals . --capture --output captures/baseline.json +``` - ```bash - arcade evals --models gpt-4o,gpt-5 . - ``` +Include conversation context in captured output: -- `--max-concurrent`, `-c`: Set the maximum number of concurrent evaluations to run in parallel. +```bash +arcade evals . --capture --include-context --output captures/detailed.json +``` - Example: +Capture mode is useful for: - ```bash - arcade evals --max-concurrent 4 . - ``` +- Creating initial test expectations +- Debugging model behavior +- Understanding tool call patterns -- `--provider`, `-p`: The provider of the models to use for evaluation. Uses OpenAI by default. +See [Capture mode](/guides/create-tools/evaluate-tools/capture-mode) for details. - Example: +## Output formats - ```bash - arcade evals --provider openai . - ``` +### Save results to files -- `--provider-api-key`, `-k`: The model provider API key. If not provided, will look for the appropriate environment variable based on the provider (e.g., OPENAI_API_KEY for openai provider), first in the current environment, then in the current working directory's .env file. +Specify output files with extensions - format is auto-detected: - Example: +```bash +# Single format +arcade evals . --output results.md - ```bash - arcade evals --provider-api-key my-api-key . - ``` +# Multiple formats +arcade evals . --output results.md --output results.html --output results.json -- `--debug`: Show debug information in the CLI. +# All formats (no extension) +arcade evals . --output results +``` - Example: +### Available formats - ```bash - arcade evals --debug . - ``` +| Extension | Format | Description | +|-----------|--------|-------------| +| `.txt` | Plain text | Pytest-style output | +| `.md` | Markdown | Tables and collapsible sections | +| `.html` | HTML | Interactive report | +| `.json` | JSON | Structured data for programmatic use | +| (none) | All formats | Generates all four formats | -- `--help`: Show help information and exit. +## Command options - Example: +### Quick reference - ```bash - arcade evals --help - ``` +| Flag | Short | Purpose | Example | +|------|-------|---------|---------| +| `--use-provider` | `-p` | Select provider/model | `-p "openai:gpt-4o"` | +| `--api-key` | `-k` | Provider API key | `-k openai:sk-...` | +| `--capture` | - | Record without scoring | `--capture` | +| `--details` | `-d` | Show critic feedback | `--details` | +| `--only-failed` | `-f` | Filter failures | `--only-failed` | +| `--output` | `-o` | Output file(s) | `-o results.md` | +| `--include-context` | - | Add messages to output | `--include-context` | +| `--max-concurrent` | `-c` | Parallel limit | `-c 10` | +| `--debug` | - | Debug info | `--debug` | -#### Example Command +### `--use-provider`, `-p` -Running evaluations in the `arcade_my_tools/evals` directory, showing detailed results, using the `gpt-5` model: +Specify provider(s) and model(s) (space-separated): ```bash -arcade evals arcade_my_tools/evals --details --models gpt-5 -k my-openai-api-key +--use-provider "[:] [[:]]" ``` -### Execution Process +**Supported providers:** +- `openai` (default: `gpt-4o`) +- `anthropic` (default: `claude-sonnet-4-5-20250929`) + + + Anthropic model names include date stamps. Check [Anthropic's model + documentation](https://docs.anthropic.com/en/docs/about-claude/models) for the + latest model versions. + + +**Examples:** + +```bash +# Default model for provider +arcade evals . -p anthropic -When you run the `arcade evals` command, the following steps occur: +# Specific model +arcade evals . -p "openai:gpt-4o-mini" -1. **Preparation**: The CLI loads the evaluation suites from the specified directory, looking for files that match the naming convention. +# Multiple models from same provider +arcade evals . -p "openai:gpt-4o,gpt-4o-mini" -2. **Execution**: The evaluation suites are executed asynchronously. Each suite's evaluation function, decorated with `@tool_eval`, is called with the appropriate configuration, including the model and concurrency settings. +# Multiple providers (space-separated) +arcade evals . -p "openai:gpt-4o anthropic:claude-sonnet-4-5-20250929" +``` -3. **Concurrency**: Evaluations can run concurrently based on the `--max-concurrent` setting, improving efficiency. +### `--api-key`, `-k` -4. **Result Aggregation**: Results from all evaluation cases and models are collected and aggregated. +Provide API keys explicitly (repeatable): -### Displaying Results +```bash +arcade evals . -k openai:sk-... -k anthropic:sk-ant-... +``` -After the evaluations are complete, the results are displayed in a concise and informative format, similar to testing frameworks like `pytest`. The output includes: +### `--capture` -- **Summary**: Shows the total number of cases, how many passed, failed, or issued warnings. +Enable capture mode to record tool calls without scoring: - Example: +```bash +arcade evals . --capture +``` - ``` - Summary -- Total: 5 -- Passed: 4 -- Failed: 1 - ``` +### `--include-context` -- **Detailed Case Results**: For each evaluation case, the status (PASSED, FAILED, WARNED), the case name, and the score are displayed. +Include system messages and conversation history in output: - Example: +```bash +arcade evals . --include-context --output results.md +``` - ``` - PASSED Add two large numbers -- Score: 1.00 - FAILED Send DM with ambiguous username -- Score: 0.75 - ``` +### `--output`, `-o` -- **Critic Feedback**: If the `--details` flag is used, detailed feedback from each critic is provided, highlighting matches, mismatches, and scores for each evaluated field. +Specify output file(s). Format is auto-detected from extension: - Example: +```bash +# Single format +arcade evals . -o results.md - ``` - Details: - user_name: - Match: False, Score: 0.00/0.50 - Expected: johndoe - Actual: john_doe - message: - Match: True, Score: 0.50/0.50 - ``` +# Multiple formats (repeat flag) +arcade evals . -o results.md -o results.html -### Interpreting the Results +# All formats (no extension) +arcade evals . -o results +``` -- **Passed**: The evaluation case met or exceeded the fail threshold specified in the rubric. +### `--details`, `-d` -- **Failed**: The evaluation case did not meet the fail threshold. +Show detailed results including critic feedback: -- **Warnings**: If the score is between the warn threshold and the fail threshold, a warning is issued. +```bash +arcade evals . --details +``` -Use the detailed feedback to understand where the model's performance can be improved, particularly focusing on mismatches identified by critics. +### `--only-failed`, `-f` -### Customizing Evaluations +Show only failed test cases: -You can customize the evaluation process by adjusting: +```bash +arcade evals . --only-failed +``` -- **Rubrics**: Modify fail and warn thresholds, and adjust weights to emphasize different aspects of evaluation. +### `--max-concurrent`, `-c` -- **Critics**: Add or modify critics in your evaluation cases to target specific arguments or behaviors. +Set maximum concurrent evaluations: -- **Concurrency**: Adjust the `--max-concurrent` option to optimize performance based on your environment. +```bash +arcade evals . --max-concurrent 10 +``` -### Handling Multiple Models +Default is 1 concurrent evaluation. -You can evaluate multiple models in a single run by specifying them in the `--models` option as a comma-separated list. This allows you to compare the performance of different models across the same evaluation suites. +### `--debug` -Example: +Show debug information for troubleshooting: ```bash -arcade evals . --models gpt-4o,gpt-5 +arcade evals . --debug ``` -### Considerations +Displays detailed error traces and connection information. -- **Evaluation Files**: Ensure your evaluation files are correctly named and contain the evaluation suites decorated with `@tool_eval`. +## Understanding results -- **Provider API Keys**: If you are using a different provider, you will need to set the appropriate API key in an environment variable, or use the `--provider-api-key` option. +Results are formatted based on evaluation type (regular, multi-model, or comparative) and selected flags. -- **Tool Catalog**: Ensure your tool catalog is correctly defined and includes all the tools you want to evaluate. +### Summary format -- **Weight distribution**: Ensure your weight distribution reflects the importance of each critic and that the sum of the weights is `1.0`. +Results show overall performance: + +``` +Summary -- Total: 5 -- Passed: 4 -- Failed: 1 +``` + +**How flags affect output:** + +- `--details`: Adds per-critic breakdown for each case +- `--only-failed`: Filters to show only failed cases (summary shows original totals) +- `--include-context`: Includes system messages and conversation history +- Multiple models: Switches to comparison table format +- Comparative tracks: Shows side-by-side track comparison + +### Case results + +Each case displays status and score: + +``` +PASSED Get weather for city -- Score: 1.00 +FAILED Weather with invalid city -- Score: 0.65 +``` + +### Detailed feedback + +Use `--details` to see critic-level analysis: + +``` +Details: + location: + Match: False, Score: 0.00/0.70 + Expected: Seattle + Actual: Seatle + units: + Match: True, Score: 0.30/0.30 +``` + +### Multi-model results + +When using multiple models, results show comparison tables: + +``` +Case: Get weather for city + Model: gpt-4o -- Score: 1.00 -- PASSED + Model: gpt-4o-mini -- Score: 0.95 -- WARNED +``` + +## Advanced usage + +### High concurrency for fast execution + +Increase concurrent evaluations: + +```bash +arcade evals . --max-concurrent 20 +``` + + + High concurrency may hit API rate limits. Start with default (1) and increase + gradually. + + +### Save comprehensive results + +Generate all formats with full details: + +```bash +arcade evals . --details --include-context --output results +``` + +This creates: +- `results.txt` +- `results.md` +- `results.html` +- `results.json` + +## Troubleshooting + +### Missing dependencies + +If you see `ImportError: MCP SDK is required`, install the full package: + +```bash +pip install 'arcade-mcp[evals]' +``` + +For Anthropic support: + +```bash +pip install anthropic +``` + +### Tool name mismatches + +Tool names are normalized (dots become underscores). Check your tool definitions if you see unexpected names. + +### API rate limits + +Reduce `--max-concurrent` value: + +```bash +arcade evals . --max-concurrent 2 +``` -## Conclusion +### No evaluation files found -Running evaluations using the Arcade CLI provides a powerful and convenient way to assess the tool-calling capabilities of your language models. By leveraging the `arcade evals` command, you can efficiently execute your evaluation suites, analyze results, and iterate on your models and tools. +Ensure your evaluation files: -Integrating this evaluation process into your development workflow helps ensure that your models interact with tools as expected, enhances reliability, and builds confidence in deploying actionable language models in production environments. +- Start with `eval_` +- End with `.py` +- Contain functions decorated with `@tool_eval()` ## Next steps -- **See an example MCP server with evaluations**: [Source code of a server with evaluations](https://github.com/ArcadeAI/arcade-mcp/tree/139cc2e54db0e5815f1c79dbe9e3285b4fe2bd66/examples/mcp_servers/server_with_evaluations) +- Explore [capture mode](/guides/create-tools/evaluate-tools/capture-mode) for recording tool calls +- Learn about [comparative evaluations](/guides/create-tools/evaluate-tools/comparative-evaluations) for comparing tool sources diff --git a/app/en/guides/create-tools/evaluate-tools/why-evaluate/page.mdx b/app/en/guides/create-tools/evaluate-tools/why-evaluate/page.mdx index 4f2af3a55..fc9ba861f 100644 --- a/app/en/guides/create-tools/evaluate-tools/why-evaluate/page.mdx +++ b/app/en/guides/create-tools/evaluate-tools/why-evaluate/page.mdx @@ -3,16 +3,19 @@ title: "Why evaluate tools?" description: "Learn why evaluating your tools is important" --- +import { Callout } from "nextra/components"; + # Why evaluate tools?
- When deploying language models with tool-calling capabilities in production environments, it's essential to ensure their effectiveness and reliability. This evaluation process goes beyond traditional testing and focuses on two key aspects: + Tool evaluations ensure AI models use your tools correctly in production. Unlike traditional testing, evaluations measure two key aspects: + + 1. **Tool selection**: Does the model choose the right tools for the task? + 2. **Parameter accuracy**: Does the model provide correct arguments? - 1. **Tool Utilization**: Assessing how efficiently the language model uses the available tools. - 2. **Intent Understanding**: Evaluating the language model's ability to comprehend user intents and select the appropriate tools to fulfill those intents. + Arcade's evaluation framework helps you validate tool-calling capabilities before deployment, ensuring reliability in real-world applications. You can evaluate tools from MCP servers, Arcade Gateways, or custom implementations. - Arcade's Evaluation Framework provides a comprehensive approach to assess and validate the tool-calling capabilities of language models, ensuring they meet the high standards required for real-world applications.
-## Why Evaluate Tool Calling by Task? - -Language models augmented with tool-use capabilities can perform complex tasks by invoking external tools or APIs. However, without proper evaluation, these models might: - -- **Misinterpret user intents**, leading to incorrect tool selection. -- **Provide incorrect arguments** to tools, causing failures or undesired outcomes. -- **Fail to execute the necessary sequence of tool calls**, especially in tasks requiring multiple steps. - -Evaluating tool calling by task ensures that the language model can handle specific scenarios reliably, providing confidence in its performance in production settings. - -## Evaluation Scoring - -Scoring in the evaluation framework is based on comparing the model's actual tool calls with the expected ones for each evaluation case. The total score for a case depends on: - -1. **Tool Selection**: Whether the model selected the correct tools for the task. -2. **Tool Call Arguments**: The correctness of the arguments provided to the tools, evaluated by critics. -3. **Evaluation Rubric**: Each aspect of the evaluation is weighted according to the rubric, affecting its impact on the final score. - -The evaluation result includes: - -- **Score**: A normalized value between 0.0 and 1.0. -- **Result**: - - _Passed_: Score is above the fail threshold. - - _Failed_: Score is below the fail threshold. - - _Warned_: Score is between the warning and fail thresholds. +## What can go wrong? -## Critics: Types and Usage +Without proper evaluation, AI models might: -Critics are essential for evaluating the correctness of tool call arguments. Different types of critics serve various evaluation needs: +- **Misinterpret user intents**, selecting the wrong tools +- **Provide incorrect arguments**, causing failures or unexpected behavior +- **Skip necessary tool calls**, missing steps in multi-step tasks +- **Make incorrect assumptions** about parameter defaults or formats -### BinaryCritic +## How evaluation works -`BinaryCritic`s check for exact matches between expected and actual values after casting. +Evaluations compare the model's actual tool calls with expected tool calls for each test case. -- **Use Case**: When exact values are required (e.g., specific numeric parameters). -- **Example**: Ensuring the model provides the exact user ID in a function call. +### Scoring components -### NumericCritic +1. **Tool selection**: Did the model choose the correct tool? +2. **Parameter evaluation**: Are the arguments correct? (evaluated by critics) +3. **Weighted scoring**: Each aspect has a weight that affects the final score -`NumericCritic` evaluates numeric values within a specified range, allowing for acceptable deviations. +### Evaluation results -- **Use Case**: When values can be approximate but should be within a certain threshold. -- **Example**: Accepting approximate results in mathematical computations due to floating-point precision. +Each test case receives: -### SimilarityCritic +- **Score**: Calculated from weighted critic scores, normalized proportionally (weights can be any positive value) +- **Status**: + - **Passed**: Score meets or exceeds fail threshold (default: 0.8) + - **Failed**: Score falls below fail threshold + - **Warned**: Score is between warn and fail thresholds (default: 0.9) -`SimilarityCritic` measures the similarity between expected and actual string values using metrics like cosine similarity. +Example output: -- **Use Case**: When the exact wording isn't critical, but the content should be similar. -- **Example**: Evaluating if the message content in a communication tool is similar to the expected message. - -### DatetimeCritic - -`DatetimeCritic` evaluates the closeness of datetime values within a specified tolerance. - -- **Use Case**: When datetime values should be within a certain range of the expected time. -- **Example**: Verifying if a scheduled event time is close enough to the intended time. - -### Choosing the Right Critic - -- **Exact Matches Needed**: Use **BinaryCritic** for strict equality. -- **Numeric Ranges**: Use **NumericCritic** when a tolerance is acceptable. -- **Textual Similarity**: Use **SimilarityCritic** for comparing messages or descriptions. -- **Datetime Tolerance**: Use **DatetimeCritic** when a tolerance is acceptable for datetime comparisons. - -Critics are defined with fields such as `critic_field`, `weight`, and parameters specific to their types (e.g., `similarity_threshold` for `SimilarityCritic`). +``` +PASSED Get weather for city -- Score: 1.00 +WARNED Send message with typo -- Score: 0.85 +FAILED Wrong tool selected -- Score: 0.50 +``` -## Rubrics and Setting Thresholds +## Next steps -An **EvalRubric** defines the evaluation criteria and thresholds for determining pass/fail outcomes. Key components include: +- [Create an evaluation suite](/guides/create-tools/evaluate-tools/create-evaluation-suite) to start testing your tools +- [Run evaluations](/guides/create-tools/evaluate-tools/run-evaluations) with multiple providers +- Explore [capture mode](/guides/create-tools/evaluate-tools/capture-mode) to bootstrap test expectations +- Compare tool sources with [comparative evaluations](/guides/create-tools/evaluate-tools/comparative-evaluations) -- **Fail Threshold**: The minimum score required to pass the evaluation. -- **Warn Threshold**: The score threshold for issuing a warning. -- **Weights**: Assigns importance to different aspects of the evaluation (e.g., tool selection, argument correctness). +## Advanced features -### Setting Up a Rubric +Once you're comfortable with basic evaluations, explore these advanced capabilities: -- **Define Fail and Warn Thresholds**: Choose values between 0.0 and 1.0 to represent acceptable performance levels. -- **Assign Weights**: Allocate weights to tool selection and critics to reflect their importance in the overall evaluation. -- **Configure Failure Conditions**: Set flags like `fail_on_tool_selection` to enforce strict criteria. +### Capture mode -### Example Rubric Configuration: +Record tool calls without scoring to discover what models actually call. Useful for bootstrapping test expectations and debugging. [Learn more →](/guides/create-tools/evaluate-tools/capture-mode) -A rubric that requires a score of at least 0.85 to pass and issues a warning if the score is between 0.85 and 0.95: +### Comparative evaluations -- Fail Threshold: 0.85 -- Warn Threshold: 0.95 -- Fail on Tool Selection: True -- Tool Selection Weight: 1.0 +Test the same cases against different tool sources (tracks) with isolated registries. Compare how models perform with different tool implementations. [Learn more →](/guides/create-tools/evaluate-tools/comparative-evaluations) -```python -rubric = EvalRubric( - fail_threshold=0.85, - warn_threshold=0.95, - fail_on_tool_selection=True, - tool_selection_weight=1.0, -) -``` +### Output formats -## Building an Evaluation Suite - -An **EvalSuite** orchestrates the running of multiple evaluation cases. Here's how to build one: - -1. **Initialize EvalSuite**: Provide a name, system message, tool catalog, and rubric. -2. **Add Evaluation Cases**: Use `add_case` or `extend_case` to include various scenarios. -3. **Specify Expected Tool Calls**: Define the tools and arguments expected for each case. -4. **Assign Critics**: Attach critics relevant to each case to evaluate specific arguments. -5. **Run the Suite**: Execute the suite using the Arcade CLI to collect results. - -### Example: Math Tools Evaluation Suite - -An evaluation suite for math tools might include cases such as: - -- **Adding Two Large Numbers**: - - **User Message**: "Add 12345 and 987654321" - - **Expected Tool Call**: `add(a=12345, b=987654321)` - - **Critics**: - - `BinaryCritic` for arguments `a` and `b` -- **Calculating Square Roots**: - - **User Message**: "What is the square root of 3224990521?" - - **Expected Tool Call**: `sqrt(a=3224990521)` - - **Critics**: - - `BinaryCritic` for argument `a` - -### Example: Slack Messaging Tools Evaluation Suite - -An evaluation suite for Slack messaging tools might include cases such as: - -- **Sending a Direct Message**: - - **User Message**: "Send a direct message to johndoe saying 'Hello, can we meet at 3 PM?'" - - **Expected Tool Call**: `send_dm_to_user(user_name='johndoe', message='Hello, can we meet at 3 PM?')` - - **Critics**: - - `BinaryCritic` for `user_name` - - `SimilarityCritic` for `message` -- **Posting a Message to a Channel**: - - **User Message**: "Post 'The new feature is now live!' in the #announcements channel" - - **Expected Tool Call**: `send_message_to_channel(channel_name='announcements', message='The new feature is now live!')` - - **Critics**: - - `BinaryCritic` for `channel_name` - - `SimilarityCritic` for `message` +Save results in multiple formats (txt, md, html, json) for reporting and analysis. Specify output files with extensions or use no extension for all formats. [Learn more →](/guides/create-tools/evaluate-tools/run-evaluations#output-formats) diff --git a/app/en/references/cli-cheat-sheet/page.mdx b/app/en/references/cli-cheat-sheet/page.mdx index db1283113..351f12857 100644 --- a/app/en/references/cli-cheat-sheet/page.mdx +++ b/app/en/references/cli-cheat-sheet/page.mdx @@ -333,25 +333,93 @@ import '../../../cheat-sheet-print.css' Test tool-calling accuracy with evaluation suites. - ### Run Evaluations + + ### Basic Usage ```bash - arcade evals # Current dir - arcade evals ./evals/ # Specific dir + # Run all evals in current directory + arcade evals + + # Run specific directory + arcade evals ./evals/ + + # Show detailed results + arcade evals -d + + # Filter failed tests only + arcade evals --only-failed ``` + ### Capture Mode + Record tool calls without scoring to bootstrap test expectations: ```bash + # Basic capture arcade evals --capture + + # Capture with context + arcade evals --capture --include-context + + # Save to file + arcade evals --capture -o captures.json ``` -### Output Options - -| Flag | Description | -| ---------------------- | ------------------------------------------------- | -| `--details`, `-d` | Show detailed results | -| `--failed-only`, `-f` | Show only failed evals | -| `--file ` | Write results to file | -| `--format ` | Output format: `txt`, `md`, `html`, `json`, `all` | -| `--max-concurrent ` | Concurrent evaluations (default: 1) | -| `--add-context` | Include system/additional messages + + ### Provider Selection + ```bash + # Use specific provider + arcade evals -p openai + + # Use specific model + arcade evals -p openai:gpt-4o + + # Multiple providers + arcade evals -p openai:gpt-4o -p anthropic:claude-sonnet-4-5-20250929 + + # With API key + arcade evals -k openai:sk-... + ``` + + ### Output Formats + ```bash + # Save as JSON + arcade evals -o results.json + + # Multiple formats + arcade evals -o report.html -o report.md + + # All formats + arcade evals -o results + ``` + + ### Common Workflows + ```bash + # Debug failures + arcade evals --only-failed -d + + # Full report with context + arcade evals -d --include-context -o report.html + + # Parallel execution + arcade evals --max-concurrent 5 + + # Capture baseline + arcade evals --capture -o baseline.json + ``` + + ### Key Flags + + | Flag | Short | Description | + |------|-------|-------------| + | `--details` | `-d` | Show detailed results | + | `--only-failed` | `-f` | Filter failed tests | + | `--output` | `-o` | Output file(s) | + | `--use-provider` | `-p` | Select provider/model | + | `--api-key` | `-k` | Provider API key | + | `--capture` | | Capture mode | + | `--include-context` | | Add system messages | + | `--max-concurrent` | `-c` | Parallel runs | + + + Use `--capture` to generate expected tool calls, then run normal evals to validate. + diff --git a/public/llms.txt b/public/llms.txt index bec0f0ecd..dc6998727 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -1,4 +1,4 @@ - + # Arcade @@ -10,237 +10,239 @@ Arcade delivers three core capabilities: Deploy agents even your security team w ## Getting Started -- [page](https://docs.arcade.dev/en/home.md): The Arcade Docs page serves as a comprehensive resource for users, providing all the necessary information and guidance related to Arcade. It aims to help users understand and utilize Arcade effectively. +- [page](https://docs.arcade.dev/en/home.md): This documentation page provides guides, resources, sample agents, and references specifically for Arcade authentication providers and MCP servers, aimed at assisting agents and developers in their integration and development processes. ## API Reference -- [Airtable](https://docs.arcade.dev/en/references/auth-providers/airtable.md): This documentation page provides guidance on configuring the Airtable authentication provider for use with Arcade, enabling users to access Airtable APIs via OAuth 2.0. It outlines the steps for creating an Airtable app, obtaining necessary credentials, and integrating these into -- [Arcade API Reference](https://docs.arcade.dev/en/references/api.md): The Arcade API Reference documentation provides users with essential information on how to interact with the Arcade API, including the base URL for requests and links to the OpenAPI specification. It emphasizes the requirement of having an account in good standing and adherence to the Terms of -- [Arcade CLI Cheat Sheet](https://docs.arcade.dev/en/references/cli-cheat-sheet.md): The Arcade CLI Cheat Sheet serves as a quick reference guide for users to efficiently utilize all commands related to the Arcade CLI, including installation, authentication, project management, and server operations. It is designed for easy printing and provides essential commands and tips for various -- [Arcade MCP (MCP Server SDK) - Python Overview](https://docs.arcade.dev/en/references/mcp/python.md): This documentation page provides an overview of the Arcade MCP (MCP Server SDK) for Python, detailing its minimal API designed for programmatically building secure MCP servers. Users will learn how to utilize the `MCPApp` class for server configuration, tool -- [Asana](https://docs.arcade.dev/en/references/auth-providers/asana.md): This documentation page provides guidance on using and configuring the Asana authentication provider within the Arcade platform, enabling users to integrate Asana APIs into their applications or tools. It outlines steps for utilizing Arcade's default Asana auth provider as well as instructions for setting -- [Atlassian](https://docs.arcade.dev/en/references/auth-providers/atlassian.md): This documentation page provides guidance on configuring a custom Atlassian Auth Provider for use with Arcade, enabling users to authenticate and interact with the Atlassian API. It outlines the steps to create an Atlassian app, set up OAuth 2. -- [Auth Providers](https://docs.arcade.dev/en/references/auth-providers.md): This documentation page provides a comprehensive registry of authentication providers available within the Arcade ecosystem, enabling users to securely connect their data with Arcade tools. It outlines the benefits of using Arcade's built-in auth providers versus configuring custom ones, including branding and rate limit isolation -- [Calendly](https://docs.arcade.dev/en/references/auth-providers/calendly.md): This documentation page provides guidance on configuring and using the Calendly authentication provider with Arcade, enabling users to access Calendly APIs via OAuth 2.0. It outlines the steps for creating a Calendly developer account, registering an OAuth application, and integrating -- [Changelog](https://docs.arcade.dev/en/references/changelog.md): The Changelog documentation page provides users with a comprehensive overview of the latest updates, features, bug fixes, and maintenance changes for Arcade.dev. It highlights significant improvements across various components, including Arcade MCP Servers, CLI, and Toolkits, ensuring users stay -- [ClickUp](https://docs.arcade.dev/en/references/auth-providers/clickup.md): This documentation page provides guidance on configuring and using the ClickUp authentication provider within the Arcade platform, enabling applications and custom tools to access the ClickUp API on behalf of users. It outlines the steps for creating a ClickUp app, setting up OAuth credentials -- [Discord](https://docs.arcade.dev/en/references/auth-providers/discord.md): This documentation page provides guidance on how to configure and use a custom Discord authentication provider within the Arcade platform, as there is no default option available. It outlines the steps to create a Discord application, set up OAuth 2.0 credentials, and integrate -- [Dropbox](https://docs.arcade.dev/en/references/auth-providers/dropbox.md): This documentation page provides guidance on configuring a custom Dropbox Auth Provider for use with Arcade, enabling users to authenticate and interact with the Dropbox API through their applications and tools. It outlines the steps to create a Dropbox app, configure OAuth 2.0 credentials -- [Errors](https://docs.arcade.dev/en/references/mcp/python/errors.md): This documentation page provides an overview of domain-specific error types associated with the MCP server and its components, detailing the MCP exception hierarchy for improved error handling and debugging. Users can learn about various exceptions, such as `MCPError`, `ServerError`, -- [Figma](https://docs.arcade.dev/en/references/auth-providers/figma.md): This documentation page provides guidance on using the Figma authentication provider to enable tools and agents to access Figma APIs via OAuth 2.0. It outlines the necessary configurations, required scopes, and steps to create a Figma app for integration, ensuring -- [GitHub](https://docs.arcade.dev/en/references/auth-providers/github.md): This documentation page provides guidance on using the GitHub auth provider with Arcade, enabling users to configure and implement GitHub API calls securely through GitHub Apps. It emphasizes the advantages of GitHub Apps over OAuth Apps, including fine-grained permissions and centralized -- [Google](https://docs.arcade.dev/en/references/auth-providers/google.md): This documentation page provides guidance on using and configuring the Google auth provider within Arcade, enabling tools and applications to access Google/Google Workspace APIs on behalf of users. It outlines the options for utilizing Arcade's default Google OAuth provider for quick integration or creating a -- [Hubspot](https://docs.arcade.dev/en/references/auth-providers/hubspot.md): This documentation page provides guidance on using and configuring the Hubspot authentication provider within the Arcade platform, enabling users to call Hubspot APIs on behalf of their applications. It outlines the steps for leveraging Arcade's default Hubspot auth provider, creating a Hubspot -- [Linear](https://docs.arcade.dev/en/references/auth-providers/linear.md): This documentation page provides guidance on configuring and using the Linear authentication provider within the Arcade platform, enabling tools and applications to access Linear APIs on behalf of users. It outlines the steps for creating a Linear app, setting up OAuth credentials, and integrating Linear auth -- [LinkedIn](https://docs.arcade.dev/en/references/auth-providers/linkedin.md): This documentation page provides guidance on configuring and using the LinkedIn authentication provider with Arcade, enabling applications and custom tools to access LinkedIn APIs on behalf of users. It outlines the steps to create a LinkedIn app, configure app credentials, and implement Linked -- [Mailchimp](https://docs.arcade.dev/en/references/auth-providers/mailchimp.md): This documentation page provides guidance on configuring the Mailchimp authentication provider for use with Arcade, enabling users to call Mailchimp Marketing APIs via OAuth 2.0. It outlines the steps for creating a Mailchimp app, registering it, and integrating it with -- [Microsoft](https://docs.arcade.dev/en/references/auth-providers/microsoft.md): This documentation page provides guidance on configuring a custom Microsoft Auth Provider for use with Arcade, enabling users to authenticate and access Microsoft Graph APIs. It outlines the necessary steps to create a Microsoft app, set permissions, and integrate it with Arcade, ensuring that applications -- [Middleware](https://docs.arcade.dev/en/references/mcp/python/middleware.md): This documentation page provides an overview of the Middleware component in the Arcade MCP Server SDK for Python, detailing how users can intercept and modify requests and responses during processing. It outlines the base classes and methods for creating custom middleware, as well as built-in middleware -- [Miro](https://docs.arcade.dev/en/references/auth-providers/miro.md): This documentation page guides users on how to configure the Miro authentication provider for accessing Miro APIs using OAuth 2.0 within their applications or agents. It includes step-by-step instructions for creating a Miro app, obtaining necessary credentials, and integrating -- [Notion](https://docs.arcade.dev/en/references/auth-providers/notion.md): This documentation page provides guidance on configuring the Notion authentication provider for use with Arcade, enabling applications and custom tools to access Notion APIs on behalf of users. It outlines the steps for creating a Notion app, setting up OAuth credentials, and integrating -- [OAuth 2.0](https://docs.arcade.dev/en/references/auth-providers/oauth2.md): This documentation page provides guidance on configuring OAuth 2.0 for authorizing tools and agents with any OAuth 2.0-compatible API using the Arcade platform. It outlines the necessary steps for both the Arcade Cloud Engine and self-hosted environments, detailing -- [PagerDuty](https://docs.arcade.dev/en/references/auth-providers/pagerduty.md): This documentation page provides guidance on configuring the PagerDuty authentication provider for use with Arcade, enabling users to call PagerDuty APIs via OAuth 2.0. It details the steps for creating a PagerDuty app, setting up OAuth credentials, and integrating these -- [Reddit](https://docs.arcade.dev/en/references/auth-providers/reddit.md): This documentation page provides guidance on how to create and configure a custom Reddit Auth Provider using OAuth 2.0 credentials within the Arcade platform. It outlines the steps for setting up a Reddit application, integrating it into Arcade, and utilizing the Reddit API in -- [References](https://docs.arcade.dev/en/references.md): This documentation page provides comprehensive reference materials for Arcade's APIs, MCP servers, and authentication providers, enabling users to effectively integrate and manage these tools within their applications. It includes detailed sections on the Arcade REST API, MCP Server SDK, and various client libraries -- [Salesforce](https://docs.arcade.dev/en/references/auth-providers/salesforce.md): This documentation page provides guidance on configuring the Salesforce auth provider for use with Arcade, enabling users to call Salesforce APIs on behalf of a user. It outlines the steps for creating a Salesforce Connected App, setting necessary OAuth settings, and obtaining the Salesforce Org Sub -- [Server](https://docs.arcade.dev/en/references/mcp/python/server.md): This documentation page provides a reference for the `MCPServer` class in the Arcade MCP Python library, detailing its purpose as a low-level server for hosting Arcade tools over the MCP protocol. Users will learn about the server's features, including middleware support -- [Settings](https://docs.arcade.dev/en/references/mcp/python/settings.md): This documentation page provides an overview of global configuration and environment-driven settings for the Arcade MCP Server, detailing how to manage and utilize various settings containers like MCPSettings and its sub-settings. Users will learn how to create settings from environment variables, convert them to -- [Slack](https://docs.arcade.dev/en/references/auth-providers/slack.md): This documentation page provides guidance on configuring the Slack authentication provider for use with Arcade, enabling tools and agents to interact with Slack APIs on behalf of users. It includes instructions for creating a Slack app, setting up app credentials, and integrating Slack with the Arcade -- [Spotify](https://docs.arcade.dev/en/references/auth-providers/spotify.md): This documentation page provides guidance on configuring a custom Spotify Auth Provider within the Arcade platform, enabling users to authenticate and call the Spotify API on behalf of their users. It outlines the steps to create a Spotify app, set up OAuth 2.0 credentials -- [Square](https://docs.arcade.dev/en/references/auth-providers/square.md): This documentation page provides guidance on using and configuring the Square authentication provider with Arcade, enabling users to call Square APIs via OAuth 2.0 authentication. It includes steps for creating a Square app, configuring OAuth settings, and integrating the Square auth provider into -- [Telemetry](https://docs.arcade.dev/en/references/mcp/telemetry.md): This documentation page explains the telemetry data collected by the `arcade-mcp` framework, detailing its purpose, what data is tracked, and how users can opt-out of data sharing. It emphasizes that participation is optional and outlines the types of usage information -- [The Arcade CLI](https://docs.arcade.dev/en/references/arcade-cli.md): The Arcade CLI documentation provides users with instructions on how to install and utilize the Arcade command-line tool for managing Arcade deployments and MCP servers. It includes detailed command usage examples and guidance on various functionalities, such as logging in, creating projects, and deploying applications -- [TickTick](https://docs.arcade.dev/en/references/auth-providers/ticktick.md): This documentation page provides guidance on configuring the TickTick authentication provider to enable tools and applications to access TickTick APIs using OAuth 2.0. It outlines the steps for creating a TickTick app, setting up OAuth credentials, and integrating the authentication within -- [Transport Modes](https://docs.arcade.dev/en/references/mcp/python/transports.md): This documentation page provides an overview of the different transport modes (stdio and HTTP) available for MCP servers, detailing their characteristics, usage scenarios, and configuration options. Users will learn how to choose the appropriate transport based on their application needs, whether for desktop -- [Twitch](https://docs.arcade.dev/en/references/auth-providers/twitch.md): This documentation page provides guidance on how to configure and use a custom Twitch authentication provider within the Arcade platform, enabling applications and tools to access the Twitch API on behalf of users. It outlines the necessary steps for creating a Twitch app, setting up OAuth credentials -- [Types](https://docs.arcade.dev/en/references/mcp/python/types.md): This documentation page provides an overview of core Pydantic models and enums used in the MCP protocol, specifically detailing the `CallToolResult` and `SessionMessage` types. It helps users understand how to construct JSON-RPC requests and responses, as -- [X](https://docs.arcade.dev/en/references/auth-providers/x.md): This documentation page provides guidance on configuring and using the X auth provider to enable applications and agents to interact with the X (Twitter) API on behalf of users. It includes instructions for creating an X app, setting up OAuth credentials in the Arcade Dashboard, -- [Zendesk](https://docs.arcade.dev/en/references/auth-providers/zendesk.md): This documentation page provides guidance on configuring and using the Zendesk authentication provider with Arcade, enabling tools and agents to access Zendesk APIs on behalf of users. It outlines the steps to create a Zendesk app, set up OAuth clients, and configure necessary -- [Zoho](https://docs.arcade.dev/en/references/auth-providers/zoho.md): This documentation page provides guidance on using the Zoho auth provider to enable tools and agents to access Zoho APIs via OAuth 2.0 authentication. It outlines the steps for configuring Zoho app credentials, creating a Zoho app, and integrating it -- [Zoom](https://docs.arcade.dev/en/references/auth-providers/zoom.md): This documentation page provides guidance on configuring a custom Zoom authentication provider within the Arcade platform, as there is no default option available. Users will learn how to create a Zoom app, set up OAuth 2.0 credentials, and integrate Zoom API calls into +- [Airtable](https://docs.arcade.dev/en/references/auth-providers/airtable.md): This documentation page provides guidance on configuring the Airtable authentication provider using OAuth 2.0, enabling users to call Airtable APIs through their applications or tools. It outlines the steps for creating an Airtable app, obtaining necessary credentials, and integrating these +- [Arcade API Reference](https://docs.arcade.dev/en/references/api.md): The Arcade API Reference documentation provides users with essential information about the Arcade API, including the base URL for API requests and links to the Terms of Service and OpenAPI specification. It serves as a guide for developers to understand and utilize the API effectively. +- [Arcade CLI Cheat Sheet](https://docs.arcade.dev/en/references/cli-cheat-sheet.md): The Arcade CLI Cheat Sheet provides a quick reference for all commands related to the Arcade CLI, enabling users to efficiently install, authenticate, manage organizations and projects, create servers, and configure clients. It is designed for easy printing, making it a handy resource +- [Arcade MCP (MCP Server SDK) - Python Overview](https://docs.arcade.dev/en/references/mcp/python.md): This documentation page provides an overview of the Arcade MCP (MCP Server SDK) for Python, detailing its purpose as a framework for building secure MCP servers with a minimal API. Users will learn how to utilize the `MCPApp` class for server +- [Asana](https://docs.arcade.dev/en/references/auth-providers/asana.md): This documentation page provides guidance on using the Asana authentication provider within the Arcade platform, enabling users to call Asana APIs on behalf of their applications. It outlines how to quickly set up the default Asana auth provider or configure custom app credentials for production +- [Atlassian](https://docs.arcade.dev/en/references/auth-providers/atlassian.md): This documentation page provides guidance on configuring a custom Atlassian Auth Provider within the Arcade platform, enabling users to authenticate and interact with the Atlassian API. It outlines the steps for creating an Atlassian app, setting up OAuth 2. +- [Auth Providers](https://docs.arcade.dev/en/references/auth-providers.md): The "Auth Providers" documentation page serves as a comprehensive registry of authentication providers available within the Arcade ecosystem, enabling users to securely grant Arcade tools access to their data. It outlines the benefits of using Arcade's built-in auth providers versus configuring custom ones, +- [Calendly](https://docs.arcade.dev/en/references/auth-providers/calendly.md): This documentation page provides guidance on configuring the Calendly authentication provider for use with Arcade, enabling users to access Calendly APIs through OAuth 2.0. It outlines the steps for creating a Calendly developer account, registering an OAuth application, and integrating +- [Changelog](https://docs.arcade.dev/en/references/changelog.md): The Changelog documentation page provides users with a comprehensive overview of the latest updates, features, bug fixes, and maintenance changes for Arcade.dev. It details enhancements across various components, including Arcade MCP servers, the Arcade CLI, and toolkits, helping users +- [ClickUp](https://docs.arcade.dev/en/references/auth-providers/clickup.md): This documentation page provides guidance on configuring the ClickUp authentication provider within the Arcade platform, enabling users to call the ClickUp API on behalf of their users. It outlines the steps to create a ClickUp app, set up OAuth credentials, and integrate Click +- [Discord](https://docs.arcade.dev/en/references/auth-providers/discord.md): This documentation page provides guidance on configuring and using a custom Discord authentication provider within the Arcade platform, as there is no default option available. It outlines the steps for creating a Discord app, setting up OAuth 2.0 credentials, and integrating Discord auth +- [Dropbox](https://docs.arcade.dev/en/references/auth-providers/dropbox.md): This documentation page provides guidance on configuring a custom Dropbox Auth Provider in Arcade, enabling users to authenticate and interact with the Dropbox API on behalf of their applications. It outlines the steps to create a Dropbox app, set up OAuth 2.0 credentials, +- [Errors](https://docs.arcade.dev/en/references/mcp/python/errors.md): This documentation page provides an overview of domain-specific error types used in the MCP server and its components, detailing the exception hierarchy for improved error handling and debugging. Users will learn about various exceptions, including `MCPError`, `ServerError`, and ` +- [Figma](https://docs.arcade.dev/en/references/auth-providers/figma.md): This documentation page provides guidance on configuring the Figma authentication provider using OAuth 2.0, enabling users to access Figma APIs through their applications or tools. It outlines the necessary steps to create a Figma app, set up required OAuth scopes, +- [GitHub](https://docs.arcade.dev/en/references/auth-providers/github.md): This documentation page provides guidance on using the GitHub auth provider with Arcade, enabling users to configure and implement GitHub APIs securely through GitHub Apps rather than OAuth Apps. It highlights the advantages of GitHub Apps, such as fine-grained security and +- [Google](https://docs.arcade.dev/en/references/auth-providers/google.md): This documentation page provides guidance on using and configuring the Google auth provider within Arcade, enabling users to call Google/Google Workspace APIs on behalf of users. It outlines the options for utilizing Arcade's default Google OAuth provider for quick integration or creating a custom provider +- [Hubspot](https://docs.arcade.dev/en/references/auth-providers/hubspot.md): This documentation page provides guidance on using and configuring the Hubspot authentication provider within the Arcade platform, enabling users to call Hubspot APIs on behalf of their applications. It outlines the steps for utilizing Arcade's default Hubspot auth provider or configuring custom app credentials +- [Linear](https://docs.arcade.dev/en/references/auth-providers/linear.md): This documentation page provides guidance on configuring and using the Linear authentication provider with Arcade, enabling users to call Linear APIs on behalf of users. It outlines the steps for creating a Linear app, integrating it with Arcade, and using the Linear auth provider in custom +- [LinkedIn](https://docs.arcade.dev/en/references/auth-providers/linkedin.md): This documentation page provides guidance on configuring and using the LinkedIn authentication provider within Arcade, enabling applications and custom tools to access LinkedIn APIs on behalf of users. It outlines the steps for creating a LinkedIn app, setting up app credentials, and utilizing +- [Mailchimp](https://docs.arcade.dev/en/references/auth-providers/mailchimp.md): This documentation page provides guidance on configuring the Mailchimp authentication provider for integrating with Mailchimp's Marketing APIs using OAuth 2.0. It outlines the steps to create a Mailchimp app, register it for API access, and configure the Mailchimp auth +- [Microsoft](https://docs.arcade.dev/en/references/auth-providers/microsoft.md): This documentation page provides guidance on how to create and configure a custom Microsoft Auth Provider for use with Arcade, enabling applications and tools to authenticate users and access the Microsoft Graph API. It outlines the steps for registering a Microsoft app, setting permissions, and integrating +- [Middleware](https://docs.arcade.dev/en/references/mcp/python/middleware.md): This documentation page provides an overview of the Middleware component in the Arcade MCP Server SDK for Python, detailing how to intercept and modify MCP requests and responses through a structured middleware chain. It outlines the base classes, methods for handling various request types, and built +- [Miro](https://docs.arcade.dev/en/references/auth-providers/miro.md): This documentation page provides guidance on configuring the Miro authentication provider for tools and agents using OAuth 2.0, enabling them to access Miro APIs on behalf of users. It outlines the steps to create a Miro app, obtain necessary credentials, +- [Notion](https://docs.arcade.dev/en/references/auth-providers/notion.md): This documentation page provides guidance on configuring and using the Notion authentication provider with Arcade, enabling users to call Notion APIs on behalf of their applications or custom tools. It outlines the steps to create a Notion app, configure OAuth settings in the Arcade +- [OAuth 2.0](https://docs.arcade.dev/en/references/auth-providers/oauth2.md): This documentation page provides guidance on configuring OAuth 2.0 to authorize tools and agents with any OAuth 2.0-compatible provider using the Arcade platform. It details the setup process for both the Arcade Cloud Engine and self-hosted environments, including instructions +- [PagerDuty](https://docs.arcade.dev/en/references/auth-providers/pagerduty.md): This documentation page provides guidance on configuring the PagerDuty authentication provider for use with Arcade, enabling users to interact with PagerDuty APIs via OAuth 2.0. It outlines the steps to create a PagerDuty app, set up OAuth settings, and integrate +- [Reddit](https://docs.arcade.dev/en/references/auth-providers/reddit.md): This documentation page provides guidance on configuring a custom Reddit Auth Provider within the Arcade platform, enabling users to authenticate and interact with the Reddit API on behalf of their applications. It outlines the steps for creating a Reddit application, setting up OAuth credentials, and integrating +- [References](https://docs.arcade.dev/en/references.md): This documentation page provides comprehensive reference materials for Arcade's APIs, MCP servers, and authentication providers. Users can learn how to integrate Arcade's tools into their applications, utilize the MCP framework for building custom servers, and access client libraries in various programming languages. +- [Salesforce](https://docs.arcade.dev/en/references/auth-providers/salesforce.md): This documentation page provides guidance on configuring the Salesforce authentication provider for use with Arcade, enabling users to call Salesforce APIs through various tools and applications. It outlines the steps for creating a Connected App in Salesforce, including necessary OAuth settings and environment variable configurations. Users +- [Server](https://docs.arcade.dev/en/references/mcp/python/server.md): This documentation page provides a comprehensive reference for the `MCPServer` class in the Arcade MCP Python library, detailing its features, properties, and methods for hosting Arcade tools over MCP. Users will learn how to initialize the server, manage middleware, handle +- [Settings](https://docs.arcade.dev/en/references/mcp/python/settings.md): This documentation page provides guidance on configuring global and environment-driven settings for the Arcade MCP Server. It details the main settings container, sub-settings for various components (like server and middleware), and methods for creating settings from environment variables and converting them to a dictionary +- [Slack](https://docs.arcade.dev/en/references/auth-providers/slack.md): This documentation page provides guidance on configuring the Slack authentication provider for use with Arcade, enabling users to call Slack APIs on behalf of their applications or tools. It includes instructions for creating a Slack app, setting up OAuth credentials, and integrating with the Arcade platform +- [Spotify](https://docs.arcade.dev/en/references/auth-providers/spotify.md): This documentation page provides guidance on configuring a custom Spotify authentication provider within the Arcade platform, allowing users to access the Spotify API on behalf of their users. It outlines the steps to create a Spotify app, set up OAuth 2.0 credentials, and +- [Square](https://docs.arcade.dev/en/references/auth-providers/square.md): This documentation page provides guidance on configuring the Square auth provider to enable applications and agents to interact with Square APIs using OAuth 2.0 authentication. It outlines the steps for creating a Square app, obtaining necessary credentials, and integrating these into the Arcade platform +- [Telemetry](https://docs.arcade.dev/en/references/mcp/telemetry.md): This documentation page provides an overview of the telemetry data collected by the `arcade-mcp` framework, detailing what information is tracked, the purpose of data collection, and how users can opt-out if they choose. It emphasizes the importance of telemetry in +- [The Arcade CLI](https://docs.arcade.dev/en/references/arcade-cli.md): The Arcade CLI documentation provides users with instructions on how to install and utilize the Arcade command-line tool for managing Arcade deployments, generating and testing MCP servers, and performing various related tasks. It includes detailed installation commands and usage guidelines for different commands available within the +- [TickTick](https://docs.arcade.dev/en/references/auth-providers/ticktick.md): This documentation page provides guidance on configuring the TickTick authentication provider to enable tools and agents to access TickTick APIs using OAuth 2.0. It outlines the steps for creating a TickTick app, setting up OAuth settings, and integrating the authentication provider +- [Transport Modes](https://docs.arcade.dev/en/references/mcp/python/transports.md): This documentation page provides an overview of the different transport modes available for MCP servers, specifically focusing on stdio and HTTP transports. It helps users understand the characteristics, usage scenarios, and configuration options for each transport type, enabling them to choose the appropriate mode +- [Twitch](https://docs.arcade.dev/en/references/auth-providers/twitch.md): This documentation page provides guidance on configuring a custom Twitch authentication provider within the Arcade platform, allowing users to access the Twitch API on behalf of their users. It outlines the necessary steps to create a Twitch app, set up OAuth credentials, and integrate the authentication +- [Types](https://docs.arcade.dev/en/references/mcp/python/types.md): This documentation page provides an overview of core Pydantic models and enums used in the MCP protocol, specifically detailing the `CallToolResult` and `SessionMessage` types. It includes examples for constructing JSON-RPC requests and responses, as well as +- [X](https://docs.arcade.dev/en/references/auth-providers/x.md): This documentation page provides guidance on using and configuring the X auth provider to enable applications and agents to interact with the X (Twitter) API on behalf of users. It outlines the steps for creating an X app, configuring authentication within the Arcade platform, and +- [Zendesk](https://docs.arcade.dev/en/references/auth-providers/zendesk.md): This documentation page provides guidance on configuring Zendesk authentication within the Arcade platform, enabling users to call Zendesk APIs on behalf of a user. It outlines the steps to create a custom Zendesk auth provider, set up a Zendesk app, and manage +- [Zoho](https://docs.arcade.dev/en/references/auth-providers/zoho.md): This documentation page provides guidance on using the Zoho auth provider to enable applications and agents to access Zoho APIs through OAuth 2.0 authentication. It includes instructions for configuring Zoho app credentials, creating a Zoho app, and integrating with pre +- [Zoom](https://docs.arcade.dev/en/references/auth-providers/zoom.md): This documentation page guides users on how to create and configure a custom Zoom Auth Provider in Arcade, enabling applications and tools to call the Zoom API on behalf of users. It outlines the steps for registering a Zoom app, setting up OAuth credentials, and integrating ## Documentation -- [About Arcade](https://docs.arcade.dev/en/get-started/about-arcade.md): This documentation page explains how Arcade facilitates agent authorization for applications that require access to sensitive user data and services. It details the OAuth 2.0 authentication process that enables AI agents to securely act on behalf of users, allowing them to perform tasks such as -- [Add user authorization to your MCP tools](https://docs.arcade.dev/en/guides/create-tools/tool-basics/create-tool-auth.md): This documentation page guides users on how to implement user authorization in their custom MCP tools using Arcade, OAuth, and various authentication providers, such as Reddit. It covers the necessary prerequisites, explains the process of integrating OAuth for user consent, and provides example code -- [Adding Resource Server Auth to Your MCP Server](https://docs.arcade.dev/en/guides/security/secure-your-mcp-server.md): This documentation page guides users on how to secure their HTTP MCP server using OAuth 2.1 Resource Server authentication, enabling tool-level authorization and secrets management. It outlines the prerequisites, benefits, and configuration steps necessary for implementing this security feature, ensuring that -- [Agentic Development](https://docs.arcade.dev/en/get-started/setup/connect-arcade-docs.md): The "Agentic Development" documentation page guides users on how to enhance their development experience by integrating AI capabilities into their IDEs using Arcade.dev's resources. It explains how to utilize the LLMs.txt file format and the Context7 MCP server to -- [AirtableApi](https://docs.arcade.dev/en/resources/integrations/productivity/airtable-api.md): The AirtableApi documentation page provides users with a comprehensive set of tools for managing and interacting with Airtable through its API, enabling actions such as creating, updating, and deleting SCIM groups and users, managing webhooks, and handling base collaborations. -- [Arcade Cloud Infrastructure](https://docs.arcade.dev/en/guides/deployment-hosting/arcade-cloud.md): This documentation page provides an overview of the infrastructure supporting Arcade Cloud, detailing essential information such as egress IP addresses and the availability of VPC Peering for enterprise customers. Users can learn how to manage network traffic and inquire about advanced connectivity options. -- [Arcade for Slack](https://docs.arcade.dev/en/resources/integrations/social-communication/slack/install.md): The documentation page for Arcade for Slack provides users with guidance on integrating Arcade's AI tools into their Slack workspace, enhancing team efficiency through automated messaging, information retrieval, and content generation. It outlines installation steps, usage instructions, and the capabilities of the Arcade -- [Arcade for Zoom](https://docs.arcade.dev/en/resources/integrations/social-communication/zoom/install.md): The "Arcade for Zoom" documentation page guides users on how to integrate Arcade's AI tools with their Zoom accounts to enhance meeting management and information retrieval. It outlines the functionalities available, such as listing upcoming meetings and accessing invitation details, while also providing -- [Arcade Glossary](https://docs.arcade.dev/en/resources/glossary.md): The Arcade Glossary provides definitions and explanations of key terms and concepts related to the Arcade platform, including agents, tools, and MCP servers. This resource helps users understand the terminology necessary for building, testing, and deploying applications that utilize large language models ( -- [Arcade with Agent Frameworks and MCP Clients](https://docs.arcade.dev/en/guides/agent-frameworks.md): This documentation page provides guidance on integrating Arcade with various agent frameworks and MCP clients, enabling users to enhance their AI applications with advanced tool-calling capabilities. It features a list of supported frameworks, including LangChain, CrewAI, and OpenAI Agents, -- [Arcade with Google ADK](https://docs.arcade.dev/en/guides/agent-frameworks/google-adk/overview.md): This documentation page provides a comprehensive guide for integrating the Arcade platform with the Google ADK library, enabling users to enhance their AI agents with various tools such as Google Mail and GitHub. It covers installation, key features, basic usage examples, and user +- [About Arcade](https://docs.arcade.dev/en/get-started/about-arcade.md): This documentation page explains how Arcade facilitates agent authorization for applications that require access to sensitive user data and services. It details the OAuth 2.0 authorization system that enables AI agents to securely act on behalf of users, perform tasks like sending emails or creating +- [Add user authorization to your MCP tools](https://docs.arcade.dev/en/guides/create-tools/tool-basics/create-tool-auth.md): This documentation page guides users on how to implement user authorization in their custom MCP tools using Arcade, OAuth, and various authentication providers, such as Reddit. It covers the necessary prerequisites, the functioning of auth providers, and step-by-step instructions for creating an +- [Adding Resource Server Auth to Your MCP Server](https://docs.arcade.dev/en/guides/security/secure-your-mcp-server.md): This documentation page provides guidance on securing your HTTP MCP server using OAuth 2.1 Resource Server authentication, ensuring that only authorized users can access your tools. It outlines the necessary prerequisites, explains the importance of Resource Server auth, and details how to configure +- [Agentic Development](https://docs.arcade.dev/en/get-started/setup/connect-arcade-docs.md): This documentation page provides guidance on utilizing Agentic Development to enhance coding efficiency in AI-integrated IDEs by leveraging Arcade.dev's resources. It explains how to configure the LLMs.txt file for easy access to documentation and introduces Context7, a server +- [AirtableApi](https://docs.arcade.dev/en/resources/integrations/productivity/airtable-api.md): The AirtableApi documentation provides users with a comprehensive guide to tools that enable interaction with the Airtable API, focusing on managing SCIM groups, users, webhooks, and bases. Users can learn to perform various actions such as creating, updating, +- [Arcade Cloud Infrastructure](https://docs.arcade.dev/en/guides/deployment-hosting/arcade-cloud.md): This documentation page provides an overview of the infrastructure supporting Arcade Cloud, detailing essential information such as egress IP addresses and the availability of VPC Peering for enterprise customers. Users can learn how to manage traffic and explore options for enhanced connectivity. +- [Arcade for Slack](https://docs.arcade.dev/en/resources/integrations/social-communication/slack/install.md): The documentation page for Arcade for Slack provides users with guidance on integrating Arcade's AI tools into their Slack workspace to enhance team efficiency. It outlines the installation process, functionalities such as sending messages, finding information, and generating content, while also advising on the +- [Arcade for Zoom](https://docs.arcade.dev/en/resources/integrations/social-communication/zoom/install.md): The "Arcade for Zoom" documentation page provides users with guidance on integrating Arcade's AI tools with their Zoom accounts to enhance meeting management and information retrieval. It outlines the functionalities available, such as listing upcoming meetings and retrieving invitation details, while also addressing +- [Arcade Glossary](https://docs.arcade.dev/en/resources/glossary.md): The Arcade Glossary documentation provides definitions and explanations of key terms and concepts related to the Arcade platform, including agents, tools, and MCP servers. It aims to help users understand the components necessary for building, testing, and deploying applications that utilize large language +- [Arcade with Agent Frameworks and MCP Clients](https://docs.arcade.dev/en/guides/agent-frameworks.md): This documentation page provides guidance on integrating Arcade with various agent frameworks and MCP clients, enabling users to enhance their AI applications with tool-calling capabilities. It features a list of supported agent frameworks, including LangChain, CrewAI, and OpenAI Agents, +- [Arcade with Google ADK](https://docs.arcade.dev/en/guides/agent-frameworks/google-adk/overview.md): This documentation page provides a comprehensive guide on integrating the `google-adk-arcade` package with Arcade, enabling users to enhance their AI agents with various tools such as Google Mail and GitHub. It covers installation, key features, basic usage examples - [Arcade with OpenAI Agents](https://docs.arcade.dev/en/guides/agent-frameworks/openai-agents/overview.md): This documentation page provides a comprehensive guide for integrating Arcade with the OpenAI Agents library, enabling users to enhance their AI agents with various tools such as Gmail, LinkedIn, and GitHub. It covers installation, key features, basic usage examples, and -- [ArcadeEngineApi](https://docs.arcade.dev/en/resources/integrations/development/arcade-engine-api.md): The ArcadeEngineApi documentation provides users with a comprehensive guide to the tools available for managing authentication, secrets, and worker configurations within the Arcade Engine. It outlines various API functionalities, such as retrieving authentication provider details, managing user connections, and executing tools, -- [Asana](https://docs.arcade.dev/en/resources/integrations/productivity/asana.md): This documentation page provides a comprehensive guide for using the Arcade Asana MCP Server, which enables users to create agents and AI applications that interact with Asana tasks, projects, and workspaces. It details various tools available for managing teams, creating and updating -- [Asana Reference](https://docs.arcade.dev/en/resources/integrations/productivity/asana/reference.md): The Asana Reference documentation provides a comprehensive list of enumerations related to tag colors, task sorting criteria, and sort order options used in the Asana MCP Server. Users can refer to this page to understand the specific values and formats applicable for customizing and -- [AsanaApi](https://docs.arcade.dev/en/resources/integrations/productivity/asana-api.md): The AsanaApi documentation provides users with tools and guidance for interacting with the Asana API, enabling them to manage access requests, allocations, custom fields, and goals effectively. Users can learn how to perform various actions such as fetching, creating, updating -- [AshbyApi](https://docs.arcade.dev/en/resources/integrations/productivity/ashby-api.md): The AshbyApi documentation provides users with tools and functionalities to effectively manage recruitment processes within the Ashby platform. It enables users to create and update job applications, retrieve candidate information, manage interview schedules, and handle job postings, all aimed at streamlining -- [Authorized Tool Calling](https://docs.arcade.dev/en/guides/tool-calling/custom-apps/auth-tool-calling.md): The "Authorized Tool Calling" documentation provides a comprehensive guide for developers on how to securely enable AI agents to access external services using authorization methods like OAuth 2.0 and API keys. It outlines the steps for initializing the Arcade client, authorizing tools -- [BoxApi](https://docs.arcade.dev/en/resources/integrations/productivity/box-api.md): The BoxApi documentation provides users with tools and guidance for managing Box content, metadata, and collaboration workflows through direct interaction with the Box API. It covers a wide range of functionalities, including file management, document generation, and enterprise operations, enabling users to -- [Brightdata](https://docs.arcade.dev/en/resources/integrations/development/brightdata.md): The Brightdata documentation page provides users with tools and instructions for scraping and extracting web content and structured data at scale, enabling them to gather information from various websites and search engines without being blocked. Users can learn to utilize specific tools like ScrapeAsMarkdown -- [Build MCP Server QuickStart](https://docs.arcade.dev/en/get-started/quickstarts/mcp-server-quickstart.md): The "Build MCP Server QuickStart" documentation guides users through the process of creating and running a custom MCP Server using the Arcade MCP framework. It covers prerequisites, installation steps, server setup, and how to manage secrets and authorization for tools, enabling users -- [CalendlyApi](https://docs.arcade.dev/en/resources/integrations/productivity/calendly-api.md): The CalendlyApi documentation provides a comprehensive guide for developers to utilize the Calendly API for managing scheduling and event-related tasks. Users can learn how to create, retrieve, update, and manage events, invitees, and organization memberships, as well as -- [Call a tool in your IDE/MCP Client](https://docs.arcade.dev/en/get-started/quickstarts/call-tool-client.md): This documentation page guides users on how to create and utilize an MCP Gateway within their IDE or MCP Client to streamline the process of calling tools from multiple MCP servers. It outlines the steps for setting up the gateway, selecting relevant tools, and integrating them into -- [Call tools from MCP clients](https://docs.arcade.dev/en/guides/create-tools/tool-basics/call-tools-mcp.md): This documentation page provides guidance on configuring MCP clients to call tools from an MCP server, detailing necessary prerequisites and outcomes. Users will learn how to set up their clients using the `arcade configure` command, including customization options for different transport types and handling -- [Calling tools in your agent with Arcade](https://docs.arcade.dev/en/get-started/quickstarts/call-tool-agent.md): This documentation page provides a comprehensive guide on how to utilize Arcade to enable AI agents to call various hosted tools, such as sending emails or updating documents. Users will learn to install the Arcade client, set up workflows that interact with these tools, and handle -- [Checking Tool Authorization Status](https://docs.arcade.dev/en/guides/tool-calling/custom-apps/check-auth-status.md): This documentation page provides a comprehensive guide on how to check the authorization status of tools before execution, helping users understand necessary permissions and tool availability. It includes instructions for initializing the client in Python or JavaScript, checking the authorization status for all tools or specific -- [Clickhouse](https://docs.arcade.dev/en/resources/integrations/databases/clickhouse.md): This documentation page provides an overview of the Arcade Clickhouse MCP Server, a tool designed for agents to interact with Clickhouse databases in a read-only capacity. Users can learn how to discover database schemas, explore table structures, and execute SELECT queries safely, -- [Clickhouse](https://docs.arcade.dev/en/resources/integrations/databases/postgres/clickhouse.md): This documentation page provides an overview of the Arcade Clickhouse MCP Server, which enables agents to interact with Clickhouse databases in a read-only capacity. Users can learn to discover database schemas, explore table structures, and execute SELECT queries safely, while also understanding -- [Clickup](https://docs.arcade.dev/en/resources/integrations/productivity/clickup.md): This documentation page provides an overview of the ClickUp MCP Server, which enables users to build agents and applications that interact with ClickUp workspaces, tasks, and members. It outlines various tools available for managing tasks, comments, and workspace structures, as -- [Clickup Reference](https://docs.arcade.dev/en/resources/integrations/productivity/clickup/reference.md): The Clickup Reference documentation provides users with a comprehensive overview of enumerations used in various tools within the Clickup MCP Server, including task priorities, filter scopes, task ordering, and comment resolutions. It helps users understand the default values and options available for -- [ClickupApi](https://docs.arcade.dev/en/resources/integrations/productivity/clickup-api.md): The ClickupApi documentation provides users with a comprehensive set of tools to interact with the ClickUp API, facilitating efficient task and project management. Users can learn how to authenticate, manage checklists, comments, custom fields, and dependencies, as well as -- [Close.io](https://docs.arcade.dev/en/resources/integrations/productivity/closeio.md): This documentation page for Close.io provides an overview of the CRM's capabilities for managing leads, contacts, and deals. It aims to assist users in understanding how to effectively utilize Close.io for their customer relationship management needs. Currently, detailed content is marked as -- [Compare MCP Server Types](https://docs.arcade.dev/en/guides/create-tools/tool-basics/compare-server-types.md): This documentation page provides a comparison of different MCP server types offered by Arcade, detailing their functionalities based on transport methods and deployment options. Users can learn about the capabilities of each server type, including support for various tools and authentication requirements. This comparison helps users -- [Confluence](https://docs.arcade.dev/en/resources/integrations/productivity/confluence.md): This documentation page provides an overview of the Arcade Confluence MCP Server, which offers a suite of tools for building agents and AI applications that interact with Confluence. Users can learn how to create, update, and manage Confluence pages, spaces, and -- [Contact Us](https://docs.arcade.dev/en/resources/contact-us.md): This documentation page provides users with information on how to connect with the Arcade team for support through various channels. It emphasizes the team's commitment to assisting users and their agents in achieving success. -- [Create a new Mastra project](https://docs.arcade.dev/en/guides/agent-frameworks/mastra/use-arcade-tools.md): This documentation page provides a step-by-step guide for integrating Arcade tools into a new Mastra project, enabling users to leverage these tools within their Mastra applications. It covers prerequisites, project creation, client installation, API key configuration, and agent interaction methods -- [Create an MCP tool with secrets](https://docs.arcade.dev/en/guides/create-tools/tool-basics/create-tool-secrets.md): This documentation page guides users on how to create custom MCP tools that securely handle sensitive information, or secrets, using the Arcade platform. It covers the process of reading secrets from various sources, such as environment files and the Arcade Dashboard, and emphasizes best practices -- [Creating an MCP Server with Arcade](https://docs.arcade.dev/en/guides/create-tools/tool-basics/build-mcp-server.md): This documentation page provides a comprehensive guide for users to create, test, deploy, and publish a custom MCP Server using the Arcade framework. It details the installation of the necessary tools, the scaffolding of a server project, and the configuration required to run -- [CursorAgentsApi](https://docs.arcade.dev/en/resources/integrations/development/cursor-agents-api.md): The CursorAgentsApi documentation provides users with tools to manage and inspect background agents, including functionalities for listing, retrieving status, deleting agents, and accessing user authentication information. It also offers guidance on fetching recommended models and linked GitHub repositories, enabling seamless interaction -- [CustomerioApi](https://docs.arcade.dev/en/resources/integrations/customer-support/customerio-api.md): The CustomerioApi documentation provides users with a suite of tools designed for managing customer communications and marketing campaigns via the Customer.io platform. It details various functionalities, including sending transactional messages, retrieving campaign metrics, and managing broadcast actions, all accessible through an API -- [CustomerioPipelinesApi](https://docs.arcade.dev/en/resources/integrations/customer-support/customerio-pipelines-api.md): The CustomerioPipelinesApi documentation provides tools for integrating and managing user data within the Customer.io platform, enabling users to identify individuals, track events, and analyze interactions. It outlines various functionalities such as assigning traits, sending event data, and batch processing -- [CustomerioTrackApi](https://docs.arcade.dev/en/resources/integrations/customer-support/customerio-track-api.md): The CustomerioTrackApi documentation provides users with a set of tools to manage customer data and interactions within the Customer.io platform. It includes functionalities for adding, updating, and deleting customer profiles, tracking events, and managing segments, all accessible via API calls -- [DatadogApi](https://docs.arcade.dev/en/resources/integrations/development/datadog-api.md): The DatadogApi documentation provides users with tools and instructions for interacting with the Datadog API, enabling efficient management of Datadog services. It outlines the necessary authentication steps, including obtaining API and application keys, and details various tools available for -- [Deploying to the cloud with Arcade Deploy](https://docs.arcade.dev/en/guides/deployment-hosting/arcade-deploy.md): This documentation page provides a comprehensive guide on deploying an MCP server to the cloud using Arcade Deploy, enabling users to access their servers remotely and support multi-user functionality. It outlines the necessary prerequisites, step-by-step deployment instructions, and how to create MCP Gate -- [Directly call third-party APIs](https://docs.arcade.dev/en/guides/tool-calling/call-third-party-apis.md): This documentation page provides a comprehensive guide on how to directly call third-party APIs by obtaining user authorization tokens through Arcade. Users will learn to manage authentication flows, use tokens with external services, and implement the process using examples in Python and JavaScript, specifically -- [Discord](https://docs.arcade.dev/en/resources/integrations/social-communication/discord.md): This documentation page provides guidance on configuring and using the Discord authentication provider with Arcade, allowing users to call the Discord API on behalf of a user. It outlines the steps for creating a Discord application, setting up OAuth credentials, and integrating Discord auth into app -- [Dropbox](https://docs.arcade.dev/en/resources/integrations/productivity/dropbox.md): This documentation page provides users with tools to interact with Dropbox, enabling them to browse, search, and download files and folders through the Arcade Dropbox MCP Server. It includes detailed descriptions of available tools, their parameters, and example code snippets for implementation in Python -- [E2B](https://docs.arcade.dev/en/resources/integrations/development/e2b.md): The E2B documentation page provides users with tools to run code in a sandboxed environment and create static matplotlib charts, facilitating the development of agents and AI applications. It outlines available tools, their functionalities, and authentication requirements, along with code examples for -- [Engine Configuration](https://docs.arcade.dev/en/guides/deployment-hosting/configure-engine.md): This documentation page provides advanced configuration options for enterprise customers self-hosting the Arcade Engine, detailing how to install the engine on various platforms and customize its settings through configuration files. Users will learn about the purpose and management of the `engine.yaml` and ` -- [Environment Variables](https://docs.arcade.dev/en/resources/integrations/social-communication/slack/environment-variables.md): This documentation page provides information on configuring environment variables that control the behavior of tools interacting with the Slack API, specifically focusing on managing concurrent requests, pagination size limits, and pagination timeout settings. Users can learn how to adjust these variables to optimize their tool's -- [Evaluate tools](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/create-evaluation-suite.md): This documentation page provides a comprehensive guide on creating an evaluation suite using Arcade to assess the performance of AI tools. Users will learn how to define evaluation cases, utilize various critic classes to measure outcomes, and run evaluations effectively to ensure their tools are functioning correctly -- [ExaApi](https://docs.arcade.dev/en/resources/integrations/search/exa-api.md): The ExaApi documentation provides users with a comprehensive guide to utilizing the Exa.ai Search API, enabling them to conduct searches, manage websets, and handle research requests effectively. It outlines various tools available for tasks such as retrieving search results, creating -- [Fetch the "ScrapeUrl" tool from the "Firecrawl" MCP Server](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/use-arcade-tools.md): This documentation page provides a comprehensive guide on integrating Arcade tools into LangGraph applications, detailing the necessary prerequisites, setup, and configuration steps. Users will learn how to manage Arcade tools, create AI models, and implement a ReAct-style agent to enhance their -- [Figma](https://docs.arcade.dev/en/resources/integrations/development/figma.md): This documentation page provides a comprehensive guide for using the Figma MCP Server, which allows agents to interact with Figma's design files, components, and collaboration features through the Figma REST API. Users can learn how to access file structures, manage components -- [FigmaApi](https://docs.arcade.dev/en/resources/integrations/productivity/figma-api.md): The FigmaApi documentation provides users with a comprehensive set of tools to interact with the Figma API, enabling efficient management of design assets and collaboration on projects. Users can learn to perform various actions, such as retrieving files, managing comments, and handling -- [Firecrawl](https://docs.arcade.dev/en/resources/integrations/development/firecrawl.md): The Firecrawl documentation provides users with tools and guidance to enable agents to scrape, crawl, and map websites effectively. It outlines various functionalities, including scraping URLs, crawling websites, retrieving crawl statuses, and canceling ongoing crawls, all accessible via an -- [Frequently Asked Questions](https://docs.arcade.dev/en/resources/faq.md): This documentation page provides answers to common questions about the Arcade platform, including how to create and contribute tools, the differences between various API keys, and authentication methods. It helps users understand the functionalities of Arcade, enabling them to effectively build, manage, and -- [FreshserviceApi](https://docs.arcade.dev/en/resources/integrations/customer-support/freshservice-api.md): The FreshserviceApi documentation provides tools and endpoints for programmatically interacting with the Freshservice platform, enabling users to manage organizational data, assets, software, service catalogs, users, tickets, and more. It facilitates automation of ITSM workflows and administration tasks -- [Get Formatted Tool Definitions](https://docs.arcade.dev/en/guides/tool-calling/custom-apps/get-tool-definitions.md): This documentation page provides guidance on retrieving formatted tool definitions using the Arcade Client, allowing users to obtain both single and multiple tool definitions in specific model provider formats. It also explains how to convert these definitions into Zod schemas for enhanced type safety and integration with -- [Getting Your API Key](https://docs.arcade.dev/en/get-started/setup/api-keys.md): This documentation page guides users through the process of obtaining and managing their Arcade API key, highlighting both dashboard and CLI methods. It provides step-by-step instructions for generating the key, emphasizes the importance of securely storing it, and warns against sharing it publicly. -- [GitHub](https://docs.arcade.dev/en/resources/integrations/development/github.md): This documentation page provides users with guidance on utilizing the Arcade GitHub MCP Server, which enables agents to interact with GitHub repositories, issues, and pull requests. It outlines the server's capabilities, configuration requirements, and necessary permissions for creating GitHub Apps -- [GithubApi](https://docs.arcade.dev/en/resources/integrations/development/github-api.md): The GitHubApi documentation provides a set of tools for users and applications to interact with the GitHub API, enabling management of repositories, issues, pull requests, and more. It outlines various functionalities, such as creating and managing webhooks, organizations, -- [Gmail](https://docs.arcade.dev/en/resources/integrations/productivity/gmail.md): This documentation page provides a comprehensive overview of the Arcade Gmail MCP Server, which enables users to build agents and AI applications that can send, read, and manage emails through the Gmail API. It details various tools available for email management, including sending and deleting -- [Gmail Reference](https://docs.arcade.dev/en/resources/integrations/productivity/gmail/reference.md): The Gmail Reference documentation provides a list of enumerations used in the Gmail MCP Server, specifically detailing options for reply recipients and date ranges. Users can learn how to utilize these enumerations effectively in their applications. This reference serves as a quick guide for developers -- [Google Calendar](https://docs.arcade.dev/en/resources/integrations/productivity/google-calendar.md): This documentation page provides a comprehensive overview of the Arcade Google Calendar MCP Server, enabling users to build agents and applications that interact with Google Calendar. It outlines available tools for managing calendars and events, including listing calendars, creating and updating events, and finding available -- [Google Contacts](https://docs.arcade.dev/en/resources/integrations/productivity/google-contacts.md): This documentation page provides users with tools and guidance for integrating Google Contacts into their applications using the Arcade Google Contacts MCP Server. It outlines functionalities such as creating new contacts and searching for existing ones by email, name, or phone number, while also detailing the -- [Google Docs](https://docs.arcade.dev/en/resources/integrations/productivity/google-docs.md): This documentation page provides a comprehensive guide for developers to utilize the Arcade Google Docs MCP Server, enabling them to create and manage Google Docs documents through various tools. Users can learn how to perform actions such as creating, editing, and retrieving documents, as well -- [Google Drive](https://docs.arcade.dev/en/resources/integrations/productivity/google-drive.md): This documentation page provides users with tools and guidance for interacting with Google Drive through the MCP Server, enabling them to manage files effectively. Users can learn how to retrieve file structures, search for files, create and organize folders, upload and download files, and -- [Google Drive Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-drive/reference.md): The Google Drive Reference documentation provides users with a comprehensive guide to the enumerations used in the Google Drive MCP Server, including options for sorting files, filtering by file type, defining permission roles, and supported file upload types. This resource helps users effectively manage -- [Google Finance](https://docs.arcade.dev/en/resources/integrations/search/google_finance.md): This documentation page provides users with the tools and instructions to access real-time and historical stock data through the Arcade Google Finance API. Users can retrieve current stock summaries and historical price and volume data, enabling the development of intelligent agents and applications. It also includes -- [Google Flights](https://docs.arcade.dev/en/resources/integrations/search/google_flights.md): This documentation page provides users with the tools and information needed to integrate Google Flights search functionality into their applications using the Arcade platform. It specifically details how to retrieve one-way flight search results, including required parameters and optional settings for customizing the search. Additionally, -- [Google Hotels](https://docs.arcade.dev/en/resources/integrations/search/google_hotels.md): The Google Hotels documentation page provides users with the necessary tools and instructions to empower agents and applications to search for hotels globally using the Google Hotels API. It details how to utilize the `GoogleHotels.SearchHotels` function, including required parameters and optional filters for -- [Google Jobs](https://docs.arcade.dev/en/resources/integrations/search/google_jobs.md): This documentation page provides users with tools to integrate and search for job openings using Google Jobs through the Arcade Google Jobs MCP Server. It outlines how to use the API, including parameters for job searches and authentication requirements, enabling developers to build agents and AI applications -- [Google Maps](https://docs.arcade.dev/en/resources/integrations/search/google_maps.md): This documentation page provides tools for integrating Google Maps functionality into applications, specifically enabling users to obtain directions between two locations using either addresses or latitude/longitude coordinates. It outlines available tools, their parameters, and offers code examples in Python and JavaScript for implementation -- [Google News](https://docs.arcade.dev/en/resources/integrations/search/google_news.md): This documentation page provides users with tools to search for news stories using Google News through the Arcade Google News MCP Server. It outlines how to utilize the `GoogleNews.SearchNewsStories` tool, including required parameters and optional configurations for language and country. Additionally -- [Google Search](https://docs.arcade.dev/en/resources/integrations/search/google_search.md): This documentation page provides users with the tools and instructions needed to perform Google searches using the Arcade Search MCP Server and SerpAPI. It outlines how to build agents and AI applications that can retrieve organic search results, including necessary parameters and authentication details. Users -- [Google Sheets](https://docs.arcade.dev/en/resources/integrations/productivity/google-sheets.md): This documentation page provides a comprehensive overview of the Arcade Google Sheets MCP Server, enabling users to build agents and AI applications that interact with Google Sheets. It outlines various tools available for creating, reading, and updating spreadsheets, along with the necessary OAuth scopes required -- [Google Shopping Search](https://docs.arcade.dev/en/resources/integrations/search/google_shopping.md): The Google Shopping Search documentation provides users with tools and guidance for integrating product search capabilities from Google Shopping into their agents and AI applications. It outlines how to utilize the GoogleShopping.SearchProducts tool, including required parameters and authentication details, while also offering options for -- [Google Slides](https://docs.arcade.dev/en/resources/integrations/productivity/google-slides.md): This documentation page provides tools for interacting with Google Slides, enabling users to create presentations, add slides, comment on specific slides, and retrieve presentation content in markdown format. It outlines various functionalities, such as searching for presentations and managing comments, while also detailing -- [GoogleCalendar Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-calendar/reference.md): The GoogleCalendar Reference documentation provides a comprehensive list of enumerations related to event visibility, update options for Google Meet, and sending updates within the GoogleCalendar MCP Server. Users can reference these enumerations to understand and implement various functionalities in their applications effectively. -- [GoogleDocs Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-docs/reference.md): The GoogleDocs Reference documentation provides a comprehensive list of enumerations related to ordering and document formats used in the GoogleDocs MCP Server. Users can learn how to effectively sort documents by various criteria, such as creation and modification times, as well as understand the -- [GoogleSheets Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-sheets/reference.md): The GoogleSheets Reference documentation provides a comprehensive list of enumerations related to ordering data within the GoogleSheets MCP Server. Users can learn about various sorting options, such as by creation time, modification time, and name, along with their corresponding identifiers. This -- [Hosting Optoions Overview](https://docs.arcade.dev/en/guides/deployment-hosting.md): This documentation page provides an overview of the various hosting options available for the Arcade platform, including the default Arcade Cloud service and on-premise deployment alternatives. Users can learn how to quickly set up their environment, manage resources, and customize authentication while ensuring data -- [How to contribute a MCP Server](https://docs.arcade.dev/en/resources/integrations/contribute-a-server.md): This documentation page provides a step-by-step guide for developers on how to contribute their MCP Server to the Arcade documentation, enhancing visibility and accessibility for other users. It outlines prerequisites for submission, including building and publishing the MCP Server, and details the process for -- [Hubspot](https://docs.arcade.dev/en/resources/integrations/sales/hubspot.md): This documentation page provides users with tools and guidance for integrating and automating interactions with HubSpot CRM through the HubSpot MCP Server. Users can learn how to perform various tasks, such as creating and updating activities, managing contacts and deals, and retrieving user -- [Hubspot Reference](https://docs.arcade.dev/en/resources/integrations/sales/hubspot/reference.md): The Hubspot Reference documentation provides a comprehensive list of enumerations related to various tools within the Hubspot MCP Server, including call directions, email statuses, meeting outcomes, communication channels, activity types, sort orders, deal types, and deal priorities. This -- [HubspotAutomationApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-automation-api.md): The HubspotAutomationApi documentation provides users with a set of tools designed to manage and automate workflows within HubSpot, enabling tasks such as completing blocked actions, retrieving email campaign details, and enrolling contacts in sequences. It offers clear instructions and code examples for -- [HubspotCmsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-cms-api.md): The HubspotCmsApi documentation provides users with tools and guidance for managing content within the HubSpot CMS, including creating, updating, and deleting blog posts, landing pages, and managing authors and multi-language support. It outlines various API functionalities that streamline content -- [HubspotConversationsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-conversations-api.md): The HubspotConversationsApi documentation provides users with tools to effectively manage and interact with conversation threads and channels within HubSpot. It outlines various actions such as retrieving conversation inboxes, sending and updating messages, and accessing detailed thread information, all aimed at -- [HubspotCrmApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-crm-api.md): The HubspotCrmApi documentation provides users with a comprehensive set of tools for managing and interacting with HubSpot CRM, enabling actions such as retrieving, creating, updating, and deleting records for various CRM objects. It also facilitates the management of associations, -- [HubspotEventsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-events-api.md): The HubspotEventsApi documentation provides users with tools and guidance for managing and analyzing event data within the HubSpot ecosystem. It enables users to retrieve event completion data, manage custom event definitions, and send event data efficiently, enhancing their event tracking capabilities. -- [HubspotMarketingApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-marketing-api.md): The HubspotMarketingApi documentation provides users with tools to effectively manage and analyze marketing campaigns within the HubSpot platform. It enables actions such as creating, updating, and deleting campaigns, managing budgets, and retrieving performance metrics. This resource is essential for stream -- [HubspotMeetingsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-meetings-api.md): The HubspotMeetingsApi documentation provides users with tools to efficiently manage meetings through Hubspot's scheduling system, enabling functionalities such as scheduling, booking, and retrieving meeting details. It outlines various capabilities, including integration with Hubspot's calendar, listing scheduling -- [HubspotUsersApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-users-api.md): The HubspotUsersApi documentation provides users with tools to efficiently manage users and teams within a HubSpot account, including functionalities to retrieve user lists, create and update user accounts, and remove users. It outlines available API tools, their descriptions, and example -- [Imgflip](https://docs.arcade.dev/en/resources/integrations/entertainment/imgflip.md): The Imgflip documentation page provides users with tools to create and manage memes using the Imgflip API, enabling the development of agents and AI applications. Users can search for meme templates, retrieve popular templates, and create custom memes by adding text to existing templates -- [Imgflip](https://docs.arcade.dev/en/resources/integrations/entertainment/spotify/imgflip.md): The Imgflip documentation page provides users with tools to create and manage memes using the Imgflip API, enabling the development of agents and AI applications. Users can search for meme templates, retrieve popular templates, and create custom memes by adding text to existing templates -- [Import necessary classes and modules](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/user-auth-interrupts.md): This documentation page guides users in creating a LangGraph workflow that incorporates user authorization for specific Arcade tools, ensuring that only authorized tools are accessible to the language model. It provides step-by-step instructions on setting up the environment, configuring API keys, defining workflow -- [Initialize the Arcade client](https://docs.arcade.dev/en/guides/agent-frameworks/google-adk/use-arcade-tools.md): This documentation page provides a comprehensive guide for integrating Arcade tools into Google ADK applications, detailing the necessary prerequisites, environment setup, and configuration steps. Users will learn how to create and manage Arcade tools, authorize them for agents, and run these agents within -- [IntercomApi](https://docs.arcade.dev/en/resources/integrations/customer-support/intercom-api.md): The IntercomApi documentation provides a comprehensive guide for developers to utilize tools that enable interaction with the Intercom platform via its API. Users can learn to manage admin details, articles, collections, help centers, and company information effectively through various API endpoints. -- [Jira](https://docs.arcade.dev/en/resources/integrations/productivity/jira.md): This documentation page provides a comprehensive guide for using the Jira MCP Server, enabling users and AI applications to efficiently manage Jira issues and projects. It outlines the functionalities available, such as creating, updating, and searching for issues, managing labels and attachments, and -- [Jira Environment Variables](https://docs.arcade.dev/en/resources/integrations/productivity/jira/environment-variables.md): This documentation page provides guidance on configuring key environment variables for optimizing Jira API interactions within the Arcade platform. Users can learn how to set limits on concurrent requests, manage API request timeouts, and control caching behavior to enhance performance during tool execution. Each variable -- [Jira Reference](https://docs.arcade.dev/en/resources/integrations/productivity/jira/reference.md): The Jira Reference documentation provides a comprehensive list of enumerations for various tools within the Jira MCP Server, including sprint states, priority scheme ordering, and issue comment ordering. Users can utilize this reference to understand and implement the appropriate enumeration values when working with the -- [Linear](https://docs.arcade.dev/en/resources/integrations/productivity/linear.md): This documentation page provides users with a comprehensive guide to the Linear MCP Server, enabling them to interact with Linear's issue tracking, project management, and team collaboration features. Users can learn how to create, update, and manage issues, projects, initiatives, -- [LinkedIn](https://docs.arcade.dev/en/resources/integrations/social-communication/linkedin.md): This documentation page provides an overview of the Arcade LinkedIn MCP Server, which enables users to easily build agents and AI applications that interact with LinkedIn, specifically allowing the creation of text posts. Users can find available tools, example code for implementation, and -- [LumaApi](https://docs.arcade.dev/en/resources/integrations/productivity/luma-api.md): The LumaApi documentation provides users with a comprehensive guide to utilizing the Luma API for efficient event and calendar management within the Luma platform. It outlines various tools and functionalities, such as creating and managing events, retrieving guest information, and handling invitations -- [MailchimpMarketingApi](https://docs.arcade.dev/en/resources/integrations/productivity/mailchimp-marketing-api.md): The Mailchimp Marketing API documentation provides users with tools to effectively manage and optimize their email marketing campaigns through direct interaction with the Mailchimp Marketing API. It outlines various functionalities, such as retrieving audience information, managing automation workflows, and creating account exports, enabling -- [MCP Gateways](https://docs.arcade.dev/en/guides/create-tools/mcp-gateways.md): This documentation page provides a comprehensive guide on configuring and using MCP Gateways, which enable users to connect multiple MCP Servers for streamlined management and access to tools within a single project. It outlines the steps for creating an MCP Gateway, including selecting tools and setting -- [Microsoft Teams](https://docs.arcade.dev/en/resources/integrations/social-communication/microsoft-teams.md): This documentation page provides users with tools and guidance for interacting with Microsoft Teams through the MCP Server, enabling efficient management of teams, channels, and chats. Users can retrieve information, send messages, and manage users within Microsoft Teams, enhancing collaboration and communication. -- [MicrosoftTeams Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/microsoft-teams/reference.md): The MicrosoftTeams Reference documentation provides a comprehensive overview of key enumerations related to the MicrosoftTeams MCP Server, including types of matches and team membership classifications. Users can learn about specific match types such as exact and partial matches, as well as different membership roles -- [Migrate from toolkits to MCP servers](https://docs.arcade.dev/en/guides/create-tools/migrate-toolkits.md): This documentation page provides a comprehensive guide for users looking to migrate their existing Arcade toolkits to the new MCP Server framework. It outlines the necessary changes in terminology, package dependencies, and code structure, ensuring a smooth transition to the updated system. Users will -- [MiroApi](https://docs.arcade.dev/en/resources/integrations/productivity/miro-api.md): The MiroApi documentation provides users with tools and instructions for managing and interacting with Miro boards and organizational settings through the Miro API. It outlines various functionalities, such as retrieving access token information, managing board classifications, and handling legal holds, enabling -- [MongoDB](https://docs.arcade.dev/en/resources/integrations/databases/mongodb.md): This documentation page provides an overview of the Arcade MongoDB MCP Server, which enables agents to interact with MongoDB databases in a read-only capacity. Users can learn how to discover databases and collections, explore document structures, and execute safe queries, while also -- [MongoDB](https://docs.arcade.dev/en/resources/integrations/databases/postgres/mongodb.md): This documentation page provides a comprehensive overview of the Arcade MongoDB MCP Server, which enables agents to interact with MongoDB databases in a read-only capacity. Users can learn to discover databases and collections, explore document structures, and execute safe queries, while also -- [Notion](https://docs.arcade.dev/en/resources/integrations/productivity/notion.md): This documentation page provides users with a comprehensive overview of the Arcade Notion MCP Server, which facilitates interaction with Notion through a set of pre-built tools. Users can learn how to retrieve page content, create new pages, search for items, and access -- [Obsidian](https://docs.arcade.dev/en/resources/integrations/productivity/obsidian.md): This documentation page provides an overview of the Arcade Obsidian Toolkit, a community-contributed MCP Server verified by the Arcade team. It serves as a resource for users to understand the toolkit and directs them to the GitHub repository for further information. -- [On-premise MCP Servers](https://docs.arcade.dev/en/guides/deployment-hosting/on-prem.md): This documentation page provides guidance on deploying on-premises MCP servers within a hybrid architecture, enabling users to run tools in their own environment while utilizing Arcade's cloud infrastructure. It outlines the benefits of such a deployment, including data security, resource access, and +- [ArcadeEngineApi](https://docs.arcade.dev/en/resources/integrations/development/arcade-engine-api.md): The ArcadeEngineApi documentation provides users with a comprehensive guide to the tools available for interacting with the Arcade Engine API, focusing on managing authentication providers, secrets, and worker configurations. It outlines various actions users and LLMs can perform, such as retrieving +- [Asana](https://docs.arcade.dev/en/resources/integrations/productivity/asana.md): This documentation page provides users with a comprehensive guide to the Arcade Asana MCP Server, enabling them to build agents and AI applications that interact with Asana tasks, projects, and workspaces. Users can learn how to manage teams, create and update tasks +- [Asana Reference](https://docs.arcade.dev/en/resources/integrations/productivity/asana/reference.md): The Asana Reference documentation provides a comprehensive list of enumerations related to tag colors, task sorting options, and sort orders used in the Asana MCP Server. Users can utilize this reference to understand and implement the various color codes and sorting parameters effectively in +- [AsanaApi](https://docs.arcade.dev/en/resources/integrations/productivity/asana-api.md): The AsanaApi documentation provides users with tools and resources to effectively interact with the Asana API, enabling various actions such as managing access requests, allocations, custom fields, and goal relationships. It outlines a comprehensive set of functionalities that facilitate project management tasks +- [AshbyApi](https://docs.arcade.dev/en/resources/integrations/productivity/ashby-api.md): The AshbyApi documentation provides a comprehensive guide for users to effectively manage recruitment processes within the Ashby platform using various API tools. It enables users to create and update job applications, retrieve candidate information, manage interview schedules, and streamline hiring workflows through a +- [Authorized Tool Calling](https://docs.arcade.dev/en/guides/tool-calling/custom-apps/auth-tool-calling.md): The "Authorized Tool Calling" documentation provides a comprehensive guide for developers on how to securely authorize AI agents to access external services using OAuth 2.0, API keys, and user tokens. It outlines the steps for initializing the Arcade client, obtaining user +- [BoxApi](https://docs.arcade.dev/en/resources/integrations/productivity/box-api.md): The BoxApi documentation provides users with tools to manage and automate various aspects of Box content, including file management, metadata handling, collaboration, document generation, and enterprise operations. It outlines capabilities for interacting with Box's API, enabling users to build applications or +- [Brightdata](https://docs.arcade.dev/en/resources/integrations/development/brightdata.md): The Brightdata documentation page provides users with tools and guidance for scraping, searching, and extracting structured data from various websites at scale without being blocked. It outlines specific functionalities such as scraping web pages in Markdown format, performing advanced searches across major search engines, +- [Build MCP Server QuickStart](https://docs.arcade.dev/en/get-started/quickstarts/mcp-server-quickstart.md): The "Build MCP Server QuickStart" documentation provides a step-by-step guide for users to create and run a custom MCP Server using the Arcade MCP framework. It covers prerequisites, installation instructions, server setup, and how to connect and authorize tools, enabling +- [CalendlyApi](https://docs.arcade.dev/en/resources/integrations/productivity/calendly-api.md): The CalendlyApi documentation provides tools and resources for developers to integrate and manage scheduling and event-related tasks within the Calendly platform using OAuth2 authentication. Users can learn how to create, retrieve, and update event types, manage invitees, and handle +- [Call a tool in your IDE/MCP Client](https://docs.arcade.dev/en/get-started/quickstarts/call-tool-client.md): This documentation page guides users on how to create and utilize an MCP Gateway in their IDE or MCP Client to streamline the process of calling tools from multiple MCP servers. It covers the steps to set up the gateway, select relevant tools, and connect it to +- [Call tools from MCP clients](https://docs.arcade.dev/en/guides/create-tools/tool-basics/call-tools-mcp.md): This documentation page guides users on how to configure MCP clients to call tools from an MCP server, detailing necessary prerequisites and configuration steps. Users will learn to utilize the `arcade configure` command for setting up their clients, including options for different transport types +- [Calling tools in your agent with Arcade](https://docs.arcade.dev/en/get-started/quickstarts/call-tool-agent.md): This documentation page provides a comprehensive guide on how to use Arcade to enable AI agents to call various hosted tools, such as sending emails or updating documents. Users will learn to install the Arcade client, set up workflows, and implement authorization for tool calls. +- [Capture mode](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/capture-mode.md): This documentation page explains how to use Capture mode to record tool calls without scoring, enabling users to bootstrap test expectations, debug model behavior, and explore new tools. It provides a step-by-step guide on setting up evaluations in Capture mode, reviewing captured outputs +- [Checking Tool Authorization Status](https://docs.arcade.dev/en/guides/tool-calling/custom-apps/check-auth-status.md): This documentation page provides a comprehensive guide on how to check the authorization status of tools, enabling users to determine the necessary permissions and availability before execution. It includes instructions for initializing the client, checking the status for all tools or specific tools, and understanding various +- [Clickhouse](https://docs.arcade.dev/en/resources/integrations/databases/clickhouse.md): This documentation page provides users with a comprehensive guide to the Arcade Clickhouse MCP Server, which enables read-only interactions with Clickhouse databases. It outlines key features such as schema discovery, table exploration, and safe query execution, along with detailed instructions on using +- [Clickhouse](https://docs.arcade.dev/en/resources/integrations/databases/postgres/clickhouse.md): This documentation page provides a comprehensive guide for using the Arcade Clickhouse MCP Server, which enables agents to interact with Clickhouse databases in a read-only capacity. Users can learn how to discover database schemas, explore table structures, and execute safe SELECT queries, +- [Clickup](https://docs.arcade.dev/en/resources/integrations/productivity/clickup.md): This documentation page provides users with tools and guidance for integrating and interacting with ClickUp workspaces, enabling the creation, modification, and management of tasks, comments, and project structures. It outlines various functionalities available through the ClickUp MCP Server, including task +- [Clickup Reference](https://docs.arcade.dev/en/resources/integrations/productivity/clickup/reference.md): The Clickup Reference documentation provides users with a comprehensive list of enumerations for various tools within the Clickup MCP Server, including task priorities, filter scopes, task ordering, and comment resolutions. It helps users understand the predefined values they can utilize when creating +- [ClickupApi](https://docs.arcade.dev/en/resources/integrations/productivity/clickup-api.md): The ClickupApi documentation provides a suite of tools that enable users to interact with the ClickUp API for efficient task and project management. Users can learn how to authenticate via OAuth2 and utilize various functionalities, such as managing checklists, comments, custom +- [Close.io](https://docs.arcade.dev/en/resources/integrations/productivity/closeio.md): This documentation page for Close.io provides an overview of the CRM's capabilities, specifically focusing on managing leads, contacts, and deals. However, the content is currently marked as "Coming Soon," indicating that detailed information and guidance will be available in the future +- [Comparative evaluations](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/comparative-evaluations.md): The "Comparative Evaluations" documentation page guides users in testing and comparing the performance of AI models across different tool implementations using isolated tool registries, known as tracks. It outlines how to set up comparative evaluations, register tools, create test cases, +- [Compare MCP Server Types](https://docs.arcade.dev/en/guides/create-tools/tool-basics/compare-server-types.md): This documentation page provides a comparative overview of different MCP server types offered by Arcade, detailing their functionalities based on transport method and deployment options. Users can learn about the capabilities of each server type, including the availability of tools with or without authentication and secrets. +- [Confluence](https://docs.arcade.dev/en/resources/integrations/productivity/confluence.md): This documentation page provides a comprehensive overview of the Arcade Confluence MCP Server, which enables users to build agents and AI applications that interact with Confluence. It details various tools available for managing pages, spaces, and attachments, as well as searching for content +- [Contact Us](https://docs.arcade.dev/en/resources/contact-us.md): This documentation page provides users with information on how to connect with the Arcade team for support through various channels. It aims to facilitate communication and assistance for users and their agents. +- [Create a new Mastra project](https://docs.arcade.dev/en/guides/agent-frameworks/mastra/use-arcade-tools.md): This documentation page provides a step-by-step guide for integrating Arcade tools into a new Mastra project, enabling users to leverage these tools within their Mastra applications. It covers prerequisites, project setup, installation of the Arcade client, API key configuration, and +- [Create an evaluation suite](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/create-evaluation-suite.md): This documentation page provides a comprehensive guide on creating an evaluation suite to test AI models' tool usage through Arcade. Users will learn how to set up prerequisites, define evaluation files and suites, run evaluations, and interpret results, ensuring accurate tool selection and parameter +- [Create an MCP tool with secrets](https://docs.arcade.dev/en/guides/create-tools/tool-basics/create-tool-secrets.md): This documentation page guides users on how to create custom MCP tools that securely handle sensitive information, or "secrets," using the Arcade platform. It covers the process of reading secrets from various sources, such as environment variables and the Arcade Dashboard, and provides +- [Creating an MCP Server with Arcade](https://docs.arcade.dev/en/guides/create-tools/tool-basics/build-mcp-server.md): This documentation page provides a comprehensive guide for users to create, test, deploy, and publish a custom MCP Server using the Arcade framework. It details the installation of necessary tools, the scaffolding of a server project, and the setup of environment configurations, +- [CursorAgentsApi](https://docs.arcade.dev/en/resources/integrations/development/cursor-agents-api.md): The CursorAgentsApi documentation provides users with tools to manage and interact with background agents via the cursor_agents API, including functionalities for listing, inspecting, and deleting agents, retrieving their statuses and conversation histories, and accessing authentication and model recommendations. It serves as +- [CustomerioApi](https://docs.arcade.dev/en/resources/integrations/customer-support/customerio-api.md): The CustomerioApi documentation provides users with a comprehensive set of tools to interact with the Customer.io platform, enabling management of customer communications and marketing campaigns. It outlines various functionalities, such as triggering broadcasts, sending transactional messages, and retrieving campaign metrics, all +- [CustomerioPipelinesApi](https://docs.arcade.dev/en/resources/integrations/customer-support/customerio-pipelines-api.md): The CustomerioPipelinesApi documentation provides tools for integrating with the Customer.io Track API, enabling users to manage user data and track interactions effectively. It outlines various functionalities, such as identifying users, tracking events, and managing user groups, along with code +- [CustomerioTrackApi](https://docs.arcade.dev/en/resources/integrations/customer-support/customerio-track-api.md): The CustomerioTrackApi documentation provides a comprehensive overview of tools that enable users to manage customer data and interactions within the Customer.io platform. It includes detailed descriptions of various functionalities, such as adding or updating customer profiles, deleting customers, and tracking events, +- [DatadogApi](https://docs.arcade.dev/en/resources/integrations/development/datadog-api.md): The DatadogApi documentation provides users with tools and instructions for interacting with the Datadog API, enabling them to manage various aspects of Datadog services such as datastores and application keys. It includes guidance on authentication requirements and a comprehensive +- [Deploying to the cloud with Arcade Deploy](https://docs.arcade.dev/en/guides/deployment-hosting/arcade-deploy.md): This documentation page provides a comprehensive guide on deploying an MCP server to the cloud using Arcade Deploy, enabling users to access their server from any MCP client and support multiple users without the complexities of local server management. It outlines the prerequisites, deployment steps, and +- [Directly call third-party APIs](https://docs.arcade.dev/en/guides/tool-calling/call-third-party-apis.md): This documentation page provides a comprehensive guide on how to obtain user authorization tokens to directly call third-party APIs, specifically using the Gmail API as an example. Users will learn to manage authentication flows, handle authorization requests, and utilize tokens with external services, all +- [Discord](https://docs.arcade.dev/en/resources/integrations/social-communication/discord.md): This documentation page provides guidance on configuring and using the Discord authentication provider with Arcade, enabling users to call the Discord API on behalf of their users. It outlines the steps for creating a Discord application, integrating it with the Arcade Dashboard, and utilizing the Discord +- [Dropbox](https://docs.arcade.dev/en/resources/integrations/productivity/dropbox.md): This documentation page provides users with tools and guidance for integrating Dropbox functionalities into agents and AI applications. It outlines how to browse, search, and download files and folders from Dropbox using pre-built tools, along with code examples in Python and JavaScript. Additionally +- [E2B](https://docs.arcade.dev/en/resources/integrations/development/e2b.md): The E2B documentation page provides users with tools and guidance for running code in a sandboxed environment, specifically designed for building agents and AI applications. It details two main functionalities: executing code safely and generating static matplotlib charts, along with examples and authentication +- [Engine Configuration](https://docs.arcade.dev/en/guides/deployment-hosting/configure-engine.md): This documentation page provides advanced configuration options for enterprise customers self-hosting the Arcade Engine, detailing how to set up and customize engine settings using configuration files. Users will learn how to install the engine on various platforms, manage the `engine.yaml` and ` +- [Environment Variables](https://docs.arcade.dev/en/resources/integrations/social-communication/slack/environment-variables.md): This documentation page provides guidance on configuring environment variables related to Slack API interactions, specifically `SLACK_MAX_CONCURRENT_REQUESTS`, `MAX_PAGINATION_SIZE_LIMIT`, and `MAX_PAGINATION_TIMEOUT_SECONDS`. Users will learn how to adjust these settings +- [ExaApi](https://docs.arcade.dev/en/resources/integrations/search/exa-api.md): The ExaApi documentation provides users with a comprehensive guide to utilizing the Exa.ai Search API, enabling them to conduct searches, manage websets, and handle research requests effectively. It outlines various tools available within the API, detailing their functionalities such as +- [Fetch the "ScrapeUrl" tool from the "Firecrawl" MCP Server](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/use-arcade-tools.md): This documentation page provides a comprehensive guide for integrating Arcade tools into LangGraph applications, detailing prerequisites, environment setup, API key configuration, and tool management. Users will learn how to create and manage AI models, configure agents, and stream responses while utilizing code +- [Figma](https://docs.arcade.dev/en/resources/integrations/development/figma.md): This documentation page provides users with a comprehensive guide to the Figma MCP Server, enabling interaction with Figma's design files, components, and collaboration features through the Figma REST API. Users can learn to access file structures, manage components, add comments +- [FigmaApi](https://docs.arcade.dev/en/resources/integrations/productivity/figma-api.md): The FigmaApi documentation provides a comprehensive guide for developers to utilize tools that enable interaction with the Figma API, facilitating efficient management of design assets and collaboration on projects. Users can learn how to perform various actions such as retrieving Figma files, managing +- [Firecrawl](https://docs.arcade.dev/en/resources/integrations/development/firecrawl.md): The Firecrawl documentation provides users with a comprehensive guide to utilizing the Arcade Firecrawl MCP Server, enabling them to build agents and AI applications for scraping, crawling, and mapping websites. It outlines available tools, including functionalities for scraping URLs, crawling websites, +- [Frequently Asked Questions](https://docs.arcade.dev/en/resources/faq.md): This documentation page provides answers to frequently asked questions about the Arcade platform, helping users understand how to create and contribute tools, differentiate between various API keys and tools, and manage OAuth authentication. It guides users on building custom tools, collaborating effectively with project API +- [FreshserviceApi](https://docs.arcade.dev/en/resources/integrations/customer-support/freshservice-api.md): The FreshserviceApi documentation provides a comprehensive set of tools for programmatically interacting with the Freshservice platform, enabling users to manage various aspects such as organizational data, assets, software, service catalogs, users, and tickets. It outlines specific API endpoints that +- [Get Formatted Tool Definitions](https://docs.arcade.dev/en/guides/tool-calling/custom-apps/get-tool-definitions.md): This documentation page provides guidance on retrieving formatted tool definitions using the Arcade Client, enabling users to obtain single or multiple tool definitions in specific model provider formats, such as OpenAI. It also explains how to convert these definitions into Zod schemas for enhanced type +- [Getting Your API Key](https://docs.arcade.dev/en/get-started/setup/api-keys.md): This documentation page guides users on how to obtain and manage their Arcade API key, detailing the steps for generating keys via both the Arcade dashboard and CLI. It emphasizes the importance of securely storing API keys, as they serve as administrator credentials for accessing Arcade services +- [GitHub](https://docs.arcade.dev/en/resources/integrations/development/github.md): This documentation page provides a comprehensive guide for using the Arcade GitHub MCP Server, enabling users to build agents and AI applications that interact with GitHub repositories, issues, and pull requests. It outlines the necessary configurations, permissions, and features available for both +- [GithubApi](https://docs.arcade.dev/en/resources/integrations/development/github-api.md): The GitHubApi documentation provides tools for users to interact with the GitHub API, facilitating the management of repositories, issues, pull requests, and various administrative tasks within GitHub Enterprise. It outlines a range of functionalities, including creating and managing webhooks +- [Gmail](https://docs.arcade.dev/en/resources/integrations/productivity/gmail.md): This documentation page provides a comprehensive overview of the Arcade Gmail MCP Server, which enables users to build agents and AI applications that can send, read, and manage emails in Gmail. It details the available tools, their functionalities, and the necessary OAuth scopes required +- [Gmail Reference](https://docs.arcade.dev/en/resources/integrations/productivity/gmail/reference.md): The Gmail Reference documentation provides users with a list of enumerations related to the Gmail MCP Server, specifically detailing options for reply recipients and date ranges. This reference helps users understand and utilize specific parameters effectively in their Gmail-related tools. +- [Google Calendar](https://docs.arcade.dev/en/resources/integrations/productivity/google-calendar.md): This documentation page provides a comprehensive guide for developers to utilize the Arcade Google Calendar MCP Server, enabling them to create applications that interact with users' Google Calendar accounts. It outlines available tools for managing calendars and events, such as listing calendars, creating and updating +- [Google Contacts](https://docs.arcade.dev/en/resources/integrations/productivity/google-contacts.md): This documentation page provides a comprehensive guide for using the Arcade Google Contacts MCP Server, enabling users to build agents and AI applications that interact with Google Contacts. Users can learn how to create new contacts and search for existing ones by name, email, or phone +- [Google Docs](https://docs.arcade.dev/en/resources/integrations/productivity/google-docs.md): This documentation page provides an overview of the Arcade Google Docs MCP Server, enabling users to build agents and AI applications that interact with Google Docs. It outlines available tools for creating, editing, and managing documents, as well as accessing metadata and comments. Additionally +- [Google Drive](https://docs.arcade.dev/en/resources/integrations/productivity/google-drive.md): This documentation page provides users with tools and guidelines for interacting with Google Drive through the MCP Server, enabling efficient file management and access. Users can learn how to retrieve file structures, search for files, create and manage folders, and share files, all while +- [Google Drive Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-drive/reference.md): The Google Drive Reference documentation provides users with a comprehensive guide to the enumerations used in the Google Drive MCP Server, including options for sorting files, filtering by file types, defining permission roles, and supported upload MIME types. This resource helps users effectively manage +- [Google Finance](https://docs.arcade.dev/en/resources/integrations/search/google_finance.md): The Google Finance documentation page provides users with the tools and instructions necessary to retrieve real-time and historical stock data through the Arcade platform. It outlines specific API functions for obtaining stock summaries and historical data, along with code examples for implementation. Additionally, it offers +- [Google Flights](https://docs.arcade.dev/en/resources/integrations/search/google_flights.md): This documentation page provides users with the tools and instructions to integrate Google Flights search functionality into their applications using the Arcade platform. It specifically focuses on enabling the retrieval of one-way flight search results, detailing required parameters and optional settings for customization. Additionally, it +- [Google Hotels](https://docs.arcade.dev/en/resources/integrations/search/google_hotels.md): The Google Hotels documentation provides users with the tools and guidance needed to integrate hotel search functionality into their applications using the Arcade platform. It details the parameters and options available for the GoogleHotels.SearchHotels API, enabling users to retrieve hotel search results based on various +- [Google Jobs](https://docs.arcade.dev/en/resources/integrations/search/google_jobs.md): This documentation page provides users with tools and instructions for integrating the Arcade Google Jobs MCP Server, enabling them to search for job openings using the Google Jobs API. It outlines the available functionalities, including how to configure search parameters and authentication requirements. Users can learn +- [Google Maps](https://docs.arcade.dev/en/resources/integrations/search/google_maps.md): This documentation page provides tools for integrating Google Maps functionality into applications, specifically enabling users to obtain directions between two locations using either addresses or latitude/longitude coordinates. It outlines available tools, their parameters, and includes code examples for implementation in Python and JavaScript +- [Google News](https://docs.arcade.dev/en/resources/integrations/search/google_news.md): This documentation page provides users with tools to integrate Google News search functionality into their applications using the Arcade Google News MCP Server. It outlines how to use the `GoogleNews.SearchNewsStories` tool to search for news articles based on specified keywords, language, +- [Google Search](https://docs.arcade.dev/en/resources/integrations/search/google_search.md): This documentation page provides users with the tools and instructions needed to perform Google searches using the Arcade Search MCP Server and SerpAPI. It outlines how to implement the GoogleSearch.Search tool to retrieve organic search results, including necessary parameters and authentication details. Users +- [Google Sheets](https://docs.arcade.dev/en/resources/integrations/productivity/google-sheets.md): This documentation page provides a comprehensive overview of the Arcade Google Sheets MCP Server, enabling users to build agents and AI applications that interact with Google Sheets. It details various tools for creating, reading, and updating spreadsheets, as well as managing metadata and user profiles +- [Google Shopping Search](https://docs.arcade.dev/en/resources/integrations/search/google_shopping.md): The Google Shopping Search documentation provides users with tools and guidance to enable agents to search for products on Google Shopping using the Arcade platform. It outlines the available functionalities, including the required parameters for product searches, authentication details, and default settings for language and country +- [Google Slides](https://docs.arcade.dev/en/resources/integrations/productivity/google-slides.md): This documentation page provides users with tools and instructions for interacting with Google Slides presentations through the GoogleSlides MCP Server. Users can learn to create presentations, add slides, comment on specific slides, search for presentations in Google Drive, and convert content to markdown format +- [GoogleCalendar Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-calendar/reference.md): The GoogleCalendar Reference documentation provides a concise overview of key enumerations related to event visibility, update options for sending notifications, and managing Google Meet settings within the GoogleCalendar MCP Server. Users can reference these enumerations to effectively configure and manage calendar events and +- [GoogleDocs Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-docs/reference.md): The GoogleDocs Reference documentation provides a comprehensive list of enumerations used in the GoogleDocs MCP Server, specifically focusing on sorting options and document formats. Users can learn about various order criteria, such as creation and modification times, as well as supported document formats +- [GoogleSheets Reference](https://docs.arcade.dev/en/resources/integrations/productivity/google-sheets/reference.md): The GoogleSheets Reference documentation provides a comprehensive list of enumerations related to sorting and ordering data within the GoogleSheets MCP Server. Users can learn about various order criteria, such as creation time, modification time, and name, along with their corresponding identifiers for +- [Hosting Optoions Overview](https://docs.arcade.dev/en/guides/deployment-hosting.md): This documentation page provides an overview of the various hosting options available for the Arcade platform, including the Arcade Cloud service and on-premise deployments. Users can learn how to quickly set up and manage their applications with zero infrastructure through Arcade Cloud or configure on-prem +- [How to contribute a MCP Server](https://docs.arcade.dev/en/resources/integrations/contribute-a-server.md): This documentation page provides a step-by-step guide for developers on how to contribute their MCP Server to the Arcade documentation. It outlines the prerequisites for submission, including building and publishing the MCP Server, and details the process for opening a pull request and ensuring it +- [Hubspot](https://docs.arcade.dev/en/resources/integrations/sales/hubspot.md): This documentation page provides users with tools and guidance for integrating and automating interactions with HubSpot CRM through the HubSpot MCP Server. Users can learn how to perform various actions such as creating, updating, and retrieving data related to users, contacts, deals +- [Hubspot Reference](https://docs.arcade.dev/en/resources/integrations/sales/hubspot/reference.md): The Hubspot Reference documentation provides a comprehensive list of enumerations used in the Hubspot MCP Server, including various categories such as call directions, email statuses, meeting outcomes, and activity types. This reference helps users understand and implement the correct enumerations in +- [HubspotAutomationApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-automation-api.md): The HubspotAutomationApi documentation provides users with tools to manage and automate workflows within HubSpot, enabling tasks such as completing blocked action executions, retrieving email campaign details, and enrolling contacts in sequences. It offers a suite of API functionalities designed to streamline marketing +- [HubspotCmsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-cms-api.md): The HubspotCmsApi documentation provides users with a set of tools designed to manage and optimize content within the HubSpot CMS efficiently. It enables users to create, update, and delete various content types, manage blog authors and tags, and handle multi-language +- [HubspotConversationsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-conversations-api.md): The HubspotConversationsApi documentation provides users with tools and guidance for managing and interacting with conversation threads and channels within HubSpot. It enables users to perform actions such as retrieving conversation inboxes, sending messages, updating thread statuses, and accessing detailed message +- [HubspotCrmApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-crm-api.md): The HubspotCrmApi documentation provides tools for users to interact with the HubSpot CRM API, enabling efficient management of CRM records such as contacts, companies, deals, and appointments. Users can perform various operations including creating, updating, deleting records, +- [HubspotEventsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-events-api.md): The Hubspot Events API documentation provides tools and guidance for managing and analyzing event data within HubSpot, enabling users to efficiently track engagement, create and manage custom events, and send event completion data. It outlines various functionalities, including retrieving event completions, +- [HubspotMarketingApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-marketing-api.md): The HubspotMarketingApi documentation provides users with tools to efficiently manage and analyze marketing campaigns within the HubSpot platform. It enables actions such as creating, updating, and deleting campaigns, managing budgets, and retrieving performance metrics. This resource is essential for users +- [HubspotMeetingsApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-meetings-api.md): The HubspotMeetingsApi documentation provides users with tools and instructions for managing meetings through Hubspot's scheduling system. It enables users to schedule, book, and retrieve meeting details, as well as access available scheduling links and upcoming availability times. This page +- [HubspotUsersApi](https://docs.arcade.dev/en/resources/integrations/sales/hubspot-users-api.md): The HubspotUsersApi documentation provides users with tools to efficiently manage users and teams within a HubSpot account, including functionalities for retrieving user lists, creating and updating user accounts, and removing users. It offers detailed descriptions of available API tools, along with +- [Imgflip](https://docs.arcade.dev/en/resources/integrations/entertainment/imgflip.md): The Imgflip documentation provides users with tools to create and manage memes using the Imgflip API, enabling the development of agents and AI applications. Users can search for meme templates, retrieve popular memes, and create custom memes by adding text to existing templates. +- [Imgflip](https://docs.arcade.dev/en/resources/integrations/entertainment/spotify/imgflip.md): The Imgflip documentation page provides users with tools to create and manage memes using the Imgflip API, allowing them to search for meme templates, retrieve popular templates, and create custom memes. It outlines the available features, including a premium search option and customizable +- [Import necessary classes and modules](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/user-auth-interrupts.md): This documentation page guides users in creating a LangGraph workflow that integrates user authorization for specific Arcade tools, ensuring that only authorized tools are accessible to the language model. It provides step-by-step instructions on setting up the necessary environment, installing required packages, and +- [Initialize the Arcade client](https://docs.arcade.dev/en/guides/agent-frameworks/google-adk/use-arcade-tools.md): This documentation page provides a comprehensive guide for integrating Arcade tools into Google ADK applications, outlining the necessary prerequisites, setup procedures, and configuration steps. Users will learn how to manage and authorize Arcade tools, create agents, and run them effectively within their applications +- [IntercomApi](https://docs.arcade.dev/en/resources/integrations/customer-support/intercom-api.md): The IntercomApi documentation provides a comprehensive guide to tools that enable users to interact with the Intercom platform using OAuth2 authentication. It details various functionalities, such as managing admin information, creating and updating articles, and handling company data, allowing users to +- [Jira](https://docs.arcade.dev/en/resources/integrations/productivity/jira.md): This documentation page provides a comprehensive overview of the Jira MCP Server, which enables users and AI applications to efficiently manage Jira issues and projects. It outlines various functionalities such as creating, updating, and searching for issues, managing labels and attachments, and transitioning issues +- [Jira Environment Variables](https://docs.arcade.dev/en/resources/integrations/productivity/jira/environment-variables.md): This documentation page provides essential information on configuring Jira environment variables to optimize API interactions, specifically focusing on controlling the maximum concurrent requests, API request timeout, and cache size for improved performance during tool execution. Users will learn how to set numeric string values for these +- [Jira Reference](https://docs.arcade.dev/en/resources/integrations/productivity/jira/reference.md): This documentation page provides a reference for enumerations used in the Jira MCP Server, specifically detailing the various sprint states, priority scheme ordering, and issue comment ordering options available through the Jira API. Users can learn how to effectively filter and sort data within Jira +- [Linear](https://docs.arcade.dev/en/resources/integrations/productivity/linear.md): This documentation page provides users with a comprehensive guide to the Linear MCP Server, enabling them to effectively interact with Linear's issue tracking, project management, and team collaboration features. Users can learn how to create, manage, and update issues, projects, initiatives +- [LinkedIn](https://docs.arcade.dev/en/resources/integrations/social-communication/linkedin.md): This documentation page provides an overview of the Arcade LinkedIn MCP Server, which enables users to build agents and AI applications that interact with LinkedIn, specifically for creating text posts. It includes details on available tools, authentication methods, and example code for implementing +- [LumaApi](https://docs.arcade.dev/en/resources/integrations/productivity/luma-api.md): The LumaApi documentation provides users with tools and guidance for managing events and calendars within the Luma platform via its API. It covers functionalities such as creating and updating events, managing guest information, and handling invitations and coupons, all aimed at enhancing event +- [MailchimpMarketingApi](https://docs.arcade.dev/en/resources/integrations/productivity/mailchimp-marketing-api.md): The Mailchimp Marketing API documentation provides users with a set of tools for managing and optimizing their email marketing campaigns through direct interaction with the Mailchimp platform. Users can learn how to retrieve account information, manage audience contacts, create and automate marketing workflows, and +- [MCP Gateways](https://docs.arcade.dev/en/guides/create-tools/mcp-gateways.md): This documentation page provides a comprehensive guide on configuring and using MCP Gateways, which facilitate the connection of multiple MCP Servers to a single agent, application, or IDE. Users will learn how to create and manage MCP Gateways, select tools from different servers +- [Microsoft Teams](https://docs.arcade.dev/en/resources/integrations/social-communication/microsoft-teams.md): This documentation page provides users with a comprehensive guide to the Microsoft Teams MCP Server, enabling them to effectively manage teams, channels, and chats within Microsoft Teams. Users can learn how to retrieve information, send messages, and manage users and interactions, streamlining +- [MicrosoftTeams Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/microsoft-teams/reference.md): The MicrosoftTeams Reference documentation provides a comprehensive overview of key enumerations used in the MicrosoftTeams MCP Server, including types for matching criteria and team membership. Users can learn about specific values such as `PARTIAL_ALL`, `EXACT`, and `DIRECT +- [Migrate from toolkits to MCP servers](https://docs.arcade.dev/en/guides/create-tools/migrate-toolkits.md): This documentation page provides a comprehensive guide for users looking to migrate their existing Arcade toolkits to the new MCP Server framework. It outlines the necessary changes in terminology, package dependencies, and code structure, while offering step-by-step instructions for updating imports, creating +- [MiroApi](https://docs.arcade.dev/en/resources/integrations/productivity/miro-api.md): The MiroApi documentation provides users with a comprehensive set of tools to manage and interact with Miro boards and organizational settings through the Miro API. It details various functionalities, including retrieving board information, managing access tokens, and handling legal holds, enabling +- [MongoDB](https://docs.arcade.dev/en/resources/integrations/databases/mongodb.md): This documentation page provides a comprehensive overview of the Arcade MongoDB MCP Server, which enables users to interact with MongoDB databases in a read-only capacity. It outlines key features such as database discovery, collection exploration, and safe query execution, while also offering +- [MongoDB](https://docs.arcade.dev/en/resources/integrations/databases/postgres/mongodb.md): This documentation page provides a comprehensive guide for using the Arcade MongoDB MCP Server, which enables agents to interact with MongoDB databases in a read-only capacity. Users will learn how to discover databases and collections, explore document structures, and execute safe queries, +- [Notion](https://docs.arcade.dev/en/resources/integrations/productivity/notion.md): This documentation page provides users with a comprehensive guide to the Arcade Notion MCP Server, which enables the creation and management of agents and AI applications that interact with Notion. Users can learn how to utilize various tools to retrieve page content, create new pages +- [Obsidian](https://docs.arcade.dev/en/resources/integrations/productivity/obsidian.md): This documentation page provides an overview of the Arcade Obsidian Toolkit, a community-contributed MCP Sever verified by the Arcade team. It guides users to learn more about the toolkit by directing them to the associated GitHub repository. +- [On-premise MCP Servers](https://docs.arcade.dev/en/guides/deployment-hosting/on-prem.md): This documentation page provides guidance on deploying on-premises MCP servers within a hybrid architecture, enabling users to utilize Arcade's cloud infrastructure while maintaining control over their private resources and data security. Users will learn how to set up their MCP servers, create secure tunnels - [Or set it directly when initializing the client](https://docs.arcade.dev/en/guides/agent-frameworks/openai-agents/use-arcade-tools.md): This documentation page provides a comprehensive guide on integrating Arcade tools into OpenAI Agents applications, detailing the necessary prerequisites, environment setup, and configuration steps. Users will learn how to create and manage Arcade tools, set up agents, handle authentication, and run their -- [Organize your MCP server and tools](https://docs.arcade.dev/en/guides/create-tools/tool-basics/organize-mcp-tools.md): This documentation page provides best practices for organizing your MCP server and tools, including how to define and import tools from separate files and other packages. Users will learn to maintain a clean project structure, enhance readability, and effectively utilize decorators for tool management. Additionally -- [Outlook Calendar](https://docs.arcade.dev/en/resources/integrations/productivity/outlook-calendar.md): The Outlook Calendar documentation page provides tools for agents to manage calendar events using the Outlook API, enabling users to create, list, and retrieve events in their Outlook Calendar. It includes detailed descriptions of available tools, such as creating events and retrieving user information, -- [Outlook Mail](https://docs.arcade.dev/en/resources/integrations/productivity/outlook-mail.md): The Outlook Mail documentation page provides users with tools to read, write, and send emails using the Outlook API, enabling seamless email management within their applications. It outlines available functionalities, such as creating and updating draft emails, sending emails, and listing emails in -- [OutlookMail Reference](https://docs.arcade.dev/en/resources/integrations/productivity/outlook-mail/reference.md): The OutlookMail Reference documentation provides a comprehensive list of enumerations, folder names, email filter properties, and filter operators used in the OutlookMail MCP Server. It helps users understand and utilize these elements effectively for managing and filtering emails within the OutlookMail environment -- [page](https://docs.arcade.dev/en/resources/examples.md): This documentation page provides a collection of example applications that utilize Arcade's tools and MCP servers, showcasing various workflows and agent capabilities. Users can explore detailed descriptions and links to GitHub repositories for each app, as well as guidelines for submitting their own applications for -- [page](https://docs.arcade.dev/en/resources/integrations.md): This documentation page provides a comprehensive registry of all MCP Servers available within the Arcade ecosystem, helping users identify and access the various servers for their needs. -- [page](https://docs.arcade.dev/en/guides/agent-frameworks/crewai/use-arcade-tools.md): This documentation page provides a comprehensive guide on integrating Arcade tools into CrewAI applications, detailing the necessary prerequisites, setup, and configuration steps. Users will learn how to authorize and utilize these tools within their CrewAI agent teams, ensuring tailored functionality for their specific -- [page](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/auth-langchain-tools.md): This documentation page provides a step-by-step guide on how to authorize existing LangChain tools, such as the `GmailToolkit`, using the Arcade platform. It outlines the prerequisites, necessary installations, and detailed code examples in both Python and JavaScript to -- [page](https://docs.arcade.dev/en/guides/agent-frameworks/mastra/overview.md): This documentation page provides guidance on integrating Arcade's tool ecosystem with Mastra applications, enabling users to enhance their Mastra agents with a variety of pre-built tools for seamless interaction with third-party services. It outlines key mechanisms for tool discovery, schema conversion, -- [page](https://docs.arcade.dev/en/guides/agent-frameworks/mastra/user-auth-interrupts.md): This documentation page provides guidance on managing user-specific authorization for Arcade tools within Mastra applications, enabling dynamic tool loading and personalized authentication flows for multiple users. It outlines the steps to configure agents, create API endpoints for tool access, and handle authorization effectively to -- [page](https://docs.arcade.dev/en/guides/agent-frameworks/vercelai/using-arcade-tools.md): This documentation page provides a comprehensive guide on integrating Arcade tools with the Vercel AI SDK, enabling users to enhance their AI applications with features like real-time streaming responses and seamless switching between AI providers. It outlines the necessary steps for setup, including dependency -- [page](https://docs.arcade.dev/en/resources/integrations/productivity/dropbox/reference.md): This documentation page defines the various item categories used in Dropbox, including types such as image, document, PDF, spreadsheet, presentation, audio, video, folder, and paper. It helps users understand the classification of files and folders within the Dropbox ecosystem. -- [PagerDuty](https://docs.arcade.dev/en/resources/integrations/customer-support/pagerduty.md): This documentation page provides users with tools and guidance for integrating with PagerDuty, enabling them to read incidents, on-call information, services, and team data through an OAuth2 authentication process. It includes detailed descriptions of available tools, code snippets in Python and -- [PagerdutyApi](https://docs.arcade.dev/en/resources/integrations/development/pagerduty-api.md): The PagerDutyApi documentation provides users with a comprehensive set of tools to manage incidents, services, and integrations within the PagerDuty platform using OAuth2 authentication. It outlines various functionalities, including assigning tags, retrieving metrics, managing add-ons, and checking account -- [Postgres](https://docs.arcade.dev/en/resources/integrations/databases/postgres.md): This documentation page provides users with a comprehensive guide to the Arcade Postgres MCP Server, which enables agents to interact with PostgreSQL databases in a read-only capacity. Users can learn how to discover database schemas, explore table structures, and execute safe SELECT queries -- [PosthogApi](https://docs.arcade.dev/en/resources/integrations/development/posthog-api.md): The PosthogApi documentation provides users with tools and instructions for managing and analyzing data within the PostHog platform using its API. It details the necessary configuration, including authentication secrets, and outlines various available tools for retrieving application metrics, managing batch exports, -- [Provide the tool manager callback to the ArcadeToolManager](https://docs.arcade.dev/en/guides/agent-frameworks/crewai/custom-auth-flow.md): This documentation page provides a step-by-step guide for creating a custom authorization flow with CrewAI, specifically for executing Arcade tools. It outlines the necessary prerequisites, environment setup, and configuration needed to implement a tailored authorization process, allowing users to handle unique scenarios -- [Providing useful tool errors](https://docs.arcade.dev/en/guides/create-tools/error-handling/useful-tool-errors.md): This documentation page provides guidance on effectively handling errors when building tools with Arcade MCP, emphasizing the importance of robust error management. It explains the automatic error adaptation process, outlines when to raise specific errors, and offers insights into common error scenarios during tool development. -- [Pylon](https://docs.arcade.dev/en/resources/integrations/customer-support/pylon.md): The Pylon documentation provides agents with the necessary tools to manage issues, contacts, users, and teams within the Pylon MCP Server using an admin-generated API token. Users can perform actions such as listing and searching issues, assigning owners, and fetching user -- [Reddit](https://docs.arcade.dev/en/resources/integrations/social-communication/reddit.md): This documentation page provides tools and guidance for developers to create agents and AI applications that interact with Reddit using the Arcade Reddit MCP Server. Users can learn how to submit posts, comment on content, retrieve subreddit information, and manage user interactions through a variety of -- [Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/slack/reference.md): This documentation page provides a reference for enumerations related to conversation types used in the Slack MCP Server, including definitions for public channels, private channels, multi-person direct messages, and direct messages. Users can utilize this reference to understand and implement these conversation types -- [Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/twilio/reference.md): This documentation page provides a reference for the `arcade_twilio` package, which facilitates sending SMS and WhatsApp messages using Twilio. It includes installation instructions, a brief description of the package, and details on two main tools: `SendSms -- [Reference for Firecrawl Toolkit](https://docs.arcade.dev/en/resources/integrations/development/firecrawl/reference.md): The Firecrawl Toolkit documentation page provides a reference for users on the various output formats available for scraped web pages, including Markdown, HTML, raw HTML, links, and screenshots. It helps users understand how to customize the data they retrieve from web scraping based -- [RetryableToolError in Arcade](https://docs.arcade.dev/en/guides/create-tools/error-handling/retry-tools.md): This documentation page explains how to utilize the `RetryableToolError` in the Arcade Tool SDK to enhance tool call outcomes by providing additional context for input parameters. It guides users on when to raise this error to improve the model's predictions and includes an -- [Run evaluations with the Arcade CLI](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/run-evaluations.md): This documentation page provides users with guidance on how to run evaluations of tool-enabled language models using the Arcade CLI. It details the usage of the `arcade evals` command, including how to execute evaluation suites, customize the evaluation process with various options -- [Salesforce CRM](https://docs.arcade.dev/en/resources/integrations/sales/salesforce.md): This documentation page provides guidance on using the Arcade Salesforce CRM MCP Server, which enables users to interact with Salesforce data such as accounts, leads, and contacts through pre-built tools. It outlines the process for creating a custom Salesforce Auth Provider, self-hosting -- [Secure and Brand the Auth Flow in Production](https://docs.arcade.dev/en/guides/user-facing-agents/secure-auth-production.md): This documentation page provides guidance on securing and branding authentication flows in production using Arcade.dev. It outlines two primary methods: utilizing the default Arcade user verifier for development and implementing a custom user verifier for production applications, allowing developers to control the user experience during authorization -- [Securing Arcade MCP Deployments](https://docs.arcade.dev/en/guides/security/securing-arcade-mcp.md): This documentation page provides guidance on securing Arcade MCP deployments by outlining two primary methods: deploying the MCP server to the Arcade platform for built-in security features, or implementing OAuth 2.1 Resource Server authentication for self-hosted solutions. Users will learn how -- [Security Research Program](https://docs.arcade.dev/en/guides/security/security-research-program.md): The Security Research Program documentation page outlines how users can report security vulnerabilities in Arcade's tools and services, emphasizing the importance of community involvement in enhancing security. It details the scope of issues covered, the reporting process, and guidelines for responsible disclosure. Additionally, -- [Server-Level vs Tool-Level Authorization](https://docs.arcade.dev/en/learn/server-level-vs-tool-level-auth.md): This documentation page explains the differences between Server-Level Authorization and Tool-Level Authorization for Arcade MCP servers, highlighting their roles in securing access to the server and third-party APIs, respectively. It provides guidance on when to use each type of authorization, along with configuration -- [Set your API key](https://docs.arcade.dev/en/guides/agent-frameworks/openai-agents/user-auth-interrupts.md): This documentation page guides users on managing user authorization for Arcade tools within OpenAI Agents applications. It covers obtaining and configuring an Arcade API key, handling authorization errors, and implementing a complete authorization flow, ensuring users can effectively integrate and manage tool access in their -- [Setup Arcade with LangChain](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/use-arcade-with-langchain.md): This documentation page guides users on integrating Arcade tools within LangChain agents, enabling them to leverage Arcade's capabilities in their AI applications. Users will learn how to set up their environment, create a LangChain agent, and manage tool authorization through interrupts. By -- [Sharepoint](https://docs.arcade.dev/en/resources/integrations/productivity/sharepoint.md): This documentation page provides users with a comprehensive guide on how to interact with SharePoint through the SharePoint MCP Server, enabling actions such as retrieving lists, items, pages, and metadata from SharePoint sites. It outlines available tools, their functionalities, and -- [Slack](https://docs.arcade.dev/en/resources/integrations/social-communication/slack.md): This documentation page provides users with tools and instructions for integrating and interacting with the Slack platform through the Slack MCP Server. Users can learn how to manage conversations, retrieve user information, send messages, and access metadata for various Slack interactions, enhancing communication and collaboration -- [SlackApi](https://docs.arcade.dev/en/resources/integrations/social-communication/slack_api.md): The SlackApi documentation provides a comprehensive guide for administrators and developers to manage and automate various aspects of Slack workspaces, including user management, messaging, channel operations, and file sharing. It details the capabilities of the MCP Server, which supports OAuth2 authentication -- [SlackApi](https://docs.arcade.dev/en/resources/integrations/social-communication/slack-api.md): The SlackApi documentation provides a comprehensive guide for administrators and applications to effectively manage and interact with Slack workspaces. It covers a wide range of functionalities, including user management, channel operations, messaging, file sharing, and emoji management, all facilitated through OAuth -- [Spotify](https://docs.arcade.dev/en/resources/integrations/entertainment/spotify.md): This documentation page provides a comprehensive guide for using the Arcade Spotify MCP Server, which enables developers to build agents and AI applications that interact with Spotify tracks. Users can learn how to retrieve track information, control playback, and search the Spotify catalog through various available -- [SquareupApi](https://docs.arcade.dev/en/resources/integrations/productivity/squareup-api.md): The SquareupApi documentation provides users with tools to interact with the Square platform, enabling management of payments, customer data, and bookings through various API functionalities. It includes detailed descriptions of available tools, such as obtaining OAuth tokens, managing bank accounts, and -- [Stripe](https://docs.arcade.dev/en/resources/integrations/payments/stripe.md): This documentation page provides users with tools and guidance for interacting with the Stripe API, enabling them to build intelligent agents and applications that handle payment processing, customer management, and invoicing. Users can learn how to create and manage customers, products, prices, -- [StripeApi](https://docs.arcade.dev/en/resources/integrations/payments/stripe_api.md): The StripeApi documentation provides a comprehensive set of tools for developers to interact programmatically with the Stripe API, enabling them to manage accounts, customers, payments, billing, and more. Users can learn how to perform various operations such as retrieving account details, -- [Teams Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/teams/reference.md): The Teams Reference documentation provides a comprehensive overview of enumerations related to matching types and team membership within the Teams MCP Server. Users can learn about different match types, such as exact and partial matches, as well as the various membership classifications for teams and shared -- [The Arcade Registry](https://docs.arcade.dev/en/guides/create-tools/contribute/registry-early-access.md): The Arcade Registry documentation provides an overview of a platform where developers can share and monetize their tools for agentic applications, similar to HuggingFace or PyPI. It explains the benefits of coupling runtime metrics with the registry to enhance tool usability and feedback for -- [TicktickApi](https://docs.arcade.dev/en/resources/integrations/productivity/ticktick-api.md): The TicktickApi documentation provides users with tools to manage tasks and projects within the Ticktick platform through an API interface. Users can create, update, delete, and retrieve tasks and projects, enabling efficient task management. Additionally, the documentation includes examples and -- [Tool error handling](https://docs.arcade.dev/en/guides/tool-calling/error-handling.md): This documentation page provides guidance on effectively handling errors when using tools with Arcade's Tool Development Kit (TDK). It outlines the error handling philosophy, offers examples for different programming languages, and discusses best practices for creating robust applications. Users will learn how to -- [Tools](https://docs.arcade.dev/en/resources/tools.md): This documentation page provides an overview of Arcade's ecosystem of AI tools, enabling users to explore a catalog of pre-built integrations, create custom tools, and contribute their own tools to the community. It outlines the benefits of using Arcade tools, such as built -- [TrelloApi](https://docs.arcade.dev/en/resources/integrations/productivity/trello-api.md): The TrelloApi documentation provides a comprehensive guide for users and AI applications to interact with the Trello API, facilitating efficient management of boards, cards, lists, and members. It outlines various tools available for actions such as fetching, updating, and deleting -- [Twilio](https://docs.arcade.dev/en/resources/integrations/social-communication/twilio.md): This documentation page provides a guide for using Twilio to send SMS and WhatsApp messages through an MCP Server, specifically designed for integration with Arcade. It outlines the necessary prerequisites for setting up a Twilio account, configuration details, and offers usage examples to -- [Twitch auth provider](https://docs.arcade.dev/en/resources/integrations/entertainment/twitch.md): This documentation page provides instructions for creating and configuring a custom Twitch authentication provider in Arcade, enabling users to call the Twitch API on behalf of users. It outlines the steps to set up a Twitch app, integrate it with Arcade, and utilize the Twitch auth -- [Types of Tools](https://docs.arcade.dev/en/guides/create-tools/improve/types-of-tools.md): This documentation page explains the two types of tools offered by Arcade: Starter tools and Optimized tools. It highlights the differences in design and functionality, emphasizing that Optimized tools are tailored for AI interactions to enhance performance and reliability, while Starter tools provide more -- [Understanding `Context` and tools](https://docs.arcade.dev/en/guides/create-tools/tool-basics/runtime-data-access.md): This documentation page explains the purpose and usage of the `Context` class in Arcade's tools, detailing how it provides runtime capabilities and tool-specific data access. Users will learn how to utilize the `Context` object for tasks such as accessing OAuth tokens, -- [Use Arcade in Cursor](https://docs.arcade.dev/en/guides/tool-calling/mcp-clients/cursor.md): This documentation page provides a step-by-step guide for users to connect Cursor to an Arcade MCP Gateway, enabling them to utilize Arcade tools within the Cursor environment. It outlines the prerequisites for setup, including creating an Arcade account and obtaining an API key, and -- [Use Arcade in Visual Studio Code](https://docs.arcade.dev/en/guides/tool-calling/mcp-clients/visual-studio-code.md): This documentation page provides a step-by-step guide for connecting Visual Studio Code to an Arcade MCP Gateway, enabling users to set up and configure their development environment effectively. It outlines the prerequisites needed, such as creating an Arcade account and obtaining an API key, -- [Use Arcade with Claude Desktop](https://docs.arcade.dev/en/guides/tool-calling/mcp-clients/claude-desktop.md): This documentation page is intended to guide users on how to utilize Arcade with Claude Desktop. However, it currently indicates that the content is forthcoming and does not provide any specific instructions or information at this time. -- [VercelApi](https://docs.arcade.dev/en/resources/integrations/development/vercel-api.md): The VercelApi documentation provides users with a comprehensive guide to utilizing tools that enable interaction with the Vercel API for managing projects, domains, and integrations. It outlines various functionalities, such as creating and updating access groups, managing deployments, and -- [Walmart Search](https://docs.arcade.dev/en/resources/integrations/search/walmart.md): The Walmart Search documentation provides tools for developers to create agents and AI applications that can search for products on Walmart and retrieve detailed information about them. It outlines the available functionalities, including product search and detail retrieval, along with example code snippets in Python and Java -- [WeaviateApi](https://docs.arcade.dev/en/resources/integrations/databases/weaviate-api.md): The WeaviateApi documentation provides users with the necessary tools and instructions to manage and interact with the Weaviate vector search engine via its API. It covers authentication requirements, available API endpoints, and various functionalities such as user management, replication operations, -- [What are tools?](https://docs.arcade.dev/en/guides/tool-calling.md): This documentation page provides an overview of tool calling in language models, explaining how users can enhance AI interactions by enabling models to perform tasks like data retrieval and scheduling through external APIs. It outlines practical applications, such as analyzing documents and managing calendar events, while -- [Why evaluate tools?](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/why-evaluate.md): This documentation page explains the importance of evaluating tools used by language models in production environments, focusing on their effectiveness in tool utilization and intent understanding. It outlines a comprehensive evaluation framework that assesses the model's ability to select and call tools correctly, ensuring reliable performance -- [X (formerly Twitter)](https://docs.arcade.dev/en/resources/integrations/social-communication/x.md): This documentation page provides a comprehensive guide for developers to utilize the Arcade X (formerly Twitter) MCP Server, enabling them to build agents and AI applications that can interact with the X platform. Users can learn how to perform various actions such as posting, replying -- [XeroApi](https://docs.arcade.dev/en/resources/integrations/productivity/xero-api.md): The XeroApi documentation provides a comprehensive guide for developers to utilize tools that enable interaction with the Xero accounting API. Users can learn how to access and manage various accounting entities, retrieve financial reports, and handle organizational settings, facilitating integration, reporting, -- [YouTube Search](https://docs.arcade.dev/en/resources/integrations/search/youtube.md): The YouTube Search documentation page provides tools for developers to easily search for videos on YouTube and retrieve video details using the Arcade YouTube Search MCP Server. It outlines the available functionalities, including parameters for customizing searches, and offers code examples in Python and -- [Zendesk](https://docs.arcade.dev/en/resources/integrations/customer-support/zendesk.md): This documentation page provides an overview of the Zendesk MCP Server, which enables agents to efficiently manage customer support tickets and knowledge base articles through various tools. Users can list, comment on, and mark tickets as solved, as well as search for Help Center -- [Zendesk Reference](https://docs.arcade.dev/en/resources/integrations/customer-support/zendesk/reference.md): The Zendesk Reference documentation provides a comprehensive list of enumerations, including ticket statuses, sorting orders, and article sorting criteria, used in the Zendesk MCP Server. This page helps users understand and utilize these enumerations effectively in their applications. -- [ZohoBooksApi](https://docs.arcade.dev/en/resources/integrations/payments/zoho-books-api.md): The ZohoBooksApi documentation provides a comprehensive guide for users to manage financial transactions and accounting tasks within Zoho Books using various tools. It outlines functionalities such as creating, updating, and deleting bank accounts and transactions, as well as importing bank statements and -- [ZohoCreatorApi](https://docs.arcade.dev/en/resources/integrations/development/zoho-creator-api.md): The ZohoCreatorApi documentation provides users with a set of tools designed to facilitate interaction with Zoho Creator applications, enabling efficient data management and manipulation. Users can learn how to perform various actions such as fetching, updating, and deleting records, as well -- [Zoom](https://docs.arcade.dev/en/resources/integrations/social-communication/zoom.md): This documentation page provides an overview of the Arcade Zoom MCP Server, enabling users to build agents and AI applications that interact with Zoom by listing upcoming meetings and retrieving meeting invitations. It details the available tools, their functionalities, and the authentication process required to connect +- [Organize your MCP server and tools](https://docs.arcade.dev/en/guides/create-tools/tool-basics/organize-mcp-tools.md): This documentation page provides best practices for organizing your MCP server and tools, including how to define and import tools from separate files and other packages. Users will learn to maintain a clean project structure by keeping tools in a dedicated directory and using decorators effectively. Additionally +- [Outlook Calendar](https://docs.arcade.dev/en/resources/integrations/productivity/outlook-calendar.md): The Outlook Calendar documentation page provides users with tools to create, list, and retrieve events in their Outlook Calendar using the Outlook API. It includes detailed descriptions of available tools, such as creating events and fetching event details, along with example code snippets for implementation +- [Outlook Mail](https://docs.arcade.dev/en/resources/integrations/productivity/outlook-mail.md): The Outlook Mail documentation page provides users with tools to interact with the Outlook API, enabling them to read, write, and send emails efficiently. It details various functionalities, such as creating drafts, sending emails, and listing messages, along with code examples for +- [OutlookMail Reference](https://docs.arcade.dev/en/resources/integrations/productivity/outlook-mail/reference.md): The OutlookMail Reference documentation provides a comprehensive list of enumerations, folder names, email filter properties, and filter operators used in the OutlookMail MCP Server. It helps users understand and utilize these elements effectively when working with OutlookMail tools. This reference is +- [page](https://docs.arcade.dev/en/resources/examples.md): This documentation page showcases a collection of example applications that utilize Arcade's tools and MCP servers, providing users with practical implementations and templates for various workflows and agent functionalities. Users can explore detailed descriptions and links to GitHub repositories for each app, enabling them to +- [page](https://docs.arcade.dev/en/resources/integrations.md): This documentation page provides a comprehensive registry of all MCP Servers within the Arcade ecosystem, helping users to easily identify and access the available servers. +- [page](https://docs.arcade.dev/en/guides/agent-frameworks/crewai/use-arcade-tools.md): This documentation page provides a comprehensive guide for integrating Arcade tools into CrewAI applications, detailing the necessary prerequisites, environment setup, and configuration steps. Users will learn how to authorize and utilize these tools within their CrewAI agent teams, ensuring tailored functionality for their +- [page](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/auth-langchain-tools.md): This documentation page provides a step-by-step guide on how to authorize existing LangChain tools, such as the `GmailToolkit`, using the Arcade platform. It outlines the necessary prerequisites, installation of required packages, and detailed instructions for the authorization process in +- [page](https://docs.arcade.dev/en/guides/agent-frameworks/mastra/overview.md): This documentation page guides users on integrating Arcade's tool ecosystem with Mastra applications, enabling enhanced functionality for AI agents. It outlines how to access various pre-built tools, simplifies tool management, and details the processes for tool discovery, schema conversion, and execution +- [page](https://docs.arcade.dev/en/guides/agent-frameworks/mastra/user-auth-interrupts.md): This documentation page provides guidance on managing user-specific authorization for Arcade tools within Mastra applications, focusing on dynamic tool loading and per-user authentication flows. It outlines the steps to set up agents and API endpoints, ensuring each user has a secure and tailored experience +- [page](https://docs.arcade.dev/en/guides/agent-frameworks/vercelai/using-arcade-tools.md): This documentation page provides a comprehensive guide on integrating Arcade tools with the Vercel AI SDK, enabling users to enhance their AI applications with features like real-time streaming responses and easy switching between AI providers. It outlines the necessary steps for setup, including installing +- [page](https://docs.arcade.dev/en/resources/integrations/productivity/dropbox/reference.md): This documentation page defines the various item categories used in Dropbox, including types such as image, document, PDF, spreadsheet, presentation, audio, video, folder, and paper. It helps users understand the classification of files within the Dropbox system. +- [PagerDuty](https://docs.arcade.dev/en/resources/integrations/customer-support/pagerduty.md): This documentation page provides guidance on using the PagerDuty MCP Server, which enables agents to access and manage incidents, on-call information, services, and teams through read-only API tools. It includes details on OAuth configuration, available tools with descriptions, and code +- [PagerdutyApi](https://docs.arcade.dev/en/resources/integrations/development/pagerduty-api.md): The PagerDutyApi documentation provides a comprehensive overview of tools that enable users to manage incidents, services, and integrations within the PagerDuty platform using the API. It outlines various functionalities, such as assigning tags, retrieving metrics, and managing add-ons, allowing +- [Postgres](https://docs.arcade.dev/en/resources/integrations/databases/postgres.md): This documentation page provides users with a comprehensive guide to the Arcade Postgres MCP Server, which enables agents to interact with PostgreSQL databases in a read-only manner. Users can learn how to discover database schemas, explore table structures, and execute safe SELECT queries +- [PosthogApi](https://docs.arcade.dev/en/resources/integrations/development/posthog-api.md): The PosthogApi documentation provides users with tools and guidance for managing and analyzing data within the PostHog platform via its API. It details the necessary configuration, including authentication secrets, and outlines various available tools for retrieving metrics, managing exports, and handling +- [Provide the tool manager callback to the ArcadeToolManager](https://docs.arcade.dev/en/guides/agent-frameworks/crewai/custom-auth-flow.md): This documentation page provides a step-by-step guide on creating a custom authorization flow for executing Arcade tools within a CrewAI agent team. It outlines the prerequisites, environment setup, and configuration needed to implement a tailored approach to tool authorization, allowing for unique interfaces +- [Providing useful tool errors](https://docs.arcade.dev/en/guides/create-tools/error-handling/useful-tool-errors.md): This documentation page teaches developers how to effectively handle errors when building tools with Arcade MCP, emphasizing the automatic error adaptation feature that reduces boilerplate code. It outlines the error handling philosophy, explains the use of error adapters, and provides guidance on when to raise +- [Pylon](https://docs.arcade.dev/en/resources/integrations/customer-support/pylon.md): The Pylon documentation provides agents with the necessary tools and API functionalities to manage issues, contacts, users, and teams within the Pylon MCP Server. Users can learn how to list and search issues, assign ownership, and interact with user and team data +- [Reddit](https://docs.arcade.dev/en/resources/integrations/social-communication/reddit.md): This documentation page provides users with a comprehensive guide to the Arcade Reddit MCP Server, which enables the development of agents and AI applications that can interact with Reddit. It details various tools available for actions such as submitting posts, commenting, retrieving content, and checking +- [Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/slack/reference.md): This documentation page provides a reference for the enumerations related to conversation types used in the Slack MCP Server, including definitions for public channels, private channels, multi-person direct messages, and direct messages. Users can refer to this information to understand the different conversation +- [Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/twilio/reference.md): This documentation page provides a reference for the `arcade_twilio` package, which enables users to integrate Twilio services for sending SMS and WhatsApp messages. It includes installation instructions, a brief description of the package, and detailed usage guidelines for the +- [Reference for Firecrawl Toolkit](https://docs.arcade.dev/en/resources/integrations/development/firecrawl/reference.md): This documentation page provides a reference for the Firecrawl Toolkit, detailing the various output formats available for scraped web pages, including Markdown, HTML, raw HTML, links, and screenshot options. Users can learn how to customize their data extraction by selecting the desired +- [RetryableToolError in Arcade](https://docs.arcade.dev/en/guides/create-tools/error-handling/retry-tools.md): This documentation page explains how to utilize the `RetryableToolError` in the Arcade Tool SDK to enhance tool call outcomes by providing additional context for input parameters. It outlines when to raise this error and includes an example demonstrating its application in a tool that +- [Run evaluations](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/run-evaluations.md): This documentation page provides guidance on using the `arcade evals` command to run evaluation suites across multiple providers and models, allowing users to discover, execute, and analyze evaluations with various output formats. It covers basic usage, multi-provider support, API +- [Salesforce CRM](https://docs.arcade.dev/en/resources/integrations/sales/salesforce.md): This documentation page provides guidance on using the Arcade Salesforce CRM MCP Server, which enables users to interact with Salesforce accounts, leads, and contacts through a set of pre-built tools. It outlines the steps for creating a custom Salesforce Auth Provider, self-hosting +- [Secure and Brand the Auth Flow in Production](https://docs.arcade.dev/en/guides/user-facing-agents/secure-auth-production.md): This documentation page guides users on how to secure and customize their authentication flows using Arcade.dev in production environments. It explains the use of the default Arcade user verifier for development and the implementation of a custom user verifier for production applications, ensuring user verification and enhancing +- [Securing Arcade MCP Deployments](https://docs.arcade.dev/en/guides/security/securing-arcade-mcp.md): This documentation page provides guidance on securing Arcade MCP deployments, detailing two primary methods: deploying the MCP server to the Arcade platform for built-in security features and implementing OAuth 2.1 Resource Server authentication for self-hosted environments. Users will learn how to +- [Security Research Program](https://docs.arcade.dev/en/guides/security/security-research-program.md): The Security Research Program documentation page outlines how users can report security vulnerabilities in Arcade's tools and services, emphasizing the importance of community involvement in enhancing security. It details the types of vulnerabilities sought, the reporting process, and guidelines for responsible disclosure. Additionally, +- [Server-Level vs Tool-Level Authorization](https://docs.arcade.dev/en/learn/server-level-vs-tool-level-auth.md): This documentation page explains the differences between server-level authorization (Resource Server auth) and tool-level authorization in Arcade MCP servers, highlighting their roles in securing access to the server and third-party APIs. It provides guidance on when to implement each type of authorization, +- [Set your API key](https://docs.arcade.dev/en/guides/agent-frameworks/openai-agents/user-auth-interrupts.md): This documentation page provides guidance on managing user authorization for Arcade tools within OpenAI Agents applications. It covers obtaining an API key, configuring the environment, handling authorization errors, and ensuring persistence of user authorization. Users will learn how to effectively implement and manage the +- [Setup Arcade with LangChain](https://docs.arcade.dev/en/guides/agent-frameworks/langchain/use-arcade-with-langchain.md): This documentation page provides a comprehensive guide on integrating Arcade tools within LangChain agents, enabling users to build and manage AI agents effectively. It covers the necessary prerequisites, key concepts like agents and interrupts, and detailed steps for setting up a project, importing required +- [Sharepoint](https://docs.arcade.dev/en/resources/integrations/productivity/sharepoint.md): This documentation page provides a comprehensive guide for using the SharePoint MCP Server, enabling users to efficiently interact with SharePoint sites and their contents through various tools. Users can learn to retrieve lists, items, pages, and metadata, as well as search for +- [Slack](https://docs.arcade.dev/en/resources/integrations/social-communication/slack.md): This documentation page provides users with tools and functionalities to integrate and interact with the Slack platform, enabling efficient management of conversations and user information. It outlines various capabilities, such as retrieving user details, sending messages, and accessing conversation metadata, all aimed at enhancing +- [SlackApi](https://docs.arcade.dev/en/resources/integrations/social-communication/slack_api.md): The SlackApi documentation provides a comprehensive guide for administrators and applications to manage and automate various aspects of Slack workspaces, including user management, messaging, channel operations, and file sharing. It outlines key functionalities such as creating teams, managing user profiles, sending +- [SlackApi](https://docs.arcade.dev/en/resources/integrations/social-communication/slack-api.md): The SlackApi documentation provides a comprehensive guide for administrators and developers to effectively manage and automate various aspects of Slack workspaces, including user management, messaging, channel operations, and file sharing. It outlines key functionalities such as creating teams, managing user profiles, +- [Spotify](https://docs.arcade.dev/en/resources/integrations/entertainment/spotify.md): This documentation page provides tools for developers to integrate Spotify functionalities into their applications using the Arcade platform. Users can learn how to retrieve track information, control playback, and search the Spotify catalog through a set of pre-built tools, all requiring a self-hosted +- [SquareupApi](https://docs.arcade.dev/en/resources/integrations/productivity/squareup-api.md): The SquareupApi documentation provides users with tools and functionalities to interact with the Square platform, enabling management of payments, customer information, and bookings through an OAuth2 authentication process. Users can learn how to perform various actions such as obtaining OAuth tokens, managing +- [Stripe](https://docs.arcade.dev/en/resources/integrations/payments/stripe.md): This documentation page provides users with tools and guidance to interact with the Stripe API, enabling the creation and management of customers, products, invoices, and payment processing. Users can learn how to implement various functionalities, such as creating customers or products, listing invoices +- [StripeApi](https://docs.arcade.dev/en/resources/integrations/payments/stripe_api.md): The StripeApi documentation provides tools for developers to interact with the Stripe API, enabling them to manage accounts, customers, payments, billing, and various Stripe resources programmatically. Users can perform a wide range of operations, including retrieving account details, managing payment +- [Teams Reference](https://docs.arcade.dev/en/resources/integrations/social-communication/teams/reference.md): The Teams Reference documentation provides a comprehensive list of enumerations related to matching types and team membership within the Teams MCP Server. Users can learn about different match criteria, such as exact and partial matches, as well as the types of team memberships available. This +- [The Arcade Registry](https://docs.arcade.dev/en/guides/create-tools/contribute/registry-early-access.md): The Arcade Registry documentation provides an overview of a platform designed for tool developers to share and monetize their integrations for agentic applications. It explains how the registry collects real-time usage metrics and feedback to enhance tool effectiveness, while inviting users to participate in the early +- [TicktickApi](https://docs.arcade.dev/en/resources/integrations/productivity/ticktick-api.md): The TicktickApi documentation provides a set of tools for developers to manage tasks and projects within the Ticktick platform using OAuth2 authentication. Users can learn how to create, update, delete, and retrieve tasks and projects through various API endpoints, enabling seamless +- [Tool error handling](https://docs.arcade.dev/en/guides/tool-calling/error-handling.md): This documentation page provides guidance on effectively handling errors when using tools with Arcade's Tool Development Kit (TDK). It outlines the error handling philosophy, describes various error types, and offers best practices for implementing robust error management in applications. Users will learn how +- [Tools](https://docs.arcade.dev/en/resources/tools.md): This documentation page provides an overview of Arcade's ecosystem of AI tools, enabling users to explore a catalog of pre-built integrations, create custom tools, and contribute to the community. It outlines the benefits of using Arcade tools, including built-in authentication and universal +- [TrelloApi](https://docs.arcade.dev/en/resources/integrations/productivity/trello-api.md): The TrelloApi documentation provides users with a comprehensive set of tools for interacting with the Trello API, enabling efficient management of boards, cards, lists, and members. It outlines various functionalities, such as fetching and updating actions, retrieving board details, +- [Twilio](https://docs.arcade.dev/en/resources/integrations/social-communication/twilio.md): This documentation page provides users with a comprehensive guide to using Twilio for sending SMS and WhatsApp messages through an MCP Server, including setup prerequisites and configuration details. It outlines the necessary credentials required for integration and offers practical usage examples to demonstrate the server's +- [Twitch auth provider](https://docs.arcade.dev/en/resources/integrations/entertainment/twitch.md): The Twitch auth provider documentation page guides users in creating a custom authentication provider to access the Twitch API using their own OAuth 2.0 credentials. It outlines the steps to configure Twitch authentication within the Arcade platform, including creating a Twitch application and integrating it +- [Types of Tools](https://docs.arcade.dev/en/guides/create-tools/improve/types-of-tools.md): This documentation page explains the two types of tools offered by Arcade: Optimized tools and Starter tools. It highlights the differences in design and functionality, emphasizing that Optimized tools are tailored for AI-powered chat interfaces to improve performance, while Starter tools provide more +- [Understanding `Context` and tools](https://docs.arcade.dev/en/guides/create-tools/tool-basics/runtime-data-access.md): This documentation page explains the `Context` class used in Arcade tools, detailing how to access runtime capabilities and tool-specific data securely. Users will learn how to utilize the `Context` object to retrieve OAuth tokens, secrets, user information, and to log +- [Use Arcade in Cursor](https://docs.arcade.dev/en/guides/tool-calling/mcp-clients/cursor.md): This documentation page provides a step-by-step guide for users to connect Cursor to an Arcade MCP Gateway, enabling the use of Arcade tools within Cursor. It outlines the prerequisites needed for setup, including creating an Arcade account and obtaining an API key, as well +- [Use Arcade in Visual Studio Code](https://docs.arcade.dev/en/guides/tool-calling/mcp-clients/visual-studio-code.md): This documentation page provides a step-by-step guide for connecting Visual Studio Code to an Arcade MCP Gateway, enabling users to integrate and utilize Arcade tools within the IDE. It outlines the prerequisites for setup, including creating an Arcade account and obtaining an API key, +- [Use Arcade with Claude Desktop](https://docs.arcade.dev/en/guides/tool-calling/mcp-clients/claude-desktop.md): This documentation page is intended to guide users on how to utilize Arcade with Claude Desktop. However, it currently indicates that the content is forthcoming and does not provide any detailed instructions or information at this time. +- [VercelApi](https://docs.arcade.dev/en/resources/integrations/development/vercel-api.md): The VercelApi documentation provides a comprehensive guide for users to manage their Vercel projects, domains, and integrations through various API tools. It outlines available functionalities such as creating and managing access groups, handling deployments, and managing DNS records, enabling +- [Walmart Search](https://docs.arcade.dev/en/resources/integrations/search/walmart.md): The Walmart Search documentation provides tools for developers to integrate product search and details retrieval from Walmart into their applications. It outlines how to use the `Walmart.SearchProducts` and `Walmart.GetProductDetails` tools, including parameters for customizing searches and retrieving +- [WeaviateApi](https://docs.arcade.dev/en/resources/integrations/databases/weaviate-api.md): The WeaviateApi documentation provides users with essential tools and instructions for managing and interacting with the Weaviate vector search engine via its API. It covers authentication requirements, including obtaining API keys, and offers a comprehensive list of available API endpoints for various +- [What are tools?](https://docs.arcade.dev/en/guides/tool-calling.md): This documentation page provides an overview of tool calling in language models, explaining how users can leverage external tools to enhance the capabilities of AI models for tasks like data retrieval, scheduling, and email communication. It outlines the process of integrating these tools through the Arcade +- [Why evaluate tools?](https://docs.arcade.dev/en/guides/create-tools/evaluate-tools/why-evaluate.md): This documentation page explains the importance of evaluating tools for AI models to ensure they select the correct tools and provide accurate parameters in production. It outlines the evaluation process, scoring components, and potential pitfalls of inadequate evaluations, while also providing guidance on creating evaluation suites +- [X (formerly Twitter)](https://docs.arcade.dev/en/resources/integrations/social-communication/x.md): This documentation page provides users with tools and guidance for interacting with X (formerly Twitter) through the Arcade MCP Server, enabling the creation of agents and AI applications. Users can learn how to post, reply to, delete tweets, and search for tweets by +- [XeroApi](https://docs.arcade.dev/en/resources/integrations/productivity/xero-api.md): The XeroApi documentation provides tools for developers to interact with the Xero accounting API, enabling the management of various accounting entities such as invoices, payments, and financial reports. Users can access, create, and manage resources, retrieve attachments, and perform +- [YouTube Search](https://docs.arcade.dev/en/resources/integrations/search/youtube.md): The YouTube Search documentation page provides users with tools to easily search for videos on YouTube and retrieve video details using the Arcade YouTube Search MCP Server. It outlines the available functionalities, including parameters for customizing searches, and offers guidance on authentication and default +- [Zendesk](https://docs.arcade.dev/en/resources/integrations/customer-support/zendesk.md): This documentation page provides users with tools and guidance for integrating and managing customer support tickets through Zendesk. Users can learn how to list, comment on, and solve tickets, as well as search for knowledge base articles, streamlining their customer support processes. +- [Zendesk Reference](https://docs.arcade.dev/en/resources/integrations/customer-support/zendesk/reference.md): The Zendesk Reference documentation provides a comprehensive list of enumerations related to ticket statuses, sorting orders, and article sorting options used within the Zendesk MCP Server. Users can refer to this page to understand the specific values and their corresponding meanings, facilitating effective +- [ZohoBooksApi](https://docs.arcade.dev/en/resources/integrations/payments/zoho-books-api.md): The ZohoBooksApi documentation provides users with tools and guidance for managing financial transactions and accounting tasks within Zoho Books, utilizing OAuth2 for authentication. It outlines various functionalities, including creating, updating, and deleting bank accounts and transactions, as well as +- [ZohoCreatorApi](https://docs.arcade.dev/en/resources/integrations/development/zoho-creator-api.md): The ZohoCreatorApi documentation provides users with a set of tools designed to facilitate interaction with Zoho Creator applications through the API, enabling efficient data management and manipulation. Users can learn how to perform various actions such as fetching records, updating data, and +- [Zoom](https://docs.arcade.dev/en/resources/integrations/social-communication/zoom.md): This documentation page provides guidance on using the Arcade Zoom MCP Server, which allows users to build agents and AI applications that interact with Zoom by listing upcoming meetings and retrieving meeting invitations. It details the available tools, their functionalities, and necessary authentication processes for accessing