From 16979f8bee059505f6167804336875764af4beca Mon Sep 17 00:00:00 2001 From: nexxeln <95541290+nexxeln@users.noreply.github.com> Date: Thu, 4 Dec 2025 19:00:38 +0000 Subject: [PATCH] github connector docs (#594) --- apps/docs/connectors/github.mdx | 550 ++++++++++++++++++++++++++++++ apps/docs/connectors/overview.mdx | 12 +- apps/docs/docs.json | 1 + 3 files changed, 561 insertions(+), 2 deletions(-) create mode 100644 apps/docs/connectors/github.mdx diff --git a/apps/docs/connectors/github.mdx b/apps/docs/connectors/github.mdx new file mode 100644 index 00000000..71bbccf7 --- /dev/null +++ b/apps/docs/connectors/github.mdx @@ -0,0 +1,550 @@ +--- +title: "GitHub Connector" +description: "Connect GitHub repositories to sync documentation files into your Supermemory knowledge base" +icon: "github" +--- + +Connect GitHub repositories to sync documentation files into your Supermemory knowledge base with OAuth authentication, webhook support, and automatic incremental syncing. + +## Quick Setup + +### 1. Create GitHub Connection + + + + ```typescript + import Supermemory from 'supermemory'; + + const client = new Supermemory({ + apiKey: process.env.SUPERMEMORY_API_KEY! + }); + + const connection = await client.connections.create('github', { + redirectUrl: 'https://yourapp.com/auth/github/callback', + containerTags: ['user-123', 'github-sync'], + documentLimit: 5000, + metadata: { + source: 'github', + team: 'engineering' + } + }); + + // Redirect user to GitHub OAuth + window.location.href = connection.authLink; + console.log('Auth expires in:', connection.expiresIn); + ``` + + + ```python + from supermemory import Supermemory + import os + + client = Supermemory(api_key=os.environ.get("SUPERMEMORY_API_KEY")) + + connection = client.connections.create( + 'github', + redirect_url='https://yourapp.com/auth/github/callback', + container_tags=['user-123', 'github-sync'], + document_limit=10000, + metadata={ + 'source': 'github', + 'team': 'engineering' + } + ) + + # Redirect user to GitHub OAuth + print(f'Redirect to: {connection.auth_link}') + print(f'Expires in: {connection.expires_in}') + ``` + + + ```bash + curl -X POST "https://api.supermemory.ai/v3/connections/github" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "redirectUrl": "https://yourapp.com/auth/github/callback", + "containerTags": ["user-123", "github-sync"], + "documentLimit": 5000, + "metadata": { + "source": "github", + "team": "engineering" + } + }' + ``` + + + + +**OAuth Scopes:** The GitHub connector requires these scopes: +- `repo` - Access to private and public repositories +- `user:email` - Access to user's email address +- `admin:repo_hook` - Manage webhooks for incremental sync + + +### 2. Handle OAuth Callback + +After the user grants permissions, GitHub redirects to your callback URL. The connection is automatically established, and the user can now select which repositories to sync. + +### 3. List and Configure Repositories + +Unlike other connectors, GitHub requires repository selection before syncing begins. This gives your users control over which repositories to index. + + + + ```typescript + // List available repositories for the user + const repositories = await client.connections.github.listRepositories( + connectionId, + { + page: 1, + perPage: 100 + } + ); + + // Display repositories in your UI + repositories.forEach(repo => { + console.log(`${repo.full_name} - ${repo.description}`); + console.log(`Private: ${repo.private}`); + console.log(`Default branch: ${repo.default_branch}`); + console.log(`Last updated: ${repo.updated_at}`); + }); + + // After user selects repositories, configure them + await client.connections.github.configure(connectionId, { + repositories: [ + { + id: repo.id, + name: repo.full_name, + defaultBranch: repo.default_branch + } + ] + }); + + console.log('Repository sync initiated'); + ``` + + + ```python + # List available repositories for the user + repositories = client.connections.github.list_repositories( + connection_id, + page=1, + per_page=100 + ) + + # Display repositories in your UI + for repo in repositories: + print(f'{repo.full_name} - {repo.description}') + print(f'Private: {repo.private}') + print(f'Default branch: {repo.default_branch}') + print(f'Last updated: {repo.updated_at}') + + # After user selects repositories, configure them + client.connections.github.configure( + connection_id, + repositories=[ + { + 'id': repo.id, + 'name': repo.full_name, + 'defaultBranch': repo.default_branch + } + ] + ) + + print('Repository sync initiated') + ``` + + + ```bash + # List available repositories + curl -X GET "https://api.supermemory.ai/v3/connections/{connectionId}/github/repositories?page=1&per_page=100" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" + + # Configure selected repositories + curl -X POST "https://api.supermemory.ai/v3/connections/{connectionId}/github/configure" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "repositories": [ + { + "id": 123456789, + "name": "your-org/documentation", + "defaultBranch": "main" + }, + { + "id": 987654321, + "name": "your-org/api-docs", + "defaultBranch": "main" + } + ] + }' + ``` + + + + +**API-First Design:** + +Supermemory provides the API endpoints to list and configure repositories. As a Supermemory customer, you need to build the UI in your application where your end-users can: +1. View their available GitHub repositories +2. Select which repositories to sync +3. Confirm the selection + +This gives you complete control over the user experience and allows you to integrate repository selection seamlessly into your application's workflow. + + +## Supported Document Types + +The GitHub connector syncs documentation and text files with the following extensions: + +- **Markdown files**: `.md`, `.mdx`, `.markdown` +- **Text files**: `.txt` +- **reStructuredText**: `.rst` +- **AsciiDoc**: `.adoc` +- **Org-mode**: `.org` + +Files are indexed as `github_markdown` document type in Supermemory. + + +Only text-based documentation files are synced. Binary files, images, and code files (`.js`, `.py`, `.go`, etc.) are excluded by default to focus on searchable documentation content. + + +## Incremental Sync with Webhooks + +The GitHub connector automatically sets up webhooks for real-time incremental syncing. When files are pushed or deleted in configured repositories, Supermemory is notified immediately. + + +**Batch Processing:** Webhook events are processed in batches with a 10-minute delay to optimize performance and prevent excessive syncing during rapid commits. This means changes pushed to your repository will be reflected in Supermemory within approximately 10 minutes. + + +### How It Works + +1. **Webhook Setup**: When you configure repositories, a webhook is automatically installed in each repository +2. **Push Events**: When commits are pushed to the default branch, changed documentation files are synced +3. **Delete Events**: When documentation files are deleted, they're removed from your Supermemory knowledge base +4. **Incremental Updates**: Only changed files are processed, keeping sync fast and efficient + +### Webhook Security + +Webhooks are secured using HMAC-SHA256 signature validation with constant-time comparison. Supermemory automatically validates that webhook events come from GitHub before processing them. Each repository gets a unique webhook secret for maximum security. + + + + ```typescript + // Check webhook status + const connection = await client.connections.get(connectionId); + + console.log('Webhooks configured:', connection.metadata.webhooks?.length); + console.log('Last sync:', new Date(connection.metadata.lastSyncedAt)); + console.log('Repositories:', connection.metadata.repositories); + ``` + + + ```python + # Check webhook status + connection = client.connections.get(connection_id) + + print(f'Webhooks configured: {len(connection.metadata.get("webhooks", []))}') + print(f'Last sync: {connection.metadata.get("lastSyncedAt")}') + print(f'Repositories: {connection.metadata.get("repositories")}') + ``` + + + ```bash + # Get connection details including webhook status + curl -X POST "https://api.supermemory.ai/v3/connections/list" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{}' + ``` + + + +## Connection Management + +### List All Connections + + + + ```typescript + // List all GitHub connections for specific container tags + const connections = await client.connections.list({ + containerTags: ['user-123'], + provider: 'github' + }); + + connections.forEach(conn => { + console.log(`Provider: ${conn.provider}`); + console.log(`ID: ${conn.id}`); + console.log(`Email: ${conn.email}`); + console.log(`Created: ${conn.createdAt}`); + console.log(`Document limit: ${conn.documentLimit}`); + console.log(`Repositories: ${conn.metadata.repositories?.length || 0}`); + console.log('---'); + }); + ``` + + + ```python + # List all GitHub connections for specific container tags + connections = client.connections.list( + container_tags=['user-123'], + provider='github' + ) + + for conn in connections: + print(f'Provider: {conn.provider}') + print(f'ID: {conn.id}') + print(f'Email: {conn.email}') + print(f'Created: {conn.created_at}') + print(f'Document limit: {conn.document_limit}') + print(f'Repositories: {len(conn.metadata.get("repositories", []))}') + print('---') + ``` + + + ```bash + # List all GitHub connections for specific container tags + curl -X POST "https://api.supermemory.ai/v3/connections/list" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "containerTags": ["user-123"], + "provider": "github" + }' + ``` + + + +### Update Repository Configuration + +You can update which repositories are synced at any time: + + + + ```typescript + // Add or remove repositories + await client.connections.github.configure(connectionId, { + repositories: [ + { + id: 123456789, + name: 'your-org/documentation', + defaultBranch: 'main' + }, + { + id: 987654321, + name: 'your-org/new-repo', + defaultBranch: 'develop' // Can specify different branch + } + ] + }); + + console.log('Repository configuration updated'); + ``` + + + ```python + # Add or remove repositories + client.connections.github.configure( + connection_id, + repositories=[ + { + 'id': 123456789, + 'name': 'your-org/documentation', + 'defaultBranch': 'main' + }, + { + 'id': 987654321, + 'name': 'your-org/new-repo', + 'defaultBranch': 'develop' # Can specify different branch + } + ] + ) + + print('Repository configuration updated') + ``` + + + ```bash + # Update repository configuration + curl -X POST "https://api.supermemory.ai/v3/connections/{connectionId}/github/configure" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "repositories": [ + { + "id": 123456789, + "name": "your-org/documentation", + "defaultBranch": "main" + }, + { + "id": 987654321, + "name": "your-org/new-repo", + "defaultBranch": "develop" + } + ] + }' + ``` + + + + +When you update the repository configuration: +- New repositories are added and synced immediately +- Removed repositories have their webhooks deleted +- Existing documents from removed repositories remain in Supermemory unless you delete them manually + + +### Delete Connection + + + + ```typescript + // Delete by connection ID + const result = await client.connections.delete(connectionId); + console.log('Deleted connection:', result.id); + ``` + + + ```python + # Delete by connection ID + result = client.connections.delete(connection_id) + print(f'Deleted connection: {result.id}') + ``` + + + ```bash + # Delete by connection ID + curl -X DELETE "https://api.supermemory.ai/v3/connections/{connectionId}" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" + ``` + + + + +Deleting a GitHub connection will: +- Stop all future syncs from configured repositories +- Remove all webhooks from the repositories +- Revoke the OAuth authorization +- **Permanently delete all synced documents** from your Supermemory knowledge base + + +### Manual Sync + +Trigger a manual synchronization for all configured repositories: + + + + ```typescript + // Trigger sync for GitHub connections + await client.connections.import('github'); + + // Trigger sync for specific container tags + await client.connections.import('github', { + containerTags: ['user-123'] + }); + + console.log('Manual sync initiated'); + ``` + + + ```python + # Trigger sync for GitHub connections + client.connections.import_('github') + + # Trigger sync for specific container tags + client.connections.import_( + 'github', + container_tags=['user-123'] + ) + + print('Manual sync initiated') + ``` + + + ```bash + # Trigger sync for all GitHub connections + curl -X POST "https://api.supermemory.ai/v3/connections/github/import" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" + + # Trigger sync for specific container tags + curl -X POST "https://api.supermemory.ai/v3/connections/github/import" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "containerTags": ["user-123"] + }' + + # Response: {"message": "Manual sync initiated", "provider": "github"} + ``` + + + +## Advanced Configuration + +### Custom OAuth Application + +For white-label deployments or custom branding, configure your own GitHub OAuth app using the settings API: + + + + ```typescript + // Update organization settings with your GitHub OAuth app + await client.settings.update({ + githubCustomKeyEnabled: true, + githubClientId: 'Iv1.1234567890abcdef', + githubClientSecret: 'your-github-client-secret' + }); + + // Get current settings + const settings = await client.settings.get(); + console.log('GitHub custom key enabled:', settings.githubCustomKeyEnabled); + console.log('Client ID configured:', !!settings.githubClientId); + ``` + + + ```python + # Update organization settings with your GitHub OAuth app + client.settings.update( + github_custom_key_enabled=True, + github_client_id='Iv1.1234567890abcdef', + github_client_secret='your-github-client-secret' + ) + + # Get current settings + settings = client.settings.get() + print(f'GitHub custom key enabled: {settings.github_custom_key_enabled}') + print(f'Client ID configured: {bool(settings.github_client_id)}') + ``` + + + ```bash + # Update organization settings + curl -X PATCH "https://api.supermemory.ai/v3/settings" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "githubCustomKeyEnabled": true, + "githubClientId": "Iv1.1234567890abcdef", + "githubClientSecret": "your-github-client-secret" + }' + + # Get current settings + curl -X GET "https://api.supermemory.ai/v3/settings" \ + -H "Authorization: Bearer $SUPERMEMORY_API_KEY" + ``` + + + + +**Setting up a GitHub OAuth App:** + +1. Go to GitHub Settings → Developer settings → OAuth Apps +2. Click "New OAuth App" +3. Set Authorization callback URL to: `https://api.supermemory.ai/v3/connections/auth/callback/github` +4. Copy the Client ID and generate a Client Secret +5. Configure them in Supermemory using the settings API above + +After configuration, all new GitHub connections will use your custom OAuth app instead of Supermemory's default app. + diff --git a/apps/docs/connectors/overview.mdx b/apps/docs/connectors/overview.mdx index af9eaef9..c1428c10 100644 --- a/apps/docs/connectors/overview.mdx +++ b/apps/docs/connectors/overview.mdx @@ -1,10 +1,10 @@ --- title: "Connectors Overview" -description: "Integrate Google Drive, Notion, OneDrive, and Web Crawler to automatically sync documents into your knowledge base" +description: "Integrate Google Drive, Notion, OneDrive, GitHub and Web Crawler to automatically sync documents into your knowledge base" sidebarTitle: "Overview" --- -Connect external platforms to automatically sync documents into Supermemory. Supported connectors include Google Drive, Notion, OneDrive, and Web Crawler with real-time synchronization and intelligent content processing. +Connect external platforms to automatically sync documents into Supermemory. Supported connectors include Google Drive, Notion, OneDrive, GitHub and Web Crawler with real-time synchronization and intelligent content processing. ## Supported Connectors @@ -26,6 +26,13 @@ Connect external platforms to automatically sync documents into Supermemory. Sup Scheduled sync every 4 hours. Supports personal and business accounts with file versioning. + + + + **GitHub Repositories** + + Real-time incremental sync via webhooks. Supports documentation files in repositories. + **Web Pages, Documentation** @@ -212,6 +219,7 @@ graph TD | **Google Drive** | ✅ Webhooks (7-day expiry) | ✅ Every 4 hours | ✅ On-demand | | **Notion** | ✅ Webhooks | ✅ Every 4 hours | ✅ On-demand | | **OneDrive** | ✅ Webhooks (30-day expiry) | ✅ Every 4 hours | ✅ On-demand | +| **GitHub** | ✅ Webhooks | ✅ Every 4 hours | ✅ On-demand | | **Web Crawler** | ❌ Not supported | ✅ Scheduled recrawling (7+ days) | ✅ On-demand | diff --git a/apps/docs/docs.json b/apps/docs/docs.json index 347d6df5..e2d1f202 100644 --- a/apps/docs/docs.json +++ b/apps/docs/docs.json @@ -138,6 +138,7 @@ "connectors/notion", "connectors/google-drive", "connectors/onedrive", + "connectors/github", "connectors/web-crawler", "connectors/troubleshooting" ]