Skip to content

Commit 015948a

Browse files
waleedlatif1waleedlatif1
committed
feat(search): add Gmail, Jira, GitHub and Calendar sources
1 parent fd6fc44 commit 015948a

70 files changed

Lines changed: 3907 additions & 354 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,18 @@ One app registration in [Entra ID](https://entra.microsoft.com) covers all of th
114114

115115
The same variables also power "Sign in with Microsoft".
116116

117+
### GitHub Search
118+
119+
Register a [GitHub App](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app) with repository **Contents: read-only**, **Metadata: read-only**, and account **Email addresses: read-only** permissions. Keep user access token expiration enabled so Sim can rotate access and refresh tokens.
120+
121+
| Environment variables | Provider ID |
122+
|---|---|
123+
| `GITHUB_APP_CLIENT_ID`<br />`GITHUB_APP_CLIENT_SECRET` | `github-repositories` |
124+
125+
Register `https://<your-domain>/api/auth/oauth2/callback/github-repositories` as the callback. These App OAuth client credentials are separate from `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` used for Sim sign-in. Sim does not require an App private key.
126+
127+
A repository or organization administrator installs the App on the repositories to search. Each member connects their own GitHub account, with a verified email matching their Sim account. Search indexes repository files that both the member and the installed App can access. GitHub workflow blocks and existing knowledge-base token connections continue to use personal access tokens.
128+
117129
### Everything else
118130

119131
| Service | Environment variables | Provider ID |

‎apps/docs/openapi-v2-resources.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8048,6 +8048,7 @@
80488048
"providerId": {
80498049
"type": "string",
80508050
"enum": [
8051+
"github-repositories",
80518052
"google-email",
80528053
"google-drive",
80538054
"google-docs",

‎apps/sim/app/api/settings/allowed-integrations/route.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { createMockRequest } from '@sim/testing'
5+
import { beforeEach, describe, expect, it, vi } from 'vitest'
6+
7+
const mocks = vi.hoisted(() => ({
8+
getSession: vi.fn(),
9+
getIntegrationAvailability: vi.fn(),
10+
getOAuthServiceAvailability: vi.fn(),
11+
getAllOAuthServices: vi.fn(),
12+
}))
13+
vi.mock('@/lib/auth', () => ({ getSession: mocks.getSession }))
14+
vi.mock('@/lib/core/config/env-flags', () => ({ getAllowedIntegrationsFromEnv: () => null }))
15+
vi.mock('@/lib/integrations/availability.server', () => ({
16+
getIntegrationAvailability: mocks.getIntegrationAvailability,
17+
getOAuthServiceAvailability: mocks.getOAuthServiceAvailability,
18+
}))
19+
vi.mock('@/lib/oauth/utils', () => ({ getAllOAuthServices: mocks.getAllOAuthServices }))
20+
21+
import { getAllowedIntegrationsContract } from '@/lib/api/contracts/common'
22+
import { GET } from '@/app/api/settings/allowed-integrations/route'
23+
24+
describe('allowed integrations response', () => {
25+
beforeEach(() => {
26+
vi.clearAllMocks()
27+
mocks.getSession.mockResolvedValue({ user: { id: 'user-1' } })
28+
mocks.getIntegrationAvailability.mockReturnValue([
29+
{ type: 'github_v2', state: 'ready', oauthAvailable: false, missingFields: [] },
30+
])
31+
mocks.getAllOAuthServices.mockReturnValue([
32+
{ providerId: 'github-repositories', authType: 'oauth' },
33+
])
34+
mocks.getOAuthServiceAvailability.mockReturnValue([
35+
{ providerId: 'github-repositories', available: false },
36+
])
37+
})
38+
39+
it('authenticates before projecting deployment capabilities', async () => {
40+
mocks.getSession.mockResolvedValue(null)
41+
const response = await GET(
42+
createMockRequest(
43+
'GET',
44+
undefined,
45+
undefined,
46+
'http://localhost/api/settings/allowed-integrations'
47+
),
48+
{}
49+
)
50+
expect(response.status).toBe(401)
51+
expect(mocks.getIntegrationAvailability).not.toHaveBeenCalled()
52+
expect(mocks.getOAuthServiceAvailability).not.toHaveBeenCalled()
53+
expect(mocks.getAllOAuthServices).not.toHaveBeenCalled()
54+
})
55+
56+
it('returns block and OAuth service readiness as distinct contract fields', async () => {
57+
const response = await GET(
58+
createMockRequest(
59+
'GET',
60+
undefined,
61+
undefined,
62+
'http://localhost/api/settings/allowed-integrations'
63+
),
64+
{}
65+
)
66+
expect(response.status).toBe(200)
67+
const body = await response.json()
68+
expect(getAllowedIntegrationsContract.response.schema.safeParse(body).success).toBe(true)
69+
expect(body).toEqual({
70+
allowedIntegrations: null,
71+
integrationAvailability: [{ type: 'github_v2', state: 'ready', oauthAvailable: false }],
72+
oauthServiceAvailability: [{ providerId: 'github-repositories', available: false }],
73+
})
74+
expect(mocks.getOAuthServiceAvailability).toHaveBeenCalledWith(
75+
mocks.getAllOAuthServices.mock.results[0].value
76+
)
77+
})
78+
})

‎apps/sim/app/api/settings/allowed-integrations/route.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ import { NextResponse } from 'next/server'
22
import { getSession } from '@/lib/auth'
33
import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags'
44
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
5-
import { getIntegrationAvailability } from '@/lib/integrations/availability.server'
5+
import {
6+
getIntegrationAvailability,
7+
getOAuthServiceAvailability,
8+
} from '@/lib/integrations/availability.server'
9+
import { getAllOAuthServices } from '@/lib/oauth/utils'
610

711
export const GET = withRouteHandler(async () => {
812
const session = await getSession()
@@ -15,5 +19,6 @@ export const GET = withRouteHandler(async () => {
1519
integrationAvailability: getIntegrationAvailability().map(
1620
({ type, state, oauthAvailable }) => ({ type, state, oauthAvailable })
1721
),
22+
oauthServiceAvailability: getOAuthServiceAvailability(getAllOAuthServices()),
1823
})
1924
})

‎apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.test.tsx‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,27 @@ vi.mock('@/hooks/queries/workspace', () => ({
1717
useWorkspacePermissionsQuery: () => ({ data: { viewer: { isAdmin: mocks.admin() } } }),
1818
}))
1919
vi.mock('@/hooks/use-permission-config', () => ({
20-
usePermissionConfig: () => ({ integrationAvailability: new Map() }),
20+
usePermissionConfig: () => ({
21+
integrationAvailability: new Map([
22+
['slack', { oauthAvailable: true, state: 'ready' }],
23+
['slack_v2', { oauthAvailable: true, state: 'ready' }],
24+
]),
25+
oauthServiceAvailability: new Map(
26+
[
27+
'confluence',
28+
'google-drive',
29+
'google_drive',
30+
'google-email',
31+
'google-calendar',
32+
'jira',
33+
'github-repositories',
34+
].map((providerId) => [providerId, true])
35+
),
36+
isIntegrationAvailabilityReady: true,
37+
isIntegrationAvailabilityLoading: false,
38+
integrationAvailabilityError: null,
39+
refetchIntegrationAvailability: vi.fn(),
40+
}),
2141
}))
2242
vi.mock('@/hooks/queries/kb/connectors', () => ({
2343
useWorkspaceMemberConnectors: () => ({ data: mocks.rows() }),

‎apps/sim/app/workspace/[workspaceId]/home/components/search-sources/search-sources.tsx‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,8 @@ interface SearchSourcesProps {
131131
* as workspace connectors do not appear here.
132132
*/
133133
export function SearchSources({ workspaceId }: SearchSourcesProps) {
134-
const { integrationAvailability } = usePermissionConfig()
134+
const { integrationAvailability, oauthServiceAvailability, isIntegrationAvailabilityReady } =
135+
usePermissionConfig()
135136
/** With per-member access off, a connect is refused, so the chips say so instead. */
136137
const memberAccessAvailable = useMemberAccessAvailable()
137138
const { data: workspacePermissions } = useWorkspacePermissionsQuery(workspaceId)
@@ -191,7 +192,13 @@ export function SearchSources({ workspaceId }: SearchSourcesProps) {
191192
unavailableReason={searchConnectorUnavailableReason(
192193
connector,
193194
integrationAvailability,
194-
{ memberAccessAvailable, hasConnection: connection !== undefined, canCreate }
195+
{
196+
memberAccessAvailable,
197+
hasConnection: connection !== undefined,
198+
canCreate,
199+
oauthServiceAvailability,
200+
isIntegrationAvailabilityReady,
201+
}
195202
)}
196203
waiting={
197204
connection ? isAwaiting(connection.connectorId) : isAwaitingSource(connector.type)

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,27 @@ vi.mock('@/hooks/use-member-access', () => ({
3636
useMemberAccessAvailable: () => mocks.memberAccess,
3737
}))
3838
vi.mock('@/hooks/use-permission-config', () => ({
39-
usePermissionConfig: () => ({ integrationAvailability: new Map() }),
39+
usePermissionConfig: () => ({
40+
integrationAvailability: new Map([
41+
['slack', { oauthAvailable: true, state: 'ready' }],
42+
['slack_v2', { oauthAvailable: true, state: 'ready' }],
43+
]),
44+
oauthServiceAvailability: new Map(
45+
[
46+
'confluence',
47+
'google-drive',
48+
'google_drive',
49+
'google-email',
50+
'google-calendar',
51+
'jira',
52+
'github-repositories',
53+
].map((providerId) => [providerId, true])
54+
),
55+
isIntegrationAvailabilityReady: true,
56+
isIntegrationAvailabilityLoading: false,
57+
integrationAvailabilityError: null,
58+
refetchIntegrationAvailability: vi.fn(),
59+
}),
4060
}))
4161
vi.mock('@/hooks/queries/kb/connectors', () => ({
4262
useCreateConnector: () => ({ mutate: mocks.create, isPending: false }),

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx‎

Lines changed: 67 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,13 @@ import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/component
5353
import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields'
5454
import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider'
5555
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
56-
import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state'
56+
import {
57+
SettingsEmptyState,
58+
SettingsQueryErrorState,
59+
} from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state'
5760
import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row'
5861
import { withBrandIcon } from '@/blocks/brand-icon'
62+
import { getConnectorApiKeyConfig } from '@/connectors/auth'
5963
import { CONNECTOR_META_REGISTRY } from '@/connectors/registry'
6064
import type { ConnectorMeta } from '@/connectors/types'
6165
import { useWorkspaceAccounts } from '@/hooks/queries/credential-groups'
@@ -137,6 +141,7 @@ export function AddConnectorModal({
137141
const [showServiceAccountModal, setShowServiceAccountModal] = useState(false)
138142

139143
const [apiKeyValue, setApiKeyValue] = useState('')
144+
const [useApiKey, setUseApiKey] = useState(!isSearchIndex)
140145
const [apiKeyFocused, setApiKeyFocused] = useState(false)
141146
const [searchTerm, setSearchTerm] = useState('')
142147

@@ -154,13 +159,25 @@ export function AddConnectorModal({
154159
const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling)
155160

156161
const connectorConfig = selectedType ? CONNECTOR_META_REGISTRY[selectedType] : null
157-
const isApiKeyMode = connectorConfig?.auth.mode === 'apiKey'
158162
const isMembersMode = access.accessMode === 'members'
159-
const { integrationAvailability } = usePermissionConfig()
163+
const apiKeyConfig = connectorConfig ? getConnectorApiKeyConfig(connectorConfig.auth) : undefined
164+
const isApiKeyMode =
165+
connectorConfig?.auth.mode === 'apiKey' || Boolean(apiKeyConfig && !isMembersMode && useApiKey)
166+
const {
167+
integrationAvailability,
168+
oauthServiceAvailability,
169+
isIntegrationAvailabilityReady,
170+
isIntegrationAvailabilityFetching,
171+
isIntegrationAvailabilityLoading,
172+
integrationAvailabilityError,
173+
refetchIntegrationAvailability,
174+
} = usePermissionConfig()
160175
const { admin: allowAdmin, members: allowMembers } = connectorConfig
161176
? getConnectorAccessAvailability(connectorConfig, integrationAvailability, {
162177
memberAccessAvailable,
163178
mirroredAccessAvailable,
179+
oauthServiceAvailability,
180+
isIntegrationAvailabilityReady,
164181
})
165182
: { admin: false, members: false }
166183
const needsSlackSetup = selectedType === 'slack' && isMembersMode
@@ -180,13 +197,10 @@ export function AddConnectorModal({
180197
/** True when the connector declares its key optional (public sources need none). */
181198
const isApiKeyOptional =
182199
connectorConfig?.auth.mode === 'apiKey' && connectorConfig.auth.optional === true
183-
const connectorProviderId = useMemo(
184-
() =>
185-
connectorConfig && connectorConfig.auth.mode === 'oauth'
186-
? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider)
187-
: null,
188-
[connectorConfig]
189-
)
200+
const connectorProviderId =
201+
connectorConfig?.auth.mode === 'oauth'
202+
? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider)
203+
: null
190204

191205
const serviceAccountProviderId = connectorProviderId
192206
? getServiceAccountProviderForProviderId(connectorProviderId)
@@ -236,10 +250,18 @@ export function AddConnectorModal({
236250
resolveSourceConfig,
237251
} = useConnectorConfigFields({
238252
connectorConfig,
253+
accessMode: access.accessMode,
239254
initialSourceConfig: draft?.sourceConfig,
240255
initialCanonicalModes: draft?.canonicalModes,
241256
})
242257

258+
const showCredentialPicker =
259+
!isMembersMode ||
260+
connectorConfig?.supportsSeparateContentCredential ||
261+
connectorConfig?.configFields.some(
262+
(field) => field.type === 'selector' && isFieldVisible(field)
263+
)
264+
243265
const saveSetup = () => {
244266
if (!setupDraftKey) return
245267
useConnectorSetupStore.getState().saveDraft(setupDraftKey, {
@@ -272,6 +294,7 @@ export function AddConnectorModal({
272294
: WORKSPACE_ACCESS
273295
)
274296
setApiKeyValue('')
297+
setUseApiKey(!isSearchIndex)
275298
setApiKeyFocused(false)
276299
setDisabledTagIds(new Set())
277300
setShowMetadata(false)
@@ -442,6 +465,17 @@ export function AddConnectorModal({
442465
</div>
443466
) : connectorConfig ? (
444467
<>
468+
{integrationAvailabilityError && (
469+
<ChipModalField type='custom' title='Connection availability'>
470+
<SettingsQueryErrorState
471+
error={integrationAvailabilityError}
472+
isRetrying={isIntegrationAvailabilityFetching}
473+
fallback='Could not load connection availability'
474+
onRetry={() => void refetchIntegrationAvailability()}
475+
variant='inline'
476+
/>
477+
</ChipModalField>
478+
)}
445479
{(memberAccessAvailable || mirroredAccessAvailable || slackSetupRequired) && (
446480
<ConnectorAccessField
447481
workspaceId={workspaceId}
@@ -462,30 +496,36 @@ export function AddConnectorModal({
462496

463497
{!slackSetupRequired && (
464498
<>
499+
{connectorConfig.auth.mode === 'oauth' && apiKeyConfig && !isMembersMode && (
500+
<ChipModalField type='custom' title='Authentication'>
501+
<ChipCombobox
502+
disabled={isCreating}
503+
value={isApiKeyMode ? 'apiKey' : 'oauth'}
504+
options={[
505+
{ label: apiKeyConfig.label || 'API key', value: 'apiKey' },
506+
{ label: 'Connected account', value: 'oauth' },
507+
]}
508+
onChange={(value) => {
509+
setUseApiKey(value === 'apiKey')
510+
setApiKeyValue('')
511+
setSelectedCredentialId(null)
512+
}}
513+
/>
514+
</ChipModalField>
515+
)}
465516
{isApiKeyMode ? (
466-
<ChipModalField
467-
type='custom'
468-
title={
469-
connectorConfig.auth.mode === 'apiKey' && connectorConfig.auth.label
470-
? connectorConfig.auth.label
471-
: 'API Key'
472-
}
473-
>
517+
<ChipModalField type='custom' title={apiKeyConfig?.label || 'API Key'}>
474518
<ChipInput
475519
type={apiKeyFocused ? 'text' : 'password'}
476520
autoComplete='new-password'
477521
value={apiKeyValue}
478522
onChange={(e) => setApiKeyValue(e.target.value)}
479523
onFocus={() => setApiKeyFocused(true)}
480524
onBlur={() => setApiKeyFocused(false)}
481-
placeholder={
482-
connectorConfig.auth.mode === 'apiKey' && connectorConfig.auth.placeholder
483-
? connectorConfig.auth.placeholder
484-
: 'Enter API key'
485-
}
525+
placeholder={apiKeyConfig?.placeholder || 'Enter API key'}
486526
/>
487527
</ChipModalField>
488-
) : (
528+
) : showCredentialPicker ? (
489529
<ChipModalField
490530
type='custom'
491531
title={isMembersMode ? 'Browse with' : 'Account'}
@@ -529,10 +569,11 @@ export function AddConnectorModal({
529569
if (isOpen) void refetchCredentials()
530570
}}
531571
placeholder={`Select ${connectorConfig.name} account`}
532-
isLoading={credentialsLoading}
572+
isLoading={credentialsLoading || isIntegrationAvailabilityLoading}
573+
disabled={!isIntegrationAvailabilityReady}
533574
/>
534575
</ChipModalField>
535-
)}
576+
) : null}
536577

537578
{isMembersMode && connectorConfig.supportsSeparateContentCredential && (
538579
<ConnectorContentCredentialField

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,10 @@ interface ConnectorAccessFieldProps {
8383
function accessHint(mode: ConnectorAccessMode, connectorConfig: ConnectorMeta): string {
8484
const sourceName = connectorConfig.name
8585
if (mode === 'members') {
86-
return `Each teammate connects their ${sourceName} account. They see only documents they can open there.`
86+
return (
87+
connectorConfig.memberSetupHint ??
88+
`Each teammate connects their ${sourceName} account. They see only documents they can open there.`
89+
)
8790
}
8891
if (mode === 'admin') {
8992
const identityHint = connectorConfig.requiresMemberIdentity

0 commit comments

Comments
 (0)