diff --git a/.env b/.env index 870fe03e..7fc7d30e 100644 --- a/.env +++ b/.env @@ -25,7 +25,7 @@ FAROS_AIRBYTE_FORCE_SETUP=false FAROS_INIT_IMAGE=farosai/faros-ce-init:latest ############################## Airbyte ######################################## -VERSION=0.39.37-alpha +VERSION=0.40.9 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_USER=docker DATABASE_PASSWORD=docker @@ -74,7 +74,7 @@ N8N_DOCKER_MOUNT=n8n_data # # # Contributors - please organise this env file according to the above linked file. -# Source: https://github.com/airbytehq/airbyte/blob/v0.39.37-alpha/.env +# Source: https://github.com/airbytehq/airbyte/blob/v0.40.9/.env ### SHARED ### @@ -125,6 +125,10 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= ### LOGGING/MONITORING/TRACKING ### TRACKING_STRATEGY=logging @@ -135,6 +139,7 @@ LOG_LEVEL=INFO ### APPLICATIONS ### # Worker # +WORKERS_MICRONAUT_ENVIRONMENTS=control # Relevant to scaling. MAX_SYNC_WORKERS=5 MAX_SPEC_WORKERS=5 @@ -158,13 +163,15 @@ METRIC_CLIENT= # Useful only when metric client is set to be otel. Must start with http:// or https://. OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317" -USE_STREAM_CAPABLE_STATE=false +USE_STREAM_CAPABLE_STATE=true -# extra settings to limit warnings during CE startup -PAPERCUPS_STORYTIME=disabled -RUN_DATABASE_MIGRATION_ON_STARTUP=true -SECRET_PERSISTENCE=NONE -WORKER_ENVIRONMENT=docker -NEW_SCHEDULER= +# extra settings to limit warnings during CE setup +RUN_DATABASE_MIGRATION_ON_STARTUP=true +SECRET_PERSISTENCE=NONE +WORKER_ENVIRONMENT=docker +NEW_SCHEDULER= DEPLOYMENT_MODE= +JOB_ERROR_REPORTING_STRATEGY= JOB_ERROR_REPORTING_SENTRY_DSN= +LOG_CONNECTOR_MESSAGES= +TEMPORAL_HISTORY_RETENTION_IN_DAYS= diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4db6e918..4d049069 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -53,9 +53,34 @@ jobs: run: npm run test:unit -- --coverage working-directory: init - # In order to optimize for speed, we use the latest init image. + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: | + farosai/faros-ce-init + flavor: | + latest=auto + tags: | + type=sha,format=long,prefix= + type=raw,value=latest,enable=true + + - name: Set up Docker QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + context: . + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: Start services - run: FAROS_EMAIL=integration.tests@faros.ai FAROS_INIT_IMAGE=farosai/faros-ce-init:latest docker compose up --quiet-pull -d + run: FAROS_EMAIL=integration.tests@faros.ai FAROS_INIT_IMAGE=farosai/faros-ce-init:${{ github.sha }} docker compose up --quiet-pull -d - name: Show logs run: docker compose logs --tail all @@ -69,9 +94,8 @@ jobs: echo "Faros init container exit code was ${CONTAINER_EXIT_CODE}" [[ $CONTAINER_EXIT_CODE -eq 0 ]] - - name: Get destination id & Hasura Admin Secret + - name: Get Hasura Admin Secret run: | - echo "DESTINATION_ID=$(cat ${{ github.workspace }}/init/resources/airbyte/workspace/airbyte_config/DESTINATION_CONNECTION.yaml | yq '.[0].destinationId')" >> "$GITHUB_ENV" echo $(cat .env | grep "^HASURA_GRAPHQL_ADMIN_SECRET") >> "$GITHUB_ENV" - name: Run integration tests (init) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7782aa30..60ec2c00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,9 +89,8 @@ jobs: echo "Faros init container exit code was ${CONTAINER_EXIT_CODE}" [[ $CONTAINER_EXIT_CODE -eq 0 ]] - - name: Get destination id & Hasura Admin Secret + - name: Get Hasura Admin Secret run: | - echo "DESTINATION_ID=$(cat ${{ github.workspace }}/init/resources/airbyte/workspace/airbyte_config/DESTINATION_CONNECTION.yaml | yq '.[0].destinationId')" >> "$GITHUB_ENV" echo $(cat .env | grep "^HASURA_GRAPHQL_ADMIN_SECRET") >> "$GITHUB_ENV" - name: Run integration tests (init) diff --git a/cli/src/airbyte/airbyte-client.ts b/cli/src/airbyte/airbyte-client.ts index 72d22547..82cb681b 100644 --- a/cli/src/airbyte/airbyte-client.ts +++ b/cli/src/airbyte/airbyte-client.ts @@ -42,6 +42,35 @@ export class Airbyte { ); } + async getFirstWorkspace(): Promise { + const response = await this.api.post('/workspaces/list', {}); + return response.data.workspaces[0].workspaceId as string; + } + + async findFarosSource(name: string): Promise { + const workspaceId = await this.getFirstWorkspace(); + const response = await this.api + .post('/sources/list', {workspaceId}) + .catch((err) => { + throw wrapApiError(err, 'Failed to call /sources/list'); + }); + return response.data.sources.filter( + (source: any) => source.name === name + )[0].sourceId; + } + + async findFarosConnection(name: string): Promise { + const workspaceId = await this.getFirstWorkspace(); + const response = await this.api + .post('/connections/list', {workspaceId}) + .catch((err) => { + throw wrapApiError(err, 'Failed to call /connections/list'); + }); + return response.data.connections.filter( + (connection: any) => connection.name === name + )[0].connectionId; + } + async setupSource(config: any): Promise { display('Setting up source %s', Emoji.SETUP); await this.api @@ -158,11 +187,13 @@ export class Airbyte { return `${this.airbyteUrl}/connections/${connectionId}/status`; } - async isActiveConnection(connectionId: string): Promise { + async isActiveConnection(connectionName: string): Promise { + const connection = await this.findFarosConnection(connectionName); + const response = await this.api .post('/jobs/list', { configTypes: ['sync'], - configId: connectionId, + configId: connection, }) .catch((err) => { throw wrapApiError(err, ' Failed to call /jobs/list'); @@ -170,7 +201,8 @@ export class Airbyte { return response.data.jobs[0]?.job.status === 'succeeded'; } - async refresh(connectionId: string, sourceName: string): Promise { + async refresh(connectionName: string, sourceName: string): Promise { + const connectionId = await this.findFarosConnection(connectionName); const job = await this.triggerSync(connectionId); await this.track(job, connectionId, sourceName); } diff --git a/cli/src/bitbucket/run.ts b/cli/src/bitbucket/run.ts index b9e7ba65..5db978c5 100644 --- a/cli/src/bitbucket/run.ts +++ b/cli/src/bitbucket/run.ts @@ -20,8 +20,6 @@ import { runSelect, } from '../utils/prompts'; -const BITBUCKET_SOURCE_ID = '5a19e927-51a2-4d5f-9b26-b35aba0910e0'; -export const BITBUCKET_CONNECTION_ID = '2093cc9f-81d5-47df-8c14-d898c89f4c81'; const DEFAULT_CUTOFF_DAYS = 30; const DEFAULT_API_URL = 'https://api.bitbucket.org/2.0'; @@ -167,6 +165,7 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise { } } + const bitbucketSourceId = await cfg.airbyte.findFarosSource('Bitbucket'); await cfg.airbyte.setupSource({ connectionConfiguration: { serverUrl, @@ -178,15 +177,18 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise { pagelen: 10, }, name: 'Bitbucket', - sourceId: BITBUCKET_SOURCE_ID, + sourceId: bitbucketSourceId, }); } catch (error) { errorLog('Setup failed %s', Emoji.FAILURE, error); return; } + const bitbucketConnectionId = await cfg.airbyte.findFarosConnection( + 'Bitbucket - Faros' + ); await cfg.airbyte.triggerAndTrackSync( - BITBUCKET_CONNECTION_ID, + bitbucketConnectionId, 'Bitbucket', cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, repos?.length || 0 diff --git a/cli/src/github/run.ts b/cli/src/github/run.ts index e4ce40d6..d300c139 100644 --- a/cli/src/github/run.ts +++ b/cli/src/github/run.ts @@ -20,8 +20,6 @@ import { runSelect, } from '../utils/prompts'; -const GITHUB_SOURCE_ID = '5d9079ca-8173-406f-bfdb-41f19c62daff'; -export const GITHUB_CONNECTION_ID = '6421df4e-0c5a-4666-a530-9c01de683518'; const DEFAULT_CUTOFF_DAYS = 30; interface GithubConfig { @@ -125,6 +123,7 @@ export async function runGithub(cfg: GithubConfig): Promise { } } + const githubSourceId = await cfg.airbyte.findFarosSource('GitHub'); await cfg.airbyte.setupSource({ connectionConfiguration: { repository: repos?.join(' '), @@ -136,15 +135,18 @@ export async function runGithub(cfg: GithubConfig): Promise { page_size_for_large_streams: 10, }, name: 'GitHub', - sourceId: GITHUB_SOURCE_ID, + sourceId: githubSourceId, }); } catch (error) { errorLog('Setup failed %s', Emoji.FAILURE, error); return; } + const githubConnectionId = await cfg.airbyte.findFarosConnection( + 'GitHub - Faros' + ); await cfg.airbyte.triggerAndTrackSync( - GITHUB_CONNECTION_ID, + githubConnectionId, 'GitHub', cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, repos?.length || 0 diff --git a/cli/src/gitlab/run.ts b/cli/src/gitlab/run.ts index a9cd033b..c36ac562 100644 --- a/cli/src/gitlab/run.ts +++ b/cli/src/gitlab/run.ts @@ -21,8 +21,6 @@ import { runSelect, } from '../utils/prompts'; -const GITLAB_SOURCE_ID = '59c74ca4-8cbb-4c65-8cb7-66bf771190fb'; -export const GITLAB_CONNECTION_ID = 'cef1b90d-ab16-4645-a0e3-b81818b8ffc7'; const DEFAULT_CUTOFF_DAYS = 30; const DEFAULT_API_URL = 'gitlab.com'; @@ -134,6 +132,7 @@ export async function runGitlab(cfg: GitLabConfig): Promise { } } + const gitlabSourceId = await cfg.airbyte.findFarosSource('GitLab'); await cfg.airbyte.setupSource({ connectionConfiguration: { api_url, @@ -142,15 +141,18 @@ export async function runGitlab(cfg: GitLabConfig): Promise { private_token: token, }, name: 'GitLab', - sourceId: GITLAB_SOURCE_ID, + sourceId: gitlabSourceId, }); } catch (error) { errorLog('Setup failed %s', Emoji.FAILURE, error); return; } + const gitlabConnectionId = await cfg.airbyte.findFarosConnection( + 'GitLab - Faros' + ); await cfg.airbyte.triggerAndTrackSync( - GITLAB_CONNECTION_ID, + gitlabConnectionId, 'GitLab', cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, projects?.length || 0 diff --git a/cli/src/jira/run.ts b/cli/src/jira/run.ts index 7c1cca2c..77751616 100644 --- a/cli/src/jira/run.ts +++ b/cli/src/jira/run.ts @@ -21,8 +21,6 @@ import { runSelect, } from '../utils/prompts'; -const JIRA_SOURCE_ID = '22852029-670c-4296-958e-c581fa76ae98'; -export const JIRA_CONNECTION_ID = '577ceecf-a92a-4785-b385-c41112d7f537'; const DEFAULT_CUTOFF_DAYS = 30; interface JiraConfig { @@ -154,6 +152,7 @@ export async function runJira(cfg: JiraConfig): Promise { } } + const jiraSourceId = await cfg.airbyte.findFarosSource('Jira'); await cfg.airbyte.setupSource({ connectionConfiguration: { email, @@ -166,15 +165,18 @@ export async function runJira(cfg: JiraConfig): Promise { expand_issue_changelog: true, }, name: 'Jira', - sourceId: JIRA_SOURCE_ID, + sourceId: jiraSourceId, }); } catch (error) { errorLog('Setup failed %s', Emoji.FAILURE, error); return; } + const jiraConnectionId = await cfg.airbyte.findFarosConnection( + 'Jira - Faros' + ); await cfg.airbyte.triggerAndTrackSync( - JIRA_CONNECTION_ID, + jiraConnectionId, 'Jira', cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, projects?.length || 0 diff --git a/cli/src/refresh/run.ts b/cli/src/refresh/run.ts index e503cd48..83676900 100644 --- a/cli/src/refresh/run.ts +++ b/cli/src/refresh/run.ts @@ -1,10 +1,6 @@ import {Command} from 'commander'; import {Airbyte} from '../airbyte/airbyte-client'; -import {BITBUCKET_CONNECTION_ID} from '../bitbucket/run'; -import {GITHUB_CONNECTION_ID} from '../github/run'; -import {GITLAB_CONNECTION_ID} from '../gitlab/run'; -import {JIRA_CONNECTION_ID} from '../jira/run'; import {Metabase} from '../metabase/metabase-client'; import {display, Emoji} from '../utils'; @@ -34,24 +30,24 @@ export async function runRefresh(cfg: RefreshConfig): Promise { await cfg.airbyte.waitUntilHealthy(); const work = []; - if (await cfg.airbyte.isActiveConnection(GITHUB_CONNECTION_ID)) { + if (await cfg.airbyte.isActiveConnection('GitHub - Faros')) { display('refreshing GitHub %s', Emoji.SYNC); - work.push(cfg.airbyte.refresh(GITHUB_CONNECTION_ID, 'GitHub')); + work.push(cfg.airbyte.refresh('GitHub - Faros', 'GitHub')); } - if (await cfg.airbyte.isActiveConnection(GITLAB_CONNECTION_ID)) { + if (await cfg.airbyte.isActiveConnection('GitLab - Faros')) { display('refreshing GitLab %s', Emoji.SYNC); - work.push(cfg.airbyte.refresh(GITLAB_CONNECTION_ID, 'GitLab')); + work.push(cfg.airbyte.refresh('GitLab - Faros', 'GitLab')); } - if (await cfg.airbyte.isActiveConnection(BITBUCKET_CONNECTION_ID)) { + if (await cfg.airbyte.isActiveConnection('Bitbucket - Faros')) { display('refreshing Bitbucket %s', Emoji.SYNC); - work.push(cfg.airbyte.refresh(BITBUCKET_CONNECTION_ID, 'Bitbucket')); + work.push(cfg.airbyte.refresh('Bitbucket - Faros', 'Bitbucket')); } - if (await cfg.airbyte.isActiveConnection(JIRA_CONNECTION_ID)) { + if (await cfg.airbyte.isActiveConnection('Jira - Faros')) { display('refreshing Jira %s', Emoji.SYNC); - work.push(cfg.airbyte.refresh(JIRA_CONNECTION_ID, 'Jira')); + work.push(cfg.airbyte.refresh('Jira - Faros', 'Jira')); } if (work.length === 0) { diff --git a/docker-compose.yaml b/docker-compose.yaml index 568c49f3..1ae66fd2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -92,8 +92,6 @@ services: - POSTGRES_USER=${DATABASE_USER} volumes: - db:/var/lib/postgresql/data - ports: - - ${DATABASE_PORT?}:5432 worker: profiles: ["airbyte"] image: airbyte/worker:${VERSION} @@ -112,6 +110,7 @@ services: - DATABASE_URL=${DATABASE_URL} - DATABASE_USER=${DATABASE_USER} - DEPLOYMENT_MODE=${DEPLOYMENT_MODE} + - INTERNAL_API_HOST=${INTERNAL_API_HOST} - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-} - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT} - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST} @@ -120,10 +119,15 @@ services: - LOCAL_DOCKER_MOUNT=${LOCAL_DOCKER_MOUNT} - LOCAL_ROOT=${LOCAL_ROOT} - LOG_LEVEL=${LOG_LEVEL} + - LOG_CONNECTOR_MESSAGES=${LOG_CONNECTOR_MESSAGES} - MAX_CHECK_WORKERS=${MAX_CHECK_WORKERS} - MAX_DISCOVER_WORKERS=${MAX_DISCOVER_WORKERS} - MAX_SPEC_WORKERS=${MAX_SPEC_WORKERS} - MAX_SYNC_WORKERS=${MAX_SYNC_WORKERS} + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT} + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST} + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT} + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST} - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS} - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS} @@ -142,10 +146,13 @@ services: - ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=${ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS} - WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=${WORKFLOW_FAILURE_RESTART_DELAY_SECONDS} - USE_STREAM_CAPABLE_STATE=${USE_STREAM_CAPABLE_STATE} + - MICRONAUT_ENVIRONMENTS=${WORKERS_MICRONAUT_ENVIRONMENTS} volumes: - /var/run/docker.sock:/var/run/docker.sock - workspace:${WORKSPACE_ROOT} - ${LOCAL_ROOT}:${LOCAL_ROOT} + ports: + - 9000:9000 server: profiles: ["airbyte"] image: airbyte/server:${VERSION} @@ -173,6 +180,8 @@ services: - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - TEMPORAL_HOST=${TEMPORAL_HOST} - TRACKING_STRATEGY=${TRACKING_STRATEGY} + - JOB_ERROR_REPORTING_STRATEGY=${JOB_ERROR_REPORTING_STRATEGY} + - JOB_ERROR_REPORTING_SENTRY_DSN=${JOB_ERROR_REPORTING_SENTRY_DSN} - WEBAPP_URL=${WEBAPP_URL} - WORKER_ENVIRONMENT=${WORKER_ENVIRONMENT} - WORKSPACE_ROOT=${WORKSPACE_ROOT} @@ -194,9 +203,7 @@ services: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} - API_URL=${API_URL:-} - - FULLSTORY=${FULLSTORY:-} - INTERNAL_API_HOST=${INTERNAL_API_HOST} - - IS_DEMO=${IS_DEMO:-} - OPENREPLAY=${OPENREPLAY:-} - PAPERCUPS_STORYTIME=${PAPERCUPS_STORYTIME:-} - TRACKING_STRATEGY=${TRACKING_STRATEGY} @@ -218,6 +225,23 @@ services: - POSTGRES_USER=${DATABASE_USER} volumes: - ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig + airbyte-cron: + profiles: ["airbyte"] + image: airbyte/cron:${VERSION} + logging: *default-logging + container_name: airbyte-cron + restart: unless-stopped + environment: + - DB=postgresql + - DB_PORT=${DATABASE_PORT} + - LOG_LEVEL=${LOG_LEVEL} + - POSTGRES_PWD=${DATABASE_PASSWORD} + - POSTGRES_SEEDS=${DATABASE_HOST} + - POSTGRES_USER=${DATABASE_USER} + - TEMPORAL_HISTORY_RETENTION_IN_DAYS=${TEMPORAL_HISTORY_RETENTION_IN_DAYS} + - WORKSPACE_ROOT=${WORKSPACE_ROOT} + volumes: + - workspace:${WORKSPACE_ROOT} # Other services hasura: diff --git a/docker-extension/.env b/docker-extension/.env index 870fe03e..7fc7d30e 100644 --- a/docker-extension/.env +++ b/docker-extension/.env @@ -25,7 +25,7 @@ FAROS_AIRBYTE_FORCE_SETUP=false FAROS_INIT_IMAGE=farosai/faros-ce-init:latest ############################## Airbyte ######################################## -VERSION=0.39.37-alpha +VERSION=0.40.9 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_USER=docker DATABASE_PASSWORD=docker @@ -74,7 +74,7 @@ N8N_DOCKER_MOUNT=n8n_data # # # Contributors - please organise this env file according to the above linked file. -# Source: https://github.com/airbytehq/airbyte/blob/v0.39.37-alpha/.env +# Source: https://github.com/airbytehq/airbyte/blob/v0.40.9/.env ### SHARED ### @@ -125,6 +125,10 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= ### LOGGING/MONITORING/TRACKING ### TRACKING_STRATEGY=logging @@ -135,6 +139,7 @@ LOG_LEVEL=INFO ### APPLICATIONS ### # Worker # +WORKERS_MICRONAUT_ENVIRONMENTS=control # Relevant to scaling. MAX_SYNC_WORKERS=5 MAX_SPEC_WORKERS=5 @@ -158,13 +163,15 @@ METRIC_CLIENT= # Useful only when metric client is set to be otel. Must start with http:// or https://. OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317" -USE_STREAM_CAPABLE_STATE=false +USE_STREAM_CAPABLE_STATE=true -# extra settings to limit warnings during CE startup -PAPERCUPS_STORYTIME=disabled -RUN_DATABASE_MIGRATION_ON_STARTUP=true -SECRET_PERSISTENCE=NONE -WORKER_ENVIRONMENT=docker -NEW_SCHEDULER= +# extra settings to limit warnings during CE setup +RUN_DATABASE_MIGRATION_ON_STARTUP=true +SECRET_PERSISTENCE=NONE +WORKER_ENVIRONMENT=docker +NEW_SCHEDULER= DEPLOYMENT_MODE= +JOB_ERROR_REPORTING_STRATEGY= JOB_ERROR_REPORTING_SENTRY_DSN= +LOG_CONNECTOR_MESSAGES= +TEMPORAL_HISTORY_RETENTION_IN_DAYS= diff --git a/init/package-lock.json b/init/package-lock.json index ead39b63..6e1d52d6 100644 --- a/init/package-lock.json +++ b/init/package-lock.json @@ -14,8 +14,10 @@ "axios": "^0.25.0", "commander": "^9.1.0", "figlet": "^1.5.2", + "fs": "^0.0.1-security", "fs-extra": "^10.0.0", "handlebars": "^4.7.7", + "js-yaml": "^4.1.0", "lodash": "^4.17.21", "p-limit": "^3.1.0", "pino": "^7.6.4", @@ -30,8 +32,9 @@ "@types/figlet": "^1.5.4", "@types/fs-extra": "^9.0.13", "@types/jest": "^27.4.1", + "@types/js-yaml": "^4.0.5", "@types/lodash": "^4.14.168", - "@types/node": "^17.0.21", + "@types/node": "^17.0.45", "@types/pluralize": "^0.0.29", "@types/tar": "^6.1.1", "@types/uuid": "^8.3.4", @@ -1193,6 +1196,12 @@ "pretty-format": "^27.0.0" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.9", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", @@ -1215,9 +1224,9 @@ } }, "node_modules/@types/node": { - "version": "17.0.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", - "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==", + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", "dev": true }, "node_modules/@types/pluralize": { @@ -1644,8 +1653,7 @@ "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "node_modules/args": { "version": "5.0.1", @@ -2948,6 +2956,11 @@ "node": ">= 6" } }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "node_modules/fs-extra": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.1.tgz", @@ -4111,7 +4124,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, "dependencies": { "argparse": "^2.0.1" }, @@ -6895,6 +6907,12 @@ "pretty-format": "^27.0.0" } }, + "@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==", + "dev": true + }, "@types/json-schema": { "version": "7.0.9", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", @@ -6917,9 +6935,9 @@ } }, "@types/node": { - "version": "17.0.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.21.tgz", - "integrity": "sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==", + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", "dev": true }, "@types/pluralize": { @@ -7212,8 +7230,7 @@ "argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "args": { "version": "5.0.1", @@ -8221,6 +8238,11 @@ "mime-types": "^2.1.12" } }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "fs-extra": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.1.tgz", @@ -9105,7 +9127,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, "requires": { "argparse": "^2.0.1" } diff --git a/init/package.json b/init/package.json index a715e930..253d3197 100644 --- a/init/package.json +++ b/init/package.json @@ -28,8 +28,10 @@ "axios": "^0.25.0", "commander": "^9.1.0", "figlet": "^1.5.2", + "fs": "^0.0.1-security", "fs-extra": "^10.0.0", "handlebars": "^4.7.7", + "js-yaml": "^4.1.0", "lodash": "^4.17.21", "p-limit": "^3.1.0", "pino": "^7.6.4", @@ -44,8 +46,9 @@ "@types/figlet": "^1.5.4", "@types/fs-extra": "^9.0.13", "@types/jest": "^27.4.1", + "@types/js-yaml": "^4.0.5", "@types/lodash": "^4.14.168", - "@types/node": "^17.0.21", + "@types/node": "^17.0.45", "@types/pluralize": "^0.0.29", "@types/tar": "^6.1.1", "@types/uuid": "^8.3.4", diff --git a/init/resources/airbyte/workspace/airbyte_config/DESTINATION_CONNECTION.yaml b/init/resources/airbyte/workspace/airbyte_config/DESTINATION_CONNECTION.yaml deleted file mode 100644 index 19ca21e0..00000000 --- a/init/resources/airbyte/workspace/airbyte_config/DESTINATION_CONNECTION.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: "Faros Destination" - destinationDefinitionId: "6c6366b8-11f9-419a-8cdc-9209def71e1f" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - configuration: - dry_run: false - jsonata_mode: "FALLBACK" - edition_configs: - edition: "community" - hasura_url: "{{ hasura_url }}" - hasura_admin_secret: "{{ hasura_admin_secret }}" - segment_user_id: "{{ segment_user_id }}" - invalid_record_strategy: "SKIP" - source_specific_configs: - jira: - exclude_fields: [] - use_board_ownership: true - bitbucket: - application_mapping: "{}" - pagerduty: - application_mapping: "{}" - squadcast: - application_mapping: "{}" - victorops: - application_field: "service" - application_mapping: "{}" - statuspage: - application_mapping: "{}" - agileaccelerator: {} - tombstone: false diff --git a/init/resources/airbyte/workspace/airbyte_config/SOURCE_CONNECTION.yaml b/init/resources/airbyte/workspace/airbyte_config/SOURCE_CONNECTION.yaml index ab6ec2c3..a5e9c44c 100644 --- a/init/resources/airbyte/workspace/airbyte_config/SOURCE_CONNECTION.yaml +++ b/init/resources/airbyte/workspace/airbyte_config/SOURCE_CONNECTION.yaml @@ -1,8 +1,5 @@ --- - name: "GitHub" - sourceDefinitionId: "ef69ef6e-aa7f-4af1-a01d-ef775033524e" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "5d9079ca-8173-406f-bfdb-41f19c62daff" configuration: repository: "" start_date: "2022-01-01T00:00:00Z" @@ -12,9 +9,6 @@ page_size_for_large_streams: 10 tombstone: false - name: "GitLab" - sourceDefinitionId: "5e6175e5-68e1-4c17-bff9-56103bbb0d80" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "59c74ca4-8cbb-4c65-8cb7-66bf771190fb" configuration: groups: "" api_url: "" @@ -23,9 +17,6 @@ private_token: "" tombstone: false - name: "Bitbucket" - sourceDefinitionId: "51d2be20-0bdc-4532-b5a2-98a2ef91b23e" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "5a19e927-51a2-4d5f-9b26-b35aba0910e0" configuration: pagelen: 100 serverUrl: "https://api.bitbucket.org/2.0" @@ -33,9 +24,6 @@ cutoff_days: 90 tombstone: false - name: "Phabricator" - sourceDefinitionId: "63aa5e31-9490-4730-b2d4-14e42a4eb899" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "8e08224a-08da-4da4-95df-79639df68962" configuration: limit: 100 token: "" @@ -43,12 +31,9 @@ cutoff_days: 90 tombstone: false - name: "Jira" - sourceDefinitionId: "68e63de2-bb83-4c7e-93fa-a8a9051e3993" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "22852029-670c-4296-958e-c581fa76ae98" configuration: email: "" - domain: "" + domain: "jira.atlassian.net" projects: [] api_token: "" start_date: "2022-01-01T00:00:00Z" @@ -57,9 +42,6 @@ enable_experimental_streams: true tombstone: false - name: "Buildkite" - sourceDefinitionId: "431b2890-39cb-4138-bc30-7dccdd0b94e0" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "407664f2-22b0-431d-bc36-b1be8a6aae96" configuration: token: "" cutoff_days: 90 @@ -67,20 +49,14 @@ rest_api_version: "v2" tombstone: false - name: "CircleCI" - sourceDefinitionId: "6727d77e-5c06-4c28-817a-c1cacc18e106" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "81c73d90-cbff-465a-ab8d-8cde1be6e718" configuration: url: "https://circleci.com/api/v2" token: "" - repo_names: [] + project_names: [] cutoff_days: 90 reject_unauthorized: true tombstone: false - name: "Harness" - sourceDefinitionId: "6fe89830-d04d-401b-aad6-6552ffa5c4af" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "c78b0bb2-5fee-46f7-a043-63e498004776" configuration: api_key: "" api_url: "https://app.harness.io" @@ -90,9 +66,6 @@ deployment_timeout: 24 tombstone: false - name: "Jenkins" - sourceDefinitionId: "d6f73702-d7a0-4e95-9758-b0fb1af0bfba" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "55b0325e-4bc7-49b4-b230-36c5a67df068" configuration: user: "" token: "" @@ -101,27 +74,18 @@ last100Builds: false tombstone: false - name: "OpsGenie" - sourceDefinitionId: "1b22f335-f5bb-442d-a345-38b860baa41d" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "6749ef2f-5dc7-4e5f-ad5d-74aac6ef49da" configuration: api_key: "" page_size: 10 cutoff_days: 90 tombstone: false - name: "Datadog" - sourceDefinitionId: "172cc5e1-9443-4f1b-a386-5a7813e05f45" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "7fc3063d-fb70-4cf7-bff3-69051b58e84c" configuration: api_key: "" page_size: 100 application_key: "" tombstone: false - name: "PagerDuty" - sourceDefinitionId: "2817b3f0-04e4-4c7a-9f32-7a5e8a83db95" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "5ed7412c-72d3-46a9-bc40-0e362ad437c6" configuration: token: "" page_size: 25 @@ -129,27 +93,18 @@ incident_log_entries_overview: true tombstone: false - name: "SquadCast" - sourceDefinitionId: "dcb04367-398c-4af9-831c-aef0ce4f8f32" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "4321e9f5-bb4d-4f6e-8625-0cb73d1cff16" configuration: token: "" cutoff_days: 90 max_content_length: 20000 tombstone: false - name: "Statuspage" - sourceDefinitionId: "4b3cd0a4-4994-4616-af28-9871e28a8368" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "a778b201-49fe-43e4-a470-391b07115fab" configuration: api_key: "" page_id: "" cutoff_days: 90 tombstone: false - name: "VictorOps" - sourceDefinitionId: "7e20ce3e-d820-4327-ad7a-88f3927fd97a" - workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - sourceId: "e1d032e0-3a8b-47dc-bac8-2786264b11f4" configuration: apiId: "" apiKey: "" diff --git a/init/resources/airbyte/workspace/airbyte_config/STANDARD_DESTINATION_DEFINITION.yaml b/init/resources/airbyte/workspace/airbyte_config/STANDARD_DESTINATION_DEFINITION.yaml deleted file mode 100644 index 68d45e1f..00000000 --- a/init/resources/airbyte/workspace/airbyte_config/STANDARD_DESTINATION_DEFINITION.yaml +++ /dev/null @@ -1,561 +0,0 @@ ---- -- destinationDefinitionId: "6c6366b8-11f9-419a-8cdc-9209def71e1f" - name: "Faros Destination" - dockerRepository: "farosai/airbyte-faros-destination" - dockerImageTag: "0.4.69" - documentationUrl: "https://docs.faros.ai" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Faros Destination Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "edition_configs" - properties: - origin: - type: "string" - order: 1 - title: "Origin name" - examples: - - "my-faros-destination" - description: "The Faros origin name used for uploaded entries. Must be unique." - dry_run: - type: "boolean" - order: 2 - title: "Dry run" - default: false - description: "Process all input records but avoid writing into Faros API" - jsonata_mode: - enum: - - "FALLBACK" - - "OVERRIDE" - type: "string" - order: 5 - title: "JSONata apply mode" - default: "FALLBACK" - description: "JSONata apply mode when converting input records." - edition_configs: - type: "object" - oneOf: - - type: "object" - title: "Community Edition" - required: - - "hasura_url" - properties: - edition: - type: "string" - const: "community" - title: "Community Edition" - description: "Community Edition" - hasura_url: - type: "string" - title: "Hasura URL" - default: "http://localhost:8080" - description: "The Community Edition Hasura URL." - segment_user_id: - type: "string" - title: "Segment User Id" - format: "uuid" - description: "The User UUID with which to track events in Segment.\ - \ If not present, then reporting is disabled. See https://community.faros.ai/docs/telemetry\ - \ for more details." - hasura_admin_secret: - type: "string" - title: "Hasura Admin Secret" - description: "The Hasura Admin Secret." - airbyte_secret: true - - type: "object" - title: "Cloud Edition" - required: - - "api_key" - - "api_url" - properties: - graph: - type: "string" - title: "Graph name" - default: "default" - description: "The Faros graph name." - api_key: - type: "string" - title: "API Key" - description: "The Faros API key to use to access the API." - airbyte_secret: true - api_url: - type: "string" - title: "API URL" - default: "https://prod.api.faros.ai" - examples: - - "https://prod.api.faros.ai" - description: "The Faros API URL." - edition: - type: "string" - const: "cloud" - title: "Cloud Edition" - description: "Cloud Edition" - expiration: - type: "string" - title: "Revision expiration" - default: "5 seconds" - examples: - - "5 seconds" - description: "The Faros graph revision expiration time." - order: 0 - title: "Faros Edition" - description: "Choose your Faros Edition." - jsonata_expression: - type: "string" - order: 4 - title: "JSONata expression" - description: "JSONata expression for converting input records. If provided\ - \ applies the expression based on specified JSONata apply mode." - invalid_record_strategy: - enum: - - "FAIL" - - "SKIP" - type: "string" - order: 3 - title: "Invalid record strategy" - default: "SKIP" - description: "Strategy to follow to handle an invalid input record." - source_specific_configs: - type: "object" - oneOf: - - type: "object" - title: "Configurations" - properties: - jira: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - exclude_fields: - type: "array" - items: - type: "string" - title: "Exclude Fields" - default: [] - description: "List of fields to exclude when saving tasks. Defaults\ - \ to no exclusions." - truncate_limit: - type: "integer" - title: "Truncate Limit" - default: 10000 - description: "Truncate the descriptions of projects, tasks,\ - \ and epics." - use_board_ownership: - type: "boolean" - title: "Use Board Ownership" - default: false - description: "Use Jira boards for assigning owners of tasks,\ - \ or use projects." - additional_fields_array_limit: - type: "integer" - title: "Additional Fields Array Limit" - default: 50 - description: "Truncates an additional field's array value to\ - \ the given length." - title: "Jira" - description: "Configuration options that apply to records generated\ - \ by the Jira Source." - docker: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - required: - - "organization" - properties: - skip_tags: - type: "array" - items: - type: "string" - title: "Skip tags" - description: "The images with the listed tags will be ignored\ - \ by the feed" - label_prefix: - type: "string" - title: "Label Prefix" - description: "Label prefix used to construct build key and commit\ - \ key for cicd_Artifact and cicd_ArtifactCommitAssociation\ - \ from labels. Every label key start with this prefix. If\ - \ the prefix is 'faros-', then the label keys will be 'faros-build-id',\ - \ 'faros-pipeline-id', etc." - organization: - type: "string" - title: "Organization" - default: "unknown-org" - description: "Organization name to associate with image tags" - title: "Docker" - description: "Configuration options that apply to records generated\ - \ by the Docker Source." - backlog: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - max_description_length: - type: "integer" - title: "Max Description Length" - default: 1000 - description: "Value to cut description to specific length" - title: "Backlog" - description: "Configuration options that apply to records generated\ - \ by the Backlog Source." - datadog: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - default_severity: - type: "string" - title: "Default Severity" - pattern: "^(Sev[1-5])?(Custom)?$" - examples: - - "Sev1" - - "Sev2" - - "Sev3" - - "Sev4" - - "Sev5" - - "Custom" - description: "A default severity category if not present" - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of Datadog service(s) name, to compute\ - \ platform specific app name and platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object." - title: "Datadog" - description: "Configuration options that apply to records generated\ - \ by the Datadog Source." - bamboohr: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - bootstrap_teams_from_managers: - type: "boolean" - title: "Bootstrap Teams From Managers" - default: false - description: "Assign employees to teams based off their managers." - title: "BambooHR" - description: "Configuration options that apply to records generated\ - \ by the BambooHR Source." - opsgenie: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of OpsGenie service(s) name, to compute\ - \ a Faros Application and Platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object.\ - \ If specified, and such service exists, the feed will write\ - \ a compute_Application object." - max_description_length: - type: "integer" - title: "Max Description Length" - default: 1000 - description: "Value to cut description to specific length" - title: "OpsGenie" - description: "Configuration options that apply to records generated\ - \ by the OpsGenie Source." - bitbucket: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of Bitbucket service(s) name, to compute\ - \ platform specific app name and platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object." - title: "Bitbucket" - description: "Configuration options that apply to records generated\ - \ by the Bitbucket Source." - pagerduty: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - default_severity: - type: "string" - title: "Default Severity" - pattern: "^(Sev[1-5])?(Custom)?$" - examples: - - "Sev1" - - "Sev2" - - "Sev3" - - "Sev4" - - "Sev5" - - "Custom" - description: "A default severity category if not present" - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of PagerDuty service(s) name, to compute\ - \ platform specific app name and platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object." - title: "PagerDuty" - description: "Configuration options that apply to records generated\ - \ by the PagerDuty Source." - squadcast: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of SquadCast service(s) name, to compute\ - \ platform specific app name and platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object." - title: "SquadCast" - description: "Configuration options that apply to records generated\ - \ by the SquadCast Source." - victorops: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - application_field: - type: "string" - title: "Application Field" - default: "service" - examples: - - "routing_key" - description: "Field name in a VictorOps incident, to compute\ - \ a Faros Application name. The value of this field will be\ - \ used to look up the app/platform in the specified application\ - \ mapping" - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of VictorOps service(s) name, to compute\ - \ a Faros Application and Platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object.\ - \ If specified, and such service exists, the feed will write\ - \ a compute_Application object." - title: "VictorOps" - description: "Configuration options that apply to records generated\ - \ by the VictorOps Source." - servicenow: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - default_priority: - type: "string" - title: "Default Priority" - pattern: "^(Critical|High|Medium|Low|Custom)$" - examples: - - "Critical" - - "High" - - "Medium" - - "Low" - - "Custom" - description: "A default priority category if not present" - default_severity: - type: "string" - title: "Default Severity" - pattern: "^(Sev[1-5])?(Custom)?$" - examples: - - "Sev1" - - "Sev2" - - "Sev3" - - "Sev4" - - "Sev5" - - "Custom" - description: "A default severity category if not present" - application_field: - type: "string" - title: "Application Field" - default: "business_service" - pattern: "^(business_service|cmdb_ci)$" - examples: - - "business_service" - - "cmdb_ci" - description: "Field name in a ServiceNow incident, to compute\ - \ a Faros Application name. The value of this field will be\ - \ used to look up the app/platform in the specified application\ - \ mapping" - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of ServiceNow service(s) name, to compute\ - \ platform specific app name and platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object." - title: "ServiceNow" - description: "Configuration options that apply to records generated\ - \ by the ServiceNow Source." - statuspage: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of Statuspage service(s) name, to compute\ - \ platform specific app name and platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object." - title: "Statuspage" - description: "Configuration options that apply to records generated\ - \ by the Statuspage Source." - firehydrant: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - application_mapping: - type: "string" - title: "Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - multiline: true - description: "JSON map of FireHydrant service(s) name, to compute\ - \ platform specific app name and platform name. Used to reference\ - \ compute_Application object, from an ims_Incident object." - max_description_length: - type: "integer" - title: "Max Description Length" - default: 1000 - description: "Value to cut description to specific length" - title: "FireHydrant" - description: "Configuration options that apply to records generated\ - \ by the FireHydrant Source." - phabricator: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - max_description_length: - type: "integer" - title: "Max Description Length" - default: 1000 - description: "Value to cut description to specific length" - title: "Phabricator" - description: "Configuration options that apply to records generated\ - \ by the Phabricator Source." - azurepipeline: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - application_mapping: - type: "string" - title: "Azure pipeline Application Mapping" - default: "{}" - examples: - - "{ \"Aion\": { \"name\": \"aion\", \"platform\": \"ECS\" }\ - \ }" - description: "JSON map of Azure Pipeline service(s) name, to\ - \ compute platform specific app name and platform name. Used\ - \ to reference compute_Application object, from an cicd_Deployment\ - \ object." - title: "Azure pipeline" - description: "Configuration options that apply to records generated\ - \ by the Azure pipeline Source." - agileaccelerator: - type: "object" - oneOf: - - type: "object" - title: "Configuration" - properties: - max_description_length: - type: "integer" - title: "Max Description Length" - default: 1000 - description: "Value to cut description to specific length" - work_additional_fields: - type: "array" - items: - type: "string" - title: "Work Additional Fields" - default: [] - description: "Fields to save to tms_Task's additionalFields\ - \ variable" - title: "AgileAccelerator" - description: "Configuration options that apply to records generated\ - \ by the AgileAccelerator Source." - order: 7 - title: "Source-specific configs" - description: "Configuration options that apply to specific sources." - jsonata_destination_models: - type: "array" - items: - type: "string" - order: 6 - title: "JSONata destination models" - examples: - - "ims_Incident" - - "vcs_Commit" - description: "Destination models when using JSONata expression." - additionalProperties: false - supportsIncremental: true - supportsNormalization: false - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" - diff --git a/init/resources/airbyte/workspace/airbyte_config/STANDARD_SOURCE_DEFINITION.yaml b/init/resources/airbyte/workspace/airbyte_config/STANDARD_SOURCE_DEFINITION.yaml deleted file mode 100644 index c8f9b7ce..00000000 --- a/init/resources/airbyte/workspace/airbyte_config/STANDARD_SOURCE_DEFINITION.yaml +++ /dev/null @@ -1,911 +0,0 @@ ---- -- sourceDefinitionId: "0eba3d46-6b9f-48de-baf5-63632f5eca50" - name: "Faros Feeds" - dockerRepository: "farosai/airbyte-faros-feeds-source" - dockerImageTag: "latest" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Faros Feeds Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "feed_cfg" - properties: - debug: - type: "boolean" - order: 2 - title: "Enable debug logs" - feed_cfg: - type: "object" - oneOf: - - type: "object" - title: "github" - properties: - feed_name: - type: "string" - const: "github-feed" - feed_path: - type: "string" - const: "vcs/github-feed" - cutoff_days: - type: "integer" - order: 2 - title: "Cutoff days" - description: "Only fetch commits, issues and pull requests updated\ - \ in the last number of days" - repositories: - type: "array" - items: - type: "string" - order: 0 - title: "GitHub Repositories" - examples: - - "apache/kafka" - - "airbytehq/airbyte" - description: "GitHub organization/repository" - personal_access_token: - type: "string" - order: 1 - title: "Personal Access Token" - airbyte_secret: true - order: 0 - title: "Feed Type" - description: "Choose your Feed Type" - feed_args: - type: "array" - items: - type: "string" - order: 1 - title: "Feed command line arguments" - examples: - - "--expiration '5 seconds'" - description: "Additional command line arguments to invoke the feed with" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "172cc5e1-9443-4f1b-a386-5a7813e05f45" - name: "Datadog" - dockerRepository: "farosai/airbyte-datadog-source" - dockerImageTag: "0.2.14" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/datadog-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Datadog Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "api_key" - - "application_key" - properties: - api_key: - type: "string" - order: 0 - title: "Datadog API Key" - airbyte_secret: true - page_size: - type: "integer" - order: 2 - title: "Page Size" - default: 100 - description: "used when retrieving paginated data from Datadog" - application_key: - type: "string" - order: 1 - title: "Datadog Application Key" - description: "requires incident_read authorization scope" - airbyte_secret: true - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "1b22f335-f5bb-442d-a345-38b860baa41d" - name: "OpsGenie" - dockerRepository: "farosai/airbyte-opsgenie-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/opsgenie-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "opsGenie Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "api_key" - - "cutoff_days" - properties: - api_key: - type: "string" - title: "API key" - description: "In order to create key, go to https://.{your_acount}.app.opsgenie.com/settings/api-key-management" - airbyte_secret: true - page_size: - type: "number" - title: "Page size" - default: 10 - description: "used when retrieving paginated data from OpsGenie" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch incidents created after cutoff" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "431b2890-39cb-4138-bc30-7dccdd0b94e0" - name: "Buildkite" - dockerRepository: "farosai/airbyte-buildkite-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/buildkite-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "BuildKite Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "token" - - "cutoff_days" - properties: - token: - type: "string" - title: "Token" - description: "To create token go to https://buildkite.com/user/api-access-tokens" - airbyte_secret: true - page_size: - type: "number" - title: "Page Size" - description: "number of pipelines, builds, jobs to fetch per call" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch data updated after cutoff" - organization: - type: "string" - title: "Organization Slug" - description: "Only pull pipelines from the specified organization. If not\ - \ set, all pipelines from all organizations will be pulled." - graphql_version: - type: "string" - title: "GraphQL Version" - default: "v1" - rest_api_version: - type: "string" - title: "Rest API Version" - default: "v2" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "4b3cd0a4-4994-4616-af28-9871e28a8368" - name: "Statuspage" - dockerRepository: "farosai/airbyte-statuspage-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/statuspage-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Statuspage Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "page_id" - - "api_key" - - "cutoff_days" - properties: - org_id: - type: "string" - title: "Organization ID" - description: "Statuspage Organization ID" - api_key: - type: "string" - title: "Statuspage API key" - description: "API key for Statuspage API authentication" - airbyte_secret: true - page_id: - type: "string" - title: "Page ID" - description: "Statuspage Page ID" - airbyte_secret: true - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch data updated after cutoff" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "d6f73702-d7a0-4e95-9758-b0fb1af0bfba" - name: "Jenkins" - dockerRepository: "farosai/airbyte-jenkins-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/jenkins-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Jenkins Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "server_url" - - "user" - - "token" - properties: - user: - type: "string" - title: "User" - description: "Jenkins User" - depth: - type: "integer" - title: "Depth" - description: "Jenkins JSON API does not support deep scan, it is required\ - \ to generate a suitable tree for the corresponding depth. Job in some\ - \ cases have many sub jobs, depth needs to quantify how many sub jobs\ - \ are showed. If depth is not provided we will try to compute it automatically" - token: - type: "string" - title: "Token" - description: "Jenkins Token for Jenkins API authentication" - airbyte_secret: true - pageSize: - type: "integer" - title: "Page Size" - default: 10 - minimum: 1 - description: "Quantity of jobs on a single page fetched from Jenkins" - server_url: - type: "string" - title: "Jenkins Server URL" - examples: - - "https://my-jenkins-server.example.com" - description: "The Server URL for fetching data from Jenkins" - last100Builds: - type: "boolean" - title: "Last 100 Builds Only" - default: false - description: "Fetch only 100 last builds from Jenkins server" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "51d2be20-0bdc-4532-b5a2-98a2ef91b23e" - name: "Bitbucket" - dockerRepository: "farosai/airbyte-bitbucket-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/bitbucket-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Bitbucket Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "workspaces" - - "cutoff_days" - properties: - token: - type: "string" - title: "Access Token" - description: "Personal Access Token for Bitbucket API authentication. See\ - \ https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html\ - \ for information on how to generate this token." - airbyte_secret: true - pagelen: - type: "integer" - title: "Page Size" - default: 100 - description: "Page size to use when requesting records from Bitbucket API" - password: - type: "string" - description: "Bitbucket password" - airbyte_secret: true - username: - type: "string" - description: "Bitbucket username. Either username/password or Access Token\ - \ must be provided." - serverUrl: - type: "string" - title: "API URL" - default: "https://api.bitbucket.org/2.0" - examples: - - "https://api.bitbucket.org/2.0" - description: "The API URL for fetching data from Bitbucket" - workspaces: - type: "array" - items: - type: "string" - title: "Workspaces" - examples: - - "blaze-lib" - description: "Names of your Bitbucket workspaces" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch data updated after cutoff" - repositories: - type: "array" - items: - type: "string" - title: "Repositories" - examples: - - "blaze-lib/blaze" - description: "Names of your Bitbucket repositories in the format 'workspace/repo-slug'.\ - \ If none are provided, data from all repositories for the specified workspaces\ - \ will be pulled." - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "5e6175e5-68e1-4c17-bff9-56103bbb0d80" - name: "Gitlab" - dockerRepository: "airbyte/source-gitlab" - dockerImageTag: "0.1.6" - documentationUrl: "https://docs.airbyte.io/integrations/sources/gitlab" - icon: "gitlab.svg" - sourceType: "api" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/gitlab" - connectionSpecification: - type: "object" - title: "Source GitLab Singer Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "api_url" - - "private_token" - - "start_date" - properties: - groups: - type: "string" - title: "Groups" - examples: - - "airbyte.io" - description: "Space-delimited list of groups. e.g. airbyte.io." - api_url: - type: "string" - title: "API URL" - examples: - - "gitlab.com" - description: "Please enter your basic URL from GitLab instance." - projects: - type: "string" - title: "Projects" - examples: - - "airbyte.io/documentation" - description: "Space-delimited list of projects. e.g. airbyte.io/documentation\ - \ meltano/tap-gitlab." - start_date: - type: "string" - title: "Start Date" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-03-01T00:00:00Z" - description: "The date from which you'd like to replicate data for GitLab\ - \ API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this\ - \ date will be replicated." - private_token: - type: "string" - title: "Privat Token" - description: "Log into your GitLab account and then generate a personal\ - \ Access Token." - airbyte_secret: true - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "63aa5e31-9490-4730-b2d4-14e42a4eb899" - name: "Phabricator" - dockerRepository: "farosai/airbyte-phabricator-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/phabricator-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Phabricator Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "server_url" - - "token" - - "cutoff_days" - properties: - limit: - type: "integer" - title: "Limit" - default: 100 - maximum: 100 - minimum: 1 - description: "Limit of results on a single page fetched from Phabricator" - token: - type: "string" - title: "Conduit Token" - description: "Conduit Token for Phabricator API authentication" - airbyte_secret: true - projects: - type: "array" - items: - type: "string" - title: "Projects" - examples: - - "my_project_1" - - "my_project_2" - description: "List of Phabricator projects slugs, e.g. `my_project_1,my_project_2`.\ - \ Slugs are the same as project hashtags. If none provided would sync\ - \ all projects." - server_url: - type: "string" - title: "Phabricator Conduit API URL" - examples: - - "https://my-phabricator-server.example.com" - description: "The API URL for fetching data from Phabricator" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch data updated after cutoff" - repositories: - type: "array" - items: - type: "string" - title: "Repositories" - examples: - - "repo-1" - - "repo-2" - description: "List of Phabricator repositories, e.g. `repo-1,repo-2`. If\ - \ none provided would sync all repositories." - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "6727d77e-5c06-4c28-817a-c1cacc18e106" - name: "CircleCI" - dockerRepository: "farosai/airbyte-circleci-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/circleci-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "CircleCI Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "token" - - "repo_names" - - "cutoff_days" - properties: - url: - type: "string" - title: "API URL" - default: "https://circleci.com/api/v2" - description: "CircleCI API URL" - token: - type: "string" - title: "token" - description: "CircleCI personal API token. See https://circleci.com/docs/2.0/managing-api-tokens/#creating-a-personal-api-token" - airbyte_secret: true - repo_names: - type: "array" - items: - type: "string" - title: "Repository Names" - description: "Names should be in the format \"project_type/organization_name/repository_name\"" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch data updated after cutoff" - reject_unauthorized: - type: "boolean" - title: "Enforce Authorized Requests" - default: true - description: "Enable certificate validation for the CircleCI server" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "68e63de2-bb83-4c7e-93fa-a8a9051e3993" - name: "Jira" - dockerRepository: "airbyte/source-jira" - dockerImageTag: "0.3.0" - documentationUrl: "https://docs.airbyte.io/integrations/sources/jira" - icon: "jira.svg" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/jira" - connectionSpecification: - type: "object" - title: "Jira Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "api_token" - - "domain" - - "email" - properties: - email: - type: "string" - title: "Email" - description: "The user email for your Jira account." - domain: - type: "string" - title: "Domain" - pattern: "^[a-zA-Z0-9._-]*\\.atlassian\\.net$" - examples: - - "domainname.atlassian.net" - description: "The Domain for your Jira account, e.g. airbyteio.atlassian.net" - projects: - type: "array" - items: - type: "string" - title: "Projects" - examples: - - "PROJ1" - - "PROJ2" - description: "List of Jira project keys to replicate data for." - api_token: - type: "string" - title: "API Token" - description: "Jira API Token. See the docs for more information on how to generate this key." - airbyte_secret: true - start_date: - type: "string" - title: "Start Date" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-03-01T00:00:00Z" - description: "The date from which you'd like to replicate data for Jira\ - \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\ - \ will be replicated. Note that it will be used only in the following\ - \ incremental streams: issues." - render_fields: - type: "boolean" - title: "Render Issue Fields" - default: false - description: "Render issue fields in HTML format in addition to Jira JSON-like\ - \ format." - additional_fields: - type: "array" - items: - type: "string" - title: "Additional Fields" - examples: - - "customfield_10096" - - "customfield_10071" - description: "List of additional fields to include in replicating issues." - expand_issue_changelog: - type: "boolean" - title: "Expand Issue Changelog" - default: false - description: "Expand the changelog when replicating issues." - enable_experimental_streams: - type: "boolean" - title: "Enable Experimental Streams" - default: false - description: "Allow the use of experimental streams which rely on undocumented\ - \ Jira API endpoints. See https://docs.airbyte.io/integrations/sources/jira#experimental-tables\ - \ for more info." - additionalProperties: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "7e20ce3e-d820-4327-ad7a-88f3927fd97a" - name: "VictorOps" - dockerRepository: "farosai/airbyte-victorops-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/victorops-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "VictorOps Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "apiId" - - "apiKey" - - "cutoff_days" - properties: - apiId: - type: "string" - title: "API ID" - description: "VictorOps API ID" - airbyte_secret: true - apiKey: - type: "string" - title: "VictorOps API key" - description: "API key for VictorOps API authentication" - airbyte_secret: true - pageLimit: - type: "integer" - title: "Page Limit" - default: 100 - description: "VictorOps Page Limit" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch data updated after cutoff" - currentPhase: - type: "string" - title: "Current Phase" - default: "triggered,acknowledged,resolved" - description: "VictorOps Current Phase" - maxContentLength: - type: "integer" - title: "VictorOps Content Length" - default: 500000 - description: "VictorOps API response content length limit, try increasing\ - \ if 'RequestError' is encountered." - additionalProperties: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "2817b3f0-04e4-4c7a-9f32-7a5e8a83db95" - name: "PagerDuty" - dockerRepository: "farosai/airbyte-pagerduty-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/pagerduty-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "PagerDuty Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "token" - properties: - token: - type: "string" - title: "API key" - description: "API key for PagerDuty API authentication" - airbyte_secret: true - page_size: - type: "integer" - title: "Page Size" - default: 25 - maximum: 25 - minimum: 1 - description: "page size to use when querying PagerDuty API" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Fetch pipelines updated in the last number of days" - default_severity: - type: "string" - title: "Severity category" - pattern: "^(Sev[0-5])?(Custom)?$" - examples: - - "Sev1" - - "Sev2" - - "Sev3" - - "Sev4" - - "Sev5" - - "Custom" - description: "A default severity category if not present" - incident_log_entries_overview: - type: "boolean" - title: "Incident Log Entries Overview" - default: true - description: "If true, will return a subset of log entries that show only\ - \ the most important changes to the incident." - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "6fe89830-d04d-401b-aad6-6552ffa5c4af" - name: "Harness" - dockerRepository: "farosai/airbyte-harness-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/harness-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "Harness Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "api_key" - - "account_id" - - "cutoff_days" - properties: - api_key: - type: "string" - title: "API key" - airbyte_secret: true - api_url: - type: "string" - title: "API URL" - default: "https://app.harness.io" - examples: - - "https://my-harness-server.example.com" - description: "The API URL for fetching data from Harness" - page_size: - type: "integer" - title: "Page Size" - default: 100 - description: "number of pipelines (builds) to fetch per call" - account_id: - type: "string" - title: "Account ID" - description: "Harness Account ID" - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch deployments updated after cutoff" - deployment_timeout: - type: "integer" - title: "Deployment Timeout" - default: 24 - description: "Max number of hours to consider for a deployment to be running/queued\ - \ before giving up on syncing it" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "dcb04367-398c-4af9-831c-aef0ce4f8f32" - name: "Squadcast" - dockerRepository: "farosai/airbyte-squadcast-source" - dockerImageTag: "0.4.23" - documentationUrl: "https://github.com/faros-ai/airbyte-connectors/tree/main/sources/squadcast-source" - spec: - documentationUrl: "https://docs.faros.ai" - connectionSpecification: - type: "object" - title: "SquadCast Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "token" - - "cutoff_days" - properties: - token: - type: "string" - title: "Token" - description: "Get API Refresh Token on https://app.squadcast.com/user/USER_ID\ - \ where USER_ID is user's ID" - airbyte_secret: true - cutoff_days: - type: "integer" - title: "Cutoff Days" - default: 90 - description: "Only fetch data updated after cutoff" - event_deduped: - type: "boolean" - title: "Events Deduped" - description: "If set to true, it will return only the deduped events. If\ - \ set to false, it will return only the non-deduped event. Otherwise it\ - \ will return all the events" - event_owner_id: - type: "string" - title: "Event Owner ID" - description: "Team ID or User ID" - event_incident_id: - type: "string" - title: "Events Incident ID" - description: "If set it will pull all events from only this Incident. If\ - \ no it pull all events from all incidents" - incident_owner_id: - type: "string" - title: "Incident Owner ID" - description: "Team ID or User ID" - max_content_length: - type: "integer" - title: "Max Content Length" - default: 20000 - description: "Max size of Squadcast API response content in bytes. Increase\ - \ this value if you get a \"maxContentLength exceeded\" error" - additionalProperties: false - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] -- sourceDefinitionId: "ef69ef6e-aa7f-4af1-a01d-ef775033524e" - name: "GitHub" - dockerRepository: "airbyte/source-github" - dockerImageTag: "0.3.8" - documentationUrl: "https://docs.airbyte.io/integrations/sources/github" - icon: "github.svg" - sourceType: "api" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/github" - connectionSpecification: - type: "object" - title: "GitHub Source Spec" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "start_date" - - "repository" - properties: - branch: - type: "string" - title: "Branch" - examples: - - "airbytehq/airbyte/master airbytehq/airbyte/my-branch" - description: "Space-delimited list of GitHub repository branches to pull\ - \ commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified\ - \ for a repository, the default branch will be pulled." - repository: - type: "string" - title: "GitHub Repositories" - examples: - - "airbytehq/airbyte airbytehq/another-repo" - - "airbytehq/*" - - "airbytehq/airbyte" - description: "Space-delimited list of GitHub organizations/repositories,\ - \ e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get\ - \ all repositories from organization and `airbytehq/airbyte airbytehq/another-repo`\ - \ for multiple repositories." - start_date: - type: "string" - title: "Start Date" - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - examples: - - "2021-03-01T00:00:00Z" - description: "The date from which you'd like to replicate data for GitHub\ - \ in the format YYYY-MM-DDT00:00:00Z. All data generated after this date\ - \ will be replicated. Note that it will be used only in the following\ - \ incremental streams: comments, commits and issues." - credentials: - type: "object" - oneOf: - - type: "object" - title: "Authenticate via GitHub (OAuth)" - required: - - "access_token" - properties: - access_token: - type: "string" - title: "Access Token" - description: "OAuth access token" - airbyte_secret: true - option_title: - type: "string" - const: "OAuth Credentials" - title: "Credentials Title" - description: "OAuth Credentials" - - type: "object" - title: "Authenticate with Personal Access Token" - required: - - "personal_access_token" - properties: - option_title: - type: "string" - const: "PAT Credentials" - title: "Credentials Title" - description: "PAT Credentials" - personal_access_token: - type: "string" - title: "Personal Access Tokens" - description: "Log into GitHub and then generate a personal access token. To load balance your API quota consumption\ - \ across multiple API tokens, input multiple tokens separated with\ - \ \",\"" - airbyte_secret: true - title: "Authentication Mechanism" - description: "Choose how to authenticate to GitHub" - page_size_for_large_streams: - type: "integer" - title: "Page size for large streams" - default: 10 - maximum: 100 - minimum: 1 - description: "The Github connector contains several streams with a large\ - \ load. The page size of such streams depends on the size of your repository.\ - \ Recommended to specify values between 10 and 30." - additionalProperties: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - authSpecification: - auth_type: "oauth2.0" - oauth2Specification: - rootObject: - - "credentials" - - "0" - oauthFlowInitParameters: [] - oauthFlowOutputParameters: - - - "access_token" diff --git a/init/resources/airbyte/workspace/airbyte_config/STANDARD_SYNC.yaml b/init/resources/airbyte/workspace/airbyte_config/STANDARD_SYNC.yaml index c3519b3f..b632f641 100644 --- a/init/resources/airbyte/workspace/airbyte_config/STANDARD_SYNC.yaml +++ b/init/resources/airbyte/workspace/airbyte_config/STANDARD_SYNC.yaml @@ -1,95 +1,10 @@ --- -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "github_source__github__" - sourceId: "5d9079ca-8173-406f-bfdb-41f19c62daff" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "6421df4e-0c5a-4666-a530-9c01de683518" - name: "default" +- prefix: "github_source__github__" + name: "GitHub - Faros" catalog: streams: - stream: name: "assignees" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - repository: - type: - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -97,65 +12,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "branches" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - name: - type: - - "null" - - "string" - commit: - type: - - "null" - - "object" - properties: - sha: - type: - - "null" - - "string" - url: - type: - - "null" - - "string" - protected: - type: - - "null" - - "boolean" - protection: - type: - - "null" - - "object" - properties: - required_status_checks: - type: - - "null" - - "object" - properties: - contexts: - type: - - "null" - - "array" - items: - type: - - "null" - - "string" - enforcement_level: - type: - - "null" - - "string" - repository: - type: - - "string" - protection_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -164,108 +25,12 @@ - - "name" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "repository" - - "name" - stream: name: "collaborators" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - repository: - type: - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - permissions: - type: - - "null" - - "object" - properties: - pull: - type: - - "null" - - "boolean" - push: - type: - - "null" - - "boolean" - admin: - type: - - "null" - - "boolean" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -273,305 +38,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "commits" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - sha: - type: - - "null" - - "string" - url: - type: - - "null" - - "string" - author: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - commit: - type: - - "null" - - "object" - properties: - url: - type: - - "null" - - "string" - tree: - type: - - "null" - - "object" - properties: - sha: - type: - - "null" - - "string" - url: - type: - - "null" - - "string" - author: - type: - - "null" - - "object" - properties: - date: - type: - - "null" - - "string" - format: "date-time" - name: - type: - - "null" - - "string" - email: - type: - - "null" - - "string" - message: - type: - - "null" - - "string" - committer: - type: - - "null" - - "object" - properties: - date: - type: - - "null" - - "string" - format: "date-time" - name: - type: - - "null" - - "string" - email: - type: - - "null" - - "string" - verification: - type: - - "null" - - "object" - properties: - reason: - type: - - "null" - - "string" - payload: - type: - - "null" - - "string" - verified: - type: - - "null" - - "boolean" - signature: - type: - - "null" - - "string" - comment_count: - type: - - "null" - - "integer" - node_id: - type: - - "null" - - "string" - parents: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - sha: - type: - - "null" - - "string" - url: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - committer: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - repository: - type: - - "string" - comments_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -583,46 +54,11 @@ sync_mode: "incremental" cursor_field: - "created_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "sha" - stream: name: "issue_labels" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - color: - type: - - "null" - - "string" - default: - type: - - "null" - - "boolean" - node_id: - type: - - "null" - - "string" - repository: - type: - - "string" - description: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -630,159 +66,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "issue_milestones" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - due_on: - type: - - "null" - - "string" - format: "date-time" - number: - type: - - "null" - - "integer" - creator: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - closed_at: - type: - - "null" - - "string" - format: "date-time" - created_at: - type: - - "null" - - "string" - format: "date-time" - labels_url: - type: - - "null" - - "string" - repository: - type: - - "string" - updated_at: - type: - - "null" - - "string" - format: "date-time" - description: - type: - - "null" - - "string" - open_issues: - type: - - "null" - - "integer" - closed_issues: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" - "incremental" @@ -794,540 +82,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "issues" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - body: - type: - - "null" - - "string" - user: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - labels: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - color: - type: - - "null" - - "string" - default: - type: - - "null" - - "boolean" - node_id: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - locked: - type: - - "null" - - "boolean" - number: - type: - - "null" - - "integer" - node_id: - type: - - "null" - - "string" - user_id: - type: - - "null" - - "integer" - assignee: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - comments: - type: - - "null" - - "integer" - html_url: - type: - - "null" - - "string" - assignees: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - closed_at: - type: - - "null" - - "string" - format: "date-time" - milestone: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - due_on: - type: - - "null" - - "string" - format: "date-time" - number: - type: - - "null" - - "integer" - creator: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - closed_at: - type: - - "null" - - "string" - format: "date-time" - created_at: - type: - - "null" - - "string" - format: "date-time" - labels_url: - type: - - "null" - - "string" - updated_at: - type: - - "null" - - "string" - format: "date-time" - description: - type: - - "null" - - "string" - open_issues: - type: - - "null" - - "integer" - closed_issues: - type: - - "null" - - "integer" - created_at: - type: - - "null" - - "string" - format: "date-time" - events_url: - type: - - "null" - - "string" - labels_url: - type: - - "null" - - "string" - repository: - type: - - "string" - updated_at: - type: - - "null" - - "string" - format: "date-time" - comments_url: - type: - - "null" - - "string" - pull_request: - type: - - "null" - - "object" - properties: - url: - type: - - "null" - - "string" - diff_url: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - patch_url: - type: - - "null" - - "string" - repository_url: - type: - - "null" - - "string" - active_lock_reason: - type: - - "null" - - "string" - author_association: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -1339,206 +98,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "organizations" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - blog: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - plan: - type: - - "null" - - "object" - properties: - name: - type: - - "null" - - "string" - seats: - type: - - "null" - - "integer" - space: - type: - - "null" - - "integer" - filled_seats: - type: - - "null" - - "integer" - private_repos: - type: - - "null" - - "integer" - type: - type: - - "null" - - "string" - email: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - company: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - location: - type: - - "null" - - "string" - followers: - type: - - "null" - - "integer" - following: - type: - - "null" - - "integer" - hooks_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - disk_usage: - type: - - "null" - - "integer" - events_url: - type: - - "null" - - "string" - issues_url: - type: - - "null" - - "string" - updated_at: - type: - - "null" - - "string" - format: "date-time" - description: - type: - - "null" - - "string" - is_verified: - type: - - "null" - - "boolean" - members_url: - type: - - "null" - - "string" - public_gists: - type: - - "null" - - "integer" - public_repos: - type: - - "null" - - "integer" - billing_email: - type: - - "null" - - "string" - collaborators: - type: - - "null" - - "integer" - private_gists: - type: - - "null" - - "integer" - twitter_username: - type: - - "null" - - "string" - public_members_url: - type: - - "null" - - "string" - owned_private_repos: - type: - - "null" - - "integer" - total_private_repos: - type: - - "null" - - "integer" - has_repository_projects: - type: - - "null" - - "boolean" - members_can_create_pages: - type: - - "null" - - "boolean" - has_organization_projects: - type: - - "null" - - "boolean" - default_repository_permission: - type: - - "null" - - "string" - two_factor_requirement_enabled: - type: - - "null" - - "boolean" - members_can_create_public_pages: - type: - - "null" - - "boolean" - members_can_create_repositories: - type: - - "null" - - "boolean" - members_can_create_private_pages: - type: - - "null" - - "boolean" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -1546,900 +110,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "pull_requests" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - base: - type: - - "null" - - "object" - properties: - ref: - type: - - "null" - - "string" - sha: - type: - - "null" - - "string" - label: - type: - - "null" - - "string" - repo_id: - type: - - "null" - - "integer" - user_id: - type: - - "null" - - "integer" - body: - type: - - "null" - - "string" - head: - type: - - "null" - - "object" - properties: - ref: - type: - - "null" - - "string" - sha: - type: - - "null" - - "string" - label: - type: - - "null" - - "string" - repo_id: - type: - - "null" - - "integer" - user_id: - type: - - "null" - - "integer" - user: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - draft: - type: - - "null" - - "boolean" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - _links: - type: - - "null" - - "object" - properties: - html: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - self: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - issue: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - commits: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - comments: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - statuses: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - review_comment: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - review_comments: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - labels: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - color: - type: - - "null" - - "string" - default: - type: - - "null" - - "boolean" - node_id: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - locked: - type: - - "null" - - "boolean" - number: - type: - - "null" - - "integer" - node_id: - type: - - "null" - - "string" - assignee: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - diff_url: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - assignees: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - closed_at: - type: - - "null" - - "string" - format: "date-time" - issue_url: - type: - - "null" - - "string" - merged_at: - type: - - "null" - - "string" - format: "date-time" - milestone: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - due_on: - type: - - "null" - - "string" - format: "date-time" - number: - type: - - "null" - - "integer" - creator: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - closed_at: - type: - - "null" - - "string" - format: "date-time" - created_at: - type: - - "null" - - "string" - format: "date-time" - labels_url: - type: - - "null" - - "string" - updated_at: - type: - - "null" - - "string" - format: "date-time" - description: - type: - - "null" - - "string" - open_issues: - type: - - "null" - - "integer" - closed_issues: - type: - - "null" - - "integer" - patch_url: - type: - - "null" - - "string" - auto_merge: - type: - - "null" - - "object" - properties: - enabled_by: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - commit_title: - type: - - "null" - - "string" - merge_method: - type: - - "null" - - "string" - commit_message: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - repository: - type: - - "string" - updated_at: - type: - - "null" - - "string" - format: "date-time" - commits_url: - type: - - "null" - - "string" - comments_url: - type: - - "null" - - "string" - statuses_url: - type: - - "null" - - "string" - requested_teams: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - slug: - type: - - "null" - - "string" - parent: - type: - - "null" - - "object" - properties: {} - node_id: - type: - - "null" - - "string" - privacy: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - permission: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - members_url: - type: - - "null" - - "string" - repositories_url: - type: - - "null" - - "string" - merge_commit_sha: - type: - - "null" - - "string" - active_lock_reason: - type: - - "null" - - "string" - author_association: - type: - - "null" - - "string" - review_comment_url: - type: - - "null" - - "string" - requested_reviewers: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - review_comments_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -2451,224 +126,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "releases" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - body: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - draft: - type: - - "null" - - "boolean" - assets: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - size: - type: - - "null" - - "integer" - label: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - updated_at: - type: - - "null" - - "string" - format: "date-time" - uploader_id: - type: - - "null" - - "integer" - content_type: - type: - - "null" - - "string" - download_count: - type: - - "null" - - "integer" - browser_download_url: - type: - - "null" - - "string" - author: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - tag_name: - type: - - "null" - - "string" - assets_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - prerelease: - type: - - "null" - - "boolean" - repository: - type: - - "string" - upload_url: - type: - - "null" - - "string" - tarball_url: - type: - - "null" - - "string" - zipball_url: - type: - - "null" - - "string" - published_at: - type: - - "null" - - "string" - format: "date-time" - target_commitish: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -2680,427 +142,11 @@ sync_mode: "incremental" cursor_field: - "created_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "repositories" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - fork: - type: - - "null" - - "boolean" - name: - type: - - "null" - - "string" - size: - type: - - "null" - - "integer" - owner: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - topics: - type: - - "null" - - "array" - items: - type: - - "null" - - "string" - git_url: - type: - - "null" - - "string" - license: - type: - - "null" - - "object" - properties: - key: - type: - - "null" - - "string" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - spdx_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - private: - type: - - "null" - - "boolean" - ssh_url: - type: - - "null" - - "string" - svn_url: - type: - - "null" - - "string" - archived: - type: - - "null" - - "boolean" - disabled: - type: - - "null" - - "boolean" - has_wiki: - type: - - "null" - - "boolean" - homepage: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - keys_url: - type: - - "null" - - "string" - language: - type: - - "null" - - "string" - tags_url: - type: - - "null" - - "string" - blobs_url: - type: - - "null" - - "string" - clone_url: - type: - - "null" - - "string" - forks_url: - type: - - "null" - - "string" - full_name: - type: - - "null" - - "string" - has_pages: - type: - - "null" - - "boolean" - hooks_url: - type: - - "null" - - "string" - pulls_url: - type: - - "null" - - "string" - pushed_at: - type: - - "null" - - "string" - format: "date-time" - teams_url: - type: - - "null" - - "string" - trees_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - events_url: - type: - - "null" - - "string" - has_issues: - type: - - "null" - - "boolean" - issues_url: - type: - - "null" - - "string" - labels_url: - type: - - "null" - - "string" - merges_url: - type: - - "null" - - "string" - mirror_url: - type: - - "null" - - "string" - updated_at: - type: "string" - format: "date-time" - visibility: - type: - - "null" - - "string" - archive_url: - type: - - "null" - - "string" - commits_url: - type: - - "null" - - "string" - compare_url: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - forks_count: - type: - - "null" - - "integer" - is_template: - type: - - "null" - - "boolean" - permissions: - type: - - "null" - - "object" - properties: - pull: - type: - - "null" - - "boolean" - push: - type: - - "null" - - "boolean" - admin: - type: - - "null" - - "boolean" - branches_url: - type: - - "null" - - "string" - comments_url: - type: - - "null" - - "string" - contents_url: - type: - - "null" - - "string" - git_refs_url: - type: - - "null" - - "string" - git_tags_url: - type: - - "null" - - "string" - has_projects: - type: - - "null" - - "boolean" - releases_url: - type: - - "null" - - "string" - statuses_url: - type: - - "null" - - "string" - assignees_url: - type: - - "null" - - "string" - downloads_url: - type: - - "null" - - "string" - has_downloads: - type: - - "null" - - "boolean" - languages_url: - type: - - "null" - - "string" - default_branch: - type: - - "null" - - "string" - milestones_url: - type: - - "null" - - "string" - stargazers_url: - type: - - "null" - - "string" - watchers_count: - type: - - "null" - - "integer" - deployments_url: - type: - - "null" - - "string" - git_commits_url: - type: - - "null" - - "string" - subscribers_url: - type: - - "null" - - "string" - contributors_url: - type: - - "null" - - "string" - issue_events_url: - type: - - "null" - - "string" - stargazers_count: - type: - - "null" - - "integer" - subscription_url: - type: - - "null" - - "string" - collaborators_url: - type: - - "null" - - "string" - issue_comment_url: - type: - - "null" - - "string" - notifications_url: - type: - - "null" - - "string" - open_issues_count: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" - "incremental" @@ -3112,221 +158,11 @@ sync_mode: "full_refresh" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "review_comments" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - body: - type: - - "null" - - "string" - line: - type: - - "null" - - "integer" - path: - type: - - "null" - - "string" - side: - type: - - "null" - - "string" - user: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - _links: - type: - - "null" - - "object" - properties: - html: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - self: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - pull_request: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - position: - type: - - "null" - - "integer" - commit_id: - type: - - "null" - - "string" - diff_hunk: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - repository: - type: - - "string" - start_line: - type: - - "null" - - "integer" - start_side: - type: - - "null" - - "string" - updated_at: - type: - - "null" - - "string" - format: "date-time" - original_line: - type: - - "null" - - "integer" - in_reply_to_id: - type: - - "null" - - "integer" - pull_request_url: - type: - - "null" - - "string" - original_position: - type: - - "null" - - "integer" - author_association: - type: - - "null" - - "string" - original_commit_id: - type: - - "null" - - "string" - original_start_line: - type: - - "null" - - "integer" - pull_request_review_id: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" - "incremental" @@ -3338,158 +174,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "reviews" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - body: - type: - - "null" - - "string" - user: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - _links: - type: - - "null" - - "object" - properties: - html: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - pull_request: - type: - - "null" - - "object" - properties: - href: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - commit_id: - type: - - "null" - - "string" - repository: - type: - - "string" - submitted_at: - type: - - "null" - - "string" - format: "date-time" - pull_request_url: - type: - - "null" - - "string" - author_association: - type: - - "null" - - "string" - pull_request_updated_at: - type: "string" - format: "date-time" supported_sync_modes: - "full_refresh" - "incremental" @@ -3501,47 +190,11 @@ sync_mode: "incremental" cursor_field: - "submitted_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "tags" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - name: - type: - - "null" - - "string" - commit: - type: - - "null" - - "object" - properties: - sha: - type: - - "null" - - "string" - url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - repository: - type: - - "string" - tarball_url: - type: - - "null" - - "string" - zipball_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -3550,94 +203,12 @@ - - "name" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "repository" - - "name" - stream: name: "users" - json_schema: - type: - - "null" - - "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - organization: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -3645,685 +216,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "workflow_runs" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - event: - type: - - "null" - - "string" - status: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - head_sha: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - jobs_url: - type: - - "null" - - "string" - logs_url: - type: - - "null" - - "string" - rerun_url: - type: - - "null" - - "string" - cancel_url: - type: - - "null" - - "string" - conclusion: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - repository: - type: "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - fork: - type: - - "null" - - "boolean" - name: - type: - - "null" - - "string" - owner: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - private: - type: - - "null" - - "boolean" - html_url: - type: - - "null" - - "string" - keys_url: - type: - - "null" - - "string" - tags_url: - type: - - "null" - - "string" - blobs_url: - type: - - "null" - - "string" - forks_url: - type: - - "null" - - "string" - full_name: - type: - - "null" - - "string" - hooks_url: - type: - - "null" - - "string" - pulls_url: - type: - - "null" - - "string" - teams_url: - type: - - "null" - - "string" - trees_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - issues_url: - type: - - "null" - - "string" - labels_url: - type: - - "null" - - "string" - merges_url: - type: - - "null" - - "string" - archive_url: - type: - - "null" - - "string" - commits_url: - type: - - "null" - - "string" - compare_url: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - branches_url: - type: - - "null" - - "string" - comments_url: - type: - - "null" - - "string" - contents_url: - type: - - "null" - - "string" - git_refs_url: - type: - - "null" - - "string" - git_tags_url: - type: - - "null" - - "string" - releases_url: - type: - - "null" - - "string" - statuses_url: - type: - - "null" - - "string" - assignees_url: - type: - - "null" - - "string" - downloads_url: - type: - - "null" - - "string" - languages_url: - type: - - "null" - - "string" - milestones_url: - type: - - "null" - - "string" - stargazers_url: - type: - - "null" - - "string" - deployments_url: - type: - - "null" - - "string" - git_commits_url: - type: - - "null" - - "string" - subscribers_url: - type: - - "null" - - "string" - contributors_url: - type: - - "null" - - "string" - issue_events_url: - type: - - "null" - - "string" - subscription_url: - type: - - "null" - - "string" - collaborators_url: - type: - - "null" - - "string" - issue_comment_url: - type: - - "null" - - "string" - notifications_url: - type: - - "null" - - "string" - run_number: - type: - - "null" - - "integer" - updated_at: - type: - - "null" - - "string" - head_branch: - type: - - "null" - - "string" - head_commit: - type: "object" - properties: - id: - type: - - "null" - - "string" - author: - type: "object" - properties: - name: - type: - - "null" - - "string" - email: - type: - - "null" - - "string" - message: - type: - - "null" - - "string" - tree_id: - type: - - "null" - - "string" - committer: - type: "object" - properties: - name: - type: - - "null" - - "string" - email: - type: - - "null" - - "string" - timestamp: - type: - - "null" - - "string" - run_attempt: - type: - - "null" - - "integer" - workflow_id: - type: - - "null" - - "integer" - workflow_url: - type: - - "null" - - "string" - artifacts_url: - type: - - "null" - - "string" - pull_requests: - type: "array" - items: {} - check_suite_id: - type: - - "null" - - "integer" - run_started_at: - type: - - "null" - - "string" - check_suite_url: - type: - - "null" - - "string" - head_repository: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - fork: - type: - - "null" - - "boolean" - name: - type: - - "null" - - "string" - owner: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "integer" - url: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - login: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - gists_url: - type: - - "null" - - "string" - repos_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - site_admin: - type: - - "null" - - "boolean" - gravatar_id: - type: - - "null" - - "string" - starred_url: - type: - - "null" - - "string" - followers_url: - type: - - "null" - - "string" - following_url: - type: - - "null" - - "string" - organizations_url: - type: - - "null" - - "string" - subscriptions_url: - type: - - "null" - - "string" - received_events_url: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - private: - type: - - "null" - - "boolean" - html_url: - type: - - "null" - - "string" - keys_url: - type: - - "null" - - "string" - tags_url: - type: - - "null" - - "string" - blobs_url: - type: - - "null" - - "string" - forks_url: - type: - - "null" - - "string" - full_name: - type: - - "null" - - "string" - hooks_url: - type: - - "null" - - "string" - pulls_url: - type: - - "null" - - "string" - teams_url: - type: - - "null" - - "string" - trees_url: - type: - - "null" - - "string" - events_url: - type: - - "null" - - "string" - issues_url: - type: - - "null" - - "string" - labels_url: - type: - - "null" - - "string" - merges_url: - type: - - "null" - - "string" - archive_url: - type: - - "null" - - "string" - commits_url: - type: - - "null" - - "string" - compare_url: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - branches_url: - type: - - "null" - - "string" - comments_url: - type: - - "null" - - "string" - contents_url: - type: - - "null" - - "string" - git_refs_url: - type: - - "null" - - "string" - git_tags_url: - type: - - "null" - - "string" - releases_url: - type: - - "null" - - "string" - statuses_url: - type: - - "null" - - "string" - assignees_url: - type: - - "null" - - "string" - downloads_url: - type: - - "null" - - "string" - languages_url: - type: - - "null" - - "string" - milestones_url: - type: - - "null" - - "string" - stargazers_url: - type: - - "null" - - "string" - deployments_url: - type: - - "null" - - "string" - git_commits_url: - type: - - "null" - - "string" - subscribers_url: - type: - - "null" - - "string" - contributors_url: - type: - - "null" - - "string" - issue_events_url: - type: - - "null" - - "string" - subscription_url: - type: - - "null" - - "string" - collaborators_url: - type: - - "null" - - "string" - issue_comment_url: - type: - - "null" - - "string" - notifications_url: - type: - - "null" - - "string" - check_suite_node_id: - type: - - "null" - - "string" - previous_attempt_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -4331,57 +228,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "workflows" - json_schema: - type: - - "null" - - "object" - $schema: "http://json-schema.org/draft-04/schema#" - properties: - id: - type: "integer" - url: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - path: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - node_id: - type: - - "null" - - "string" - html_url: - type: - - "null" - - "string" - badge_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - repository: - type: "string" - updated_at: - type: "string" - format: "date-time" supported_sync_modes: - "full_refresh" - "incremental" @@ -4393,68 +244,18 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "gitlab_source__gitlab__" - sourceId: "59c74ca4-8cbb-4c65-8cb7-66bf771190fb" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "cef1b90d-ab16-4645-a0e3-b81818b8ffc7" - name: "default" +- prefix: "gitlab_source__gitlab__" + name: "GitLab - Faros" catalog: streams: - stream: name: "branches" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - name: - type: - - "null" - - "string" - merged: - type: - - "null" - - "boolean" - default: - type: - - "null" - - "boolean" - web_url: - type: - - "null" - - "string" - can_push: - type: - - "null" - - "boolean" - commit_id: - type: - - "null" - - "string" - protected: - type: - - "null" - - "boolean" - project_id: - type: - - "null" - - "integer" - developers_can_push: - type: - - "null" - - "boolean" - developers_can_merge: - type: - - "null" - - "boolean" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -4462,99 +263,11 @@ - - "name" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "name" - stream: name: "commits" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "string" - stats: - type: - - "null" - - "object" - properties: - total: - type: - - "null" - - "integer" - additions: - type: - - "null" - - "integer" - deletions: - type: - - "null" - - "integer" - title: - type: - - "null" - - "string" - message: - type: - - "null" - - "string" - web_url: - type: - - "null" - - "string" - short_id: - type: - - "null" - - "string" - trailers: - type: - - "null" - - "object" - created_at: - type: - - "null" - - "string" - format: "date-time" - parent_ids: - type: - - "null" - - "array" - items: - type: - - "null" - - "string" - project_id: - type: - - "null" - - "integer" - author_name: - type: - - "null" - - "string" - author_email: - type: - - "null" - - "string" - authored_date: - type: - - "null" - - "string" - format: "date-time" - committed_date: - type: - - "null" - - "string" - format: "date-time" - committer_name: - type: - - "null" - - "string" - committer_email: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -4566,59 +279,11 @@ sync_mode: "incremental" cursor_field: - "created_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "group_labels" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - name: - type: - - "null" - - "string" - color: - type: - - "null" - - "string" - group_id: - type: - - "null" - - "integer" - subscribed: - type: - - "null" - - "boolean" - text_color: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - description_html: - type: - - "null" - - "string" - open_issues_count: - type: - - "null" - - "integer" - closed_issues_count: - type: - - "null" - - "integer" - open_merge_requests_count: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -4626,57 +291,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "group_members" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - name: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - web_url: - type: - - "null" - - "string" - group_id: - type: - - "null" - - "integer" - username: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - expires_at: - type: - - "null" - - "string" - format: "date-time" - access_level: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -4684,67 +303,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "group_milestones" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - iid: - type: - - "null" - - "integer" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - expired: - type: - - "null" - - "boolean" - web_url: - type: - - "null" - - "string" - due_date: - type: - - "null" - - "string" - format: "date-time" - group_id: - type: - - "null" - - "integer" - created_at: - type: - - "null" - - "string" - format: "date-time" - start_date: - type: - - "null" - - "string" - format: "date-time" - updated_at: - type: - - "null" - - "string" - format: "date-time" - description: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -4752,155 +315,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "groups" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - name: - type: - - "null" - - "string" - path: - type: - - "null" - - "string" - ldap_cn: - type: - - "null" - - "string" - web_url: - type: - - "null" - - "string" - projects: - type: - - "null" - - "array" - items: - type: "object" - properties: - id: - type: - - "null" - - "integer" - path_with_namespace: - type: - - "null" - - "string" - full_name: - type: - - "null" - - "string" - full_path: - type: - - "null" - - "string" - parent_id: - type: - - "null" - - "integer" - avatar_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - visibility: - type: - - "null" - - "string" - - "integer" - - "boolean" - description: - type: - - "null" - - "string" - ldap_access: - type: - - "null" - - "string" - - "integer" - - "boolean" - lfs_enabled: - type: - - "null" - - "boolean" - runners_token: - type: - - "null" - - "string" - emails_disabled: - type: - - "null" - - "boolean" - shared_projects: - type: - - "null" - - "array" - mentions_disabled: - type: - - "null" - - "boolean" - shared_with_groups: - type: - - "null" - - "array" - auto_devops_enabled: - type: - - "null" - - "boolean" - share_with_group_lock: - type: - - "null" - - "boolean" - project_creation_level: - type: - - "null" - - "string" - request_access_enabled: - type: - - "null" - - "boolean" - subgroup_creation_level: - type: - - "null" - - "string" - two_factor_grace_period: - type: - - "null" - - "integer" - default_branch_protection: - type: - - "null" - - "integer" - shared_runners_minutes_limit: - type: - - "null" - - "integer" - prevent_forking_outside_group: - type: - - "null" - - "boolean" - require_two_factor_authentication: - type: - - "null" - - "boolean" - extra_shared_runners_minutes_limit: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -4908,171 +327,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "issues" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - iid: - type: - - "null" - - "integer" - type: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - _links: - type: - - "null" - - "object" - labels: - type: - - "null" - - "array" - items: - type: - - "null" - - "string" - weight: - type: - - "null" - - "integer" - upvotes: - type: - - "null" - - "integer" - web_url: - type: - - "null" - - "string" - due_date: - type: - - "null" - - "string" - assignees: - type: - - "null" - - "array" - items: - type: - - "null" - - "integer" - author_id: - type: - - "null" - - "integer" - closed_at: - type: - - "null" - - "string" - format: "date-time" - downvotes: - type: - - "null" - - "integer" - has_tasks: - type: - - "null" - - "boolean" - created_at: - type: - - "null" - - "string" - format: "date-time" - issue_type: - type: - - "null" - - "string" - project_id: - type: - - "null" - - "integer" - references: - type: - - "null" - - "object" - subscribed: - type: - - "null" - - "boolean" - time_stats: - type: - - "null" - - "object" - updated_at: - type: - - "null" - - "string" - format: "date-time" - assignee_id: - type: - - "null" - - "integer" - description: - type: - - "null" - - "string" - moved_to_id: - type: - - "null" - - "integer" - - "string" - task_status: - type: - - "null" - - "string" - closed_by_id: - type: - - "null" - - "integer" - confidential: - type: - - "null" - - "boolean" - milestone_id: - type: - - "null" - - "integer" - user_notes_count: - type: - - "null" - - "integer" - discussion_locked: - type: - - "null" - - "boolean" - merge_requests_count: - type: - - "null" - - "integer" - blocking_issues_count: - type: - - "null" - - "integer" - service_desk_reply_to: - type: - - "null" - - "string" - task_completion_status: - type: - - "null" - - "object" supported_sync_modes: - "full_refresh" - "incremental" @@ -5084,108 +343,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "jobs" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - ref: - type: - - "null" - - "string" - tag: - type: - - "null" - - "boolean" - name: - type: - - "null" - - "string" - stage: - type: - - "null" - - "string" - status: - type: - - "null" - - "string" - user_id: - type: - - "null" - - "integer" - web_url: - type: - - "null" - - "string" - coverage: - type: - - "null" - - "number" - - "string" - duration: - type: - - "null" - - "number" - tag_list: - type: - - "null" - - "array" - artifacts: - type: - - "null" - - "array" - commit_id: - type: - - "null" - - "string" - runner_id: - type: - - "null" - - "integer" - created_at: - type: - - "null" - - "string" - format: "date-time" - project_id: - type: - - "null" - - "integer" - started_at: - type: - - "null" - - "string" - format: "date-time" - finished_at: - type: - - "null" - - "string" - format: "date-time" - pipeline_id: - type: - - "null" - - "integer" - allow_failure: - type: - - "null" - - "boolean" - queued_duration: - type: - - "null" - - "number" - artifacts_expire_at: - type: - - "null" - - "string" - format: "date-time" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -5193,201 +355,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "merge_requests" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - iid: - type: - - "null" - - "integer" - sha: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - labels: - type: - - "null" - - "array" - items: - type: - - "null" - - "string" - squash: - type: - - "null" - - "boolean" - upvotes: - type: - - "null" - - "integer" - web_url: - type: - - "null" - - "string" - assignees: - type: - - "null" - - "array" - items: - type: - - "null" - - "integer" - author_id: - type: - - "null" - - "integer" - closed_at: - type: - - "null" - - "string" - format: "date-time" - downvotes: - type: - - "null" - - "integer" - merged_at: - type: - - "null" - - "string" - format: "date-time" - reference: - type: - - "null" - - "string" - reviewers: - type: - - "null" - - "array" - created_at: - type: - - "null" - - "string" - format: "date-time" - project_id: - type: - - "null" - - "integer" - references: - type: - - "null" - - "object" - time_stats: - type: - - "null" - - "object" - updated_at: - type: - - "null" - - "string" - format: "date-time" - assignee_id: - type: - - "null" - - "integer" - description: - type: - - "null" - - "string" - closed_by_id: - type: - - "null" - - "integer" - merge_status: - type: - - "null" - - "string" - merged_by_id: - type: - - "null" - - "integer" - milestone_id: - type: - - "null" - - "integer" - has_conflicts: - type: - - "null" - - "boolean" - source_branch: - type: - - "null" - - "string" - target_branch: - type: - - "null" - - "string" - merge_commit_sha: - type: - - "null" - - "string" - user_notes_count: - type: - - "null" - - "integer" - work_in_progress: - type: - - "null" - - "boolean" - discussion_locked: - type: - - "null" - - "boolean" - source_project_id: - type: - - "null" - - "integer" - squash_commit_sha: - type: - - "null" - - "string" - target_project_id: - type: - - "null" - - "integer" - approvals_before_merge: - type: - - "null" - - "boolean" - - "string" - - "object" - task_completion_status: - type: - - "null" - - "object" - force_remove_source_branch: - type: - - "null" - - "boolean" - should_remove_source_branch: - type: - - "null" - - "boolean" - merge_when_pipeline_succeeds: - type: - - "null" - - "boolean" - blocking_discussions_resolved: - type: - - "null" - - "boolean" supported_sync_modes: - "full_refresh" - "incremental" @@ -5399,49 +371,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "pipelines" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - ref: - type: - - "null" - - "string" - sha: - type: - - "null" - - "string" - status: - type: - - "null" - - "string" - web_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - project_id: - type: - - "null" - - "integer" - updated_at: - type: - - "null" - - "string" - format: "date-time" supported_sync_modes: - "full_refresh" - "incremental" @@ -5453,67 +387,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "project_labels" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - name: - type: - - "null" - - "string" - color: - type: - - "null" - - "string" - priority: - type: - - "null" - - "integer" - project_id: - type: - - "null" - - "integer" - subscribed: - type: - - "null" - - "boolean" - text_color: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - description_html: - type: - - "null" - - "string" - is_project_label: - type: - - "null" - - "boolean" - open_issues_count: - type: - - "null" - - "integer" - closed_issues_count: - type: - - "null" - - "integer" - open_merge_requests_count: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -5521,67 +399,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "project_milestones" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - iid: - type: - - "null" - - "integer" - state: - type: - - "null" - - "string" - title: - type: - - "null" - - "string" - expired: - type: - - "null" - - "boolean" - web_url: - type: - - "null" - - "string" - due_date: - type: - - "null" - - "string" - format: "date-time" - created_at: - type: - - "null" - - "string" - format: "date-time" - project_id: - type: - - "null" - - "integer" - start_date: - type: - - "null" - - "string" - format: "date-time" - updated_at: - type: - - "null" - - "string" - format: "date-time" - description: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -5589,488 +411,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "projects" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - name: - type: - - "null" - - "string" - path: - type: - - "null" - - "string" - _links: - type: "object" - properties: - self: - type: - - "null" - - "string" - events: - type: - - "null" - - "string" - issues: - type: - - "null" - - "string" - labels: - type: - - "null" - - "string" - members: - type: - - "null" - - "string" - repo_branches: - type: - - "null" - - "string" - merge_requests: - type: - - "null" - - "string" - topics: - type: - - "null" - - "array" - web_url: - type: - - "null" - - "string" - archived: - type: - - "null" - - "boolean" - tag_list: - type: - - "null" - - "array" - items: - type: - - "null" - - "string" - namespace: - type: "object" - properties: - id: - type: - - "null" - - "integer" - kind: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - path: - type: - - "null" - - "string" - web_url: - type: - - "null" - - "string" - full_path: - type: - - "null" - - "string" - parent_id: - type: - - "null" - - "integer" - avatar_url: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" - created_at: - type: - - "null" - - "string" - format: "date-time" - creator_id: - type: - - "null" - - "integer" - empty_repo: - type: - - "null" - - "boolean" - readme_url: - type: - - "null" - - "string" - star_count: - type: - - "null" - - "integer" - statistics: - type: "object" - properties: - wiki_size: - type: - - "null" - - "integer" - commit_count: - type: - - "null" - - "integer" - storage_size: - type: - - "null" - - "integer" - packages_size: - type: - - "null" - - "integer" - snippets_size: - type: - - "null" - - "integer" - repository_size: - type: - - "null" - - "integer" - lfs_objects_size: - type: - - "null" - - "integer" - job_artifacts_size: - type: - - "null" - - "integer" - visibility: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - forks_count: - type: - - "null" - - "integer" - lfs_enabled: - type: - - "null" - - "boolean" - permissions: - type: "object" - properties: - group_access: - type: - - "null" - - "object" - properties: - access_level: - type: - - "null" - - "integer" - notification_level: - type: - - "null" - - "integer" - project_access: - type: - - "null" - - "object" - properties: - access_level: - type: - - "null" - - "integer" - notification_level: - type: - - "null" - - "integer" - public_jobs: - type: - - "null" - - "boolean" - import_error: - type: - - "null" - - "string" - - "boolean" - jobs_enabled: - type: - - "null" - - "boolean" - merge_method: - type: - - "null" - - "string" - wiki_enabled: - type: - - "null" - - "boolean" - build_timeout: - type: - - "null" - - "integer" - import_status: - type: - - "null" - - "string" - runners_token: - type: - - "null" - - "string" - ci_config_path: - type: - - "null" - - "string" - default_branch: - type: - - "null" - - "string" - issues_enabled: - type: - - "null" - - "boolean" - emails_disabled: - type: - - "null" - - "boolean" - ssh_url_to_repo: - type: - - "null" - - "string" - http_url_to_repo: - type: - - "null" - - "string" - last_activity_at: - type: - - "null" - - "string" - format: "date-time" - packages_enabled: - type: - - "null" - - "boolean" - snippets_enabled: - type: - - "null" - - "boolean" - open_issues_count: - type: - - "null" - - "integer" - wiki_access_level: - type: - - "null" - - "string" - build_git_strategy: - type: - - "null" - - "string" - pages_access_level: - type: - - "null" - - "string" - shared_with_groups: - type: - - "null" - - "array" - auto_devops_enabled: - type: - - "null" - - "boolean" - builds_access_level: - type: - - "null" - - "string" - issues_access_level: - type: - - "null" - - "string" - name_with_namespace: - type: - - "null" - - "string" - path_with_namespace: - type: - - "null" - - "string" - build_coverage_regex: - type: - - "null" - - "string" - ci_default_git_depth: - type: - - "null" - - "integer" - forking_access_level: - type: - - "null" - - "string" - requirements_enabled: - type: - - "null" - - "boolean" - service_desk_address: - type: - - "null" - - "string" - service_desk_enabled: - type: - - "null" - - "boolean" - compliance_frameworks: - type: - - "null" - - "array" - snippets_access_level: - type: - - "null" - - "string" - analytics_access_level: - type: - - "null" - - "string" - merge_requests_enabled: - type: - - "null" - - "boolean" - request_access_enabled: - type: - - "null" - - "boolean" - shared_runners_enabled: - type: - - "null" - - "boolean" - operations_access_level: - type: - - "null" - - "string" - repository_access_level: - type: - - "null" - - "string" - suggestion_commit_message: - type: - - "null" - - "string" - container_registry_enabled: - type: - - "null" - - "boolean" - auto_devops_deploy_strategy: - type: - - "null" - - "string" - autoclose_referenced_issues: - type: - - "null" - - "boolean" - can_create_merge_request_in: - type: - - "null" - - "boolean" - container_expiration_policy: - type: "object" - properties: - keep_n: - type: - - "null" - - "integer" - cadence: - type: - - "null" - - "string" - enabled: - type: - - "null" - - "boolean" - name_regex: - type: - - "null" - - "string" - older_than: - type: - - "null" - - "string" - next_run_at: - type: - - "null" - - "string" - format: "date-time" - name_regex_keep: - type: - - "null" - - "string" - merge_requests_access_level: - type: - - "null" - - "string" - auto_cancel_pending_pipelines: - type: - - "null" - - "string" - ci_forward_deployment_enabled: - type: - - "null" - - "boolean" - allow_merge_on_skipped_pipeline: - type: - - "null" - - "boolean" - container_registry_image_prefix: - type: - - "null" - - "string" - restrict_user_defined_variables: - type: - - "null" - - "boolean" - security_and_compliance_enabled: - type: - - "null" - - "boolean" - remove_source_branch_after_merge: - type: - - "null" - - "boolean" - resolve_outdated_diff_discussions: - type: - - "null" - - "boolean" - printing_merge_request_link_enabled: - type: - - "null" - - "boolean" - only_allow_merge_if_pipeline_succeeds: - type: - - "null" - - "boolean" - external_authorization_classification_label: - type: - - "null" - - "string" - only_allow_merge_if_all_discussions_are_resolved: - type: - - "null" - - "boolean" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -6078,101 +423,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "releases" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - name: - type: - - "null" - - "string" - _links: - type: - - "null" - - "object" - assets: - type: - - "null" - - "object" - properties: - count: - type: - - "null" - - "integer" - links: - type: "array" - sources: - type: - - "null" - - "array" - items: - type: "object" - properties: - url: - type: - - "null" - - "string" - format: - type: - - "null" - - "string" - tag_name: - type: - - "null" - - "string" - tag_path: - type: - - "null" - - "string" - author_id: - type: - - "null" - - "integer" - commit_id: - type: - - "null" - - "string" - evidences: - type: - - "null" - - "array" - created_at: - type: - - "null" - - "string" - format: "date-time" - milestones: - type: - - "null" - - "array" - items: - type: "integer" - project_id: - type: - - "null" - - "integer" - commit_path: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - released_at: - type: - - "null" - - "string" - format: "date-time" - upcoming_release: - type: - - "null" - - "boolean" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -6180,52 +435,11 @@ - - "name" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "name" - stream: name: "tags" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - name: - type: - - "null" - - "string" - target: - type: - - "null" - - "string" - message: - type: - - "null" - - "string" - release: - type: - - "null" - - "object" - properties: - tag_name: - type: - - "null" - - "string" - description: - type: - - "null" - - "string" - commit_id: - type: - - "null" - - "string" - protected: - type: - - "null" - - "boolean" - project_id: - type: - - "null" - - "integer" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -6233,39 +447,11 @@ - - "name" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "name" - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "integer" - name: - type: - - "null" - - "string" - state: - type: - - "null" - - "string" - web_url: - type: - - "null" - - "string" - username: - type: - - "null" - - "string" - avatar_url: - type: - - "null" - - "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -6273,121 +459,18 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "bitbucket_source__bitbucket__" - sourceId: "5a19e927-51a2-4d5f-9b26-b35aba0910e0" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "2093cc9f-81d5-47df-8c14-d898c89f4c81" - name: "default" +- prefix: "bitbucket_source__bitbucket__" + name: "Bitbucket - Faros" catalog: streams: - stream: name: "branches" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - name: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - target: - type: "object" - properties: - date: - type: "string" - format: "date-time" - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - author: - type: "object" - properties: - raw: - type: "string" - type: - type: "string" - user: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - message: - type: "string" - parents: - type: "array" - items: - type: "object" - properties: - hash: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" - mergeStrategies: - type: "array" - items: - type: "string" - defaultMergeStrategy: - type: "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -6395,127 +478,11 @@ - - "name" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "name" - stream: name: "commits" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - date: - type: "string" - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - diffUrl: - type: "string" - format: "uri" - htmlUrl: - type: "string" - format: "uri" - approveUrl: - type: "string" - format: "uri" - commentsUrl: - type: "string" - format: "uri" - statusesUrl: - type: "string" - format: "uri" - author: - type: "object" - properties: - raw: - type: "string" - type: - type: "string" - user: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - message: - type: "string" - parents: - type: "array" - items: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - summary: - type: "object" - properties: - raw: - type: "string" - html: - type: "string" - type: - type: "string" - markup: - type: "string" - rendered: - type: "object" - properties: - message: - type: "object" - properties: - raw: - type: "string" - html: - type: "string" - type: - type: "string" - markup: - type: "string" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -6527,195 +494,11 @@ sync_mode: "incremental" cursor_field: - "date" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "hash" - stream: name: "deployments" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - key: - type: "string" - name: - type: "string" - step: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - type: - type: "string" - uuid: - type: "string" - state: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - status: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - startedOn: - type: "string" - format: "date-time" - completedOn: - type: "string" - format: "date-time" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - release: - type: "object" - properties: - key: - type: "string" - url: - type: "string" - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - pipeline: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - createdOn: - type: "string" - format: "date-time" - version: - type: "number" - deployable: - type: "object" - properties: - key: - type: "string" - url: - type: "string" - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - pipeline: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - createdOn: - type: "string" - format: "date-time" - environment: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - lastUpdateTime: - type: "string" - format: "date-time" - fullEnvironment: - type: "object" - properties: - lock: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - name: - type: "string" - rank: - type: "number" - slug: - type: "string" - type: - type: "string" - uuid: - type: "string" - hidden: - type: "boolean" - category: - type: "object" - properties: - name: - type: "string" - restrictions: - type: "object" - properties: - type: - type: "string" - adminOnly: - type: "boolean" - environmentType: - type: "object" - properties: - name: - type: "string" - rank: - type: "number" - type: - type: "string" - deploymentGateEnabled: - type: "boolean" - environmentLockEnabled: - type: "boolean" - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -6723,166 +506,11 @@ - - "uuid" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "uuid" - stream: name: "issues" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - id: - type: "number" - kind: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - voteUrl: - type: "string" - format: "uri" - watchUrl: - type: "string" - format: "uri" - commentsUrl: - type: "string" - format: "uri" - attachmentsUrl: - type: "string" - format: "uri" - state: - type: "string" - title: - type: "string" - votes: - type: "number" - content: - type: "object" - properties: - raw: - type: "string" - html: - type: "string" - type: - type: "string" - markup: - type: "string" - parents: - type: "array" - items: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - summary: - type: "object" - properties: - raw: - type: "string" - html: - type: "string" - type: - type: "string" - markup: - type: "string" - version: - type: - - "object" - - "null" - watches: - type: "number" - assignee: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - editedOn: - type: - - "object" - - "null" - priority: - type: "string" - reporter: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - component: - type: - - "object" - - "null" - createdOn: - type: "string" - milestone: - type: - - "object" - - "null" - updatedOn: - type: "string" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -6894,101 +522,11 @@ sync_mode: "incremental" cursor_field: - "updatedOn" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "pipeline_steps" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - image: - type: "object" - properties: - name: - type: "string" - state: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - result: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - maxTime: - type: "number" - trigger: - type: "object" - properties: - type: - type: "string" - pipeline: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - runNumber: - type: "number" - startedOn: - type: "string" - format: "date-time" - completedOn: - type: "string" - format: "date-time" - setupCommands: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - command: - type: "string" - commandType: - type: "string" - scriptCommands: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - command: - type: "string" - commandType: - type: "string" - buildSecondsUsed: - type: "number" - teardownCommands: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - action: - type: "string" - command: - type: "string" - commandType: - type: "string" - durationInSeconds: - type: "number" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -6996,129 +534,11 @@ - - "uuid" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "uuid" - stream: name: "pipelines" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - type: - type: "string" - uuid: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - state: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - stage: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - target: - type: "object" - properties: - hash: - type: "string" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - refName: - type: "string" - refType: - type: "string" - selector: - type: "object" - properties: - type: - type: "string" - creator: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - displayName: - type: "string" - expired: - type: "boolean" - trigger: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - createdOn: - type: "string" - format: "date-time" - runNumber: - type: "number" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" - buildNumber: - type: "number" - completedOn: - type: "string" - format: "date-time" - hasVariables: - type: "boolean" - firstSuccessful: - type: "boolean" - buildSecondsUsed: - type: "number" - durationInSeconds: - type: "number" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -7126,285 +546,11 @@ - - "uuid" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "uuid" - stream: name: "pull_request_activities" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - update: - type: "object" - properties: - date: - type: "string" - state: - type: "string" - title: - type: "string" - author: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - reason: - type: "string" - source: - type: "object" - properties: - branch: - type: "object" - properties: - name: - type: "string" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" - changes: - type: "object" - properties: - status: - type: "object" - properties: - new: - type: "string" - old: - type: "string" - reviewers: - type: "array" - description: - type: "string" - destination: - type: "object" - properties: - branch: - type: "object" - properties: - name: - type: "string" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" - mergeCommit: - type: - - "array" - - "null" - items: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - comment: - type: "object" - properties: - id: - type: "number" - type: - type: "string" - user: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - content: - type: "object" - properties: - raw: - type: "string" - html: - type: "string" - type: - type: "string" - markup: - type: "string" - deleted: - type: "boolean" - createdOn: - type: "string" - updatedOn: - type: "string" - pullrequest: - type: "object" - properties: - id: - type: "number" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - title: - type: "string" - approval: - type: "object" - properties: - date: - type: "string" - user: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - pullRequest: - type: "object" - properties: - id: - type: "number" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - title: - type: "string" - pullRequest: - type: "object" - properties: - id: - type: "number" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - title: - type: "string" - workspace: - type: "string" - repositorySlug: - type: "string" - pullRequestUpdatedOn: - type: "string" - additionalProperties: true supported_sync_modes: - "full_refresh" - "incremental" @@ -7415,236 +561,10 @@ sync_mode: "incremental" cursor_field: - "pullRequestUpdatedOn" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: [] - stream: name: "pull_requests" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - id: - type: "number" - type: - type: "string" - links: - type: "object" - properties: - diffUrl: - type: "string" - format: "uri" - htmlUrl: - type: "string" - format: "uri" - mergeUrl: - type: "string" - format: "uri" - approveUrl: - type: "string" - format: "uri" - commitsUrl: - type: "string" - format: "uri" - declineUrl: - type: "string" - format: "uri" - activityUrl: - type: "string" - format: "uri" - commentsUrl: - type: "string" - format: "uri" - diffstatUrl: - type: "string" - format: "uri" - statusesUrl: - type: "string" - format: "uri" - state: - type: "string" - title: - type: "string" - author: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - reason: - type: "string" - source: - type: "object" - properties: - branch: - type: "object" - properties: - name: - type: "string" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" - summary: - type: "object" - properties: - raw: - type: "string" - html: - type: "string" - type: - type: "string" - markup: - type: "string" - closedBy: - type: - - "object" - - "null" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - diffStat: - type: "object" - properties: - linesAdded: - type: "number" - filesChanged: - type: "number" - linesDeleted: - type: "number" - createdOn: - type: "string" - taskCount: - type: - - "number" - updatedOn: - type: "string" - description: - type: "string" - destination: - type: "object" - properties: - branch: - type: "object" - properties: - name: - type: "string" - commit: - type: "object" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - repository: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - fullName: - type: "string" - mergeCommit: - type: - - "object" - - "null" - properties: - hash: - type: "string" - type: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - commentCount: - type: "number" - closeSourceBranch: - type: "boolean" - calculatedActivity: - type: "object" - properties: - mergedAt: - type: "string" - commitCount: - type: "number" - additionalItems: true - additionalProperties: true supported_sync_modes: - "full_refresh" - "incremental" @@ -7656,1118 +576,48 @@ sync_mode: "incremental" cursor_field: - "updatedOn" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "repositories" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - scm: - type: "string" - name: - type: "string" - size: - type: "number" - slug: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - branchesUrl: - type: "string" - format: "uri" - owner: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - username: - type: "string" - displayName: - type: "string" - hasWiki: - type: "boolean" - project: - type: "object" - properties: - key: - type: "string" - name: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - website: - type: - - "null" - - "string" - fullName: - type: "string" - language: - type: "string" - createdOn: - type: "string" - format: "date-time" - hasIssues: - type: "boolean" - isPrivate: - type: "boolean" - updatedOn: - type: "string" - format: "date-time" - workspace: - type: "object" - properties: - name: - type: "string" - slug: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - forkPolicy: - type: "string" - mainBranch: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - description: - type: "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] source_defined_primary_key: [] sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: [] - stream: name: "workspace_users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - type: - type: "string" - user: - type: "object" - properties: - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - nickname: - type: "string" - accountId: - type: "string" - format: "uuid" - displayName: - type: "string" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - workspace: - type: "object" - properties: - name: - type: "string" - slug: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" supported_sync_modes: - "full_refresh" default_cursor_field: [] source_defined_primary_key: [] sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: [] - stream: name: "workspaces" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema" - properties: - name: - type: "string" - slug: - type: "string" - type: - type: "string" - uuid: - type: "string" - format: "uuid" - links: - type: "object" - properties: - htmlUrl: - type: "string" - format: "uri" - ownersUrl: - type: "string" - format: "uri" - repositoriesUrl: - type: "string" - format: "uri" - createdOn: - type: "string" - format: "date-time" - isPrivate: - type: "boolean" supported_sync_modes: - "full_refresh" default_cursor_field: [] source_defined_primary_key: [] sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: [] status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "phabricator_source__phabricator" - sourceId: "8e08224a-08da-4da4-95df-79639df68962" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "285b3dea-7fd5-4fb1-a088-0585eac66fa2" - name: "default" +- prefix: "phabricator_source__phabricator__" + name: "Phabricator - Faros" catalog: streams: - stream: name: "commits" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - required: - - "id" - - "type" - - "phid" - - "fields" - - "attachments" - - "repository" - properties: - id: - type: "integer" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "identifier" - - "repositoryPHID" - - "author" - - "committer" - - "isUnreachable" - - "isImported" - - "auditStatus" - - "message" - - "policy" - properties: - author: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - policy: - type: "object" - required: - - "view" - - "edit" - properties: - edit: - type: "string" - view: - type: "string" - message: - type: "string" - committer: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - identifier: - type: "string" - isImported: - type: "boolean" - auditStatus: - type: "object" - required: - - "value" - - "name" - - "closed" - - "color.ansi" - properties: - name: - type: "string" - value: - type: "string" - closed: - type: "boolean" - color.ansi: - type: "null" - isUnreachable: - type: "boolean" - repositoryPHID: - type: "string" - repository: - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - - "attachments" - properties: - id: - type: "integer" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "name" - - "vcs" - - "callsign" - - "shortName" - - "status" - - "isImporting" - - "almanacServicePHID" - - "refRules" - - "defaultBranch" - - "description" - - "spacePHID" - - "dateCreated" - - "dateModified" - - "policy" - properties: - vcs: - type: "string" - name: - type: "string" - policy: - type: "object" - required: - - "view" - - "edit" - - "diffusion.push" - properties: - edit: - type: "string" - view: - type: "string" - diffusion.push: - type: "string" - status: - type: "string" - callsign: - type: "null" - refRules: - type: "object" - required: - - "fetchRules" - - "trackRules" - - "permanentRefRules" - properties: - fetchRules: - type: "array" - items: {} - trackRules: - type: "array" - items: {} - permanentRefRules: - type: "array" - items: {} - shortName: - type: "string" - spacePHID: - type: "null" - dateCreated: - type: "integer" - description: - type: "object" - required: - - "raw" - properties: - raw: - type: "string" - isImporting: - type: "boolean" - dateModified: - type: "integer" - defaultBranch: - type: "string" - almanacServicePHID: - type: "null" - attachments: - type: "object" - required: - - "projects" - - "uris" - - "metrics" - properties: - uris: - type: "object" - required: - - "uris" - properties: - uris: - type: "array" - items: - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - metrics: - type: "object" - required: - - "commitCount" - - "recentCommit" - properties: - commitCount: - type: "integer" - recentCommit: - type: "object" - required: - - "identifier" - - "repositoryPHID" - - "author" - - "committer" - - "isUnreachable" - - "isImported" - - "auditStatus" - - "message" - properties: - author: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - message: - type: "string" - committer: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - identifier: - type: "string" - isImported: - type: "boolean" - auditStatus: - type: "object" - required: - - "value" - - "name" - - "closed" - - "color.ansi" - properties: - name: - type: "string" - value: - type: "string" - closed: - type: "boolean" - color.ansi: - type: "null" - isUnreachable: - type: "boolean" - repositoryPHID: - type: "string" - projects: - type: "object" - required: - - "projectPHIDs" - properties: - projectPHIDs: - type: "array" - items: - type: "string" - attachments: - type: "object" - required: - - "projects" - - "subscribers" - properties: - projects: - type: "object" - required: - - "projectPHIDs" - properties: - projectPHIDs: - type: "array" - items: - type: "string" - subscribers: - type: "object" - required: - - "subscriberPHIDs" - - "subscriberCount" - - "viewerIsSubscribed" - properties: - subscriberCount: - type: "integer" - subscriberPHIDs: - type: "array" - items: {} - viewerIsSubscribed: - type: "boolean" supported_sync_modes: - "full_refresh" - "incremental" @@ -8782,171 +632,10 @@ - "fields" - "committer" - "epoch" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: [] - stream: name: "projects" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - required: - - "id" - - "type" - - "phid" - - "fields" - - "attachments" - properties: - id: - type: "integer" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "name" - - "slug" - - "subtype" - - "milestone" - - "depth" - - "parent" - - "icon" - - "color" - - "spacePHID" - - "dateCreated" - - "dateModified" - - "policy" - - "description" - properties: - icon: - type: "object" - required: - - "key" - - "name" - - "icon" - properties: - key: - type: "string" - icon: - type: "string" - name: - type: "string" - name: - type: "string" - slug: - type: "string" - color: - type: "object" - required: - - "key" - - "name" - properties: - key: - type: "string" - name: - type: "string" - depth: - type: "integer" - parent: - type: "object" - required: - - "id" - - "phid" - - "name" - properties: - id: - type: "integer" - name: - type: "string" - phid: - type: "string" - policy: - type: "object" - required: - - "view" - - "edit" - - "join" - properties: - edit: - type: "string" - join: - type: "string" - view: - type: "string" - subtype: - type: "string" - milestone: - type: "null" - spacePHID: - type: "null" - dateCreated: - type: "integer" - description: - type: "null" - dateModified: - type: "integer" - attachments: - type: "object" - required: - - "members" - - "ancestors" - - "watchers" - properties: - members: - type: "object" - required: - - "members" - properties: - members: - type: "array" - items: - - type: "object" - required: - - "phid" - properties: - phid: - type: "string" - - type: "object" - required: - - "phid" - properties: - phid: - type: "string" - watchers: - type: "object" - required: - - "watchers" - properties: - watchers: - type: "array" - items: - - type: "object" - required: - - "phid" - properties: - phid: - type: "string" - ancestors: - type: "object" - required: - - "ancestors" - properties: - ancestors: - type: "array" - items: - - type: "object" - required: - - "id" - - "phid" - - "name" - properties: - id: - type: "integer" - name: - type: "string" - phid: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -8960,745 +649,11 @@ cursor_field: - "fields" - "dateModified" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "phid" - stream: name: "repositories" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - required: - - "id" - - "type" - - "phid" - - "fields" - - "attachments" - properties: - id: - type: "integer" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "name" - - "vcs" - - "callsign" - - "shortName" - - "status" - - "isImporting" - - "almanacServicePHID" - - "refRules" - - "defaultBranch" - - "description" - - "spacePHID" - - "dateCreated" - - "dateModified" - - "policy" - properties: - vcs: - type: "string" - name: - type: "string" - policy: - type: "object" - required: - - "view" - - "edit" - - "diffusion.push" - properties: - edit: - type: "string" - view: - type: "string" - diffusion.push: - type: "string" - status: - type: "string" - callsign: - type: "null" - refRules: - type: "object" - required: - - "fetchRules" - - "trackRules" - - "permanentRefRules" - properties: - fetchRules: - type: "array" - items: {} - trackRules: - type: "array" - items: {} - permanentRefRules: - type: "array" - items: {} - shortName: - type: "string" - spacePHID: - type: "null" - dateCreated: - type: "integer" - description: - type: "object" - required: - - "raw" - properties: - raw: - type: "string" - isImporting: - type: "boolean" - dateModified: - type: "integer" - defaultBranch: - type: "string" - almanacServicePHID: - type: "null" - attachments: - type: "object" - required: - - "projects" - - "uris" - - "metrics" - properties: - uris: - type: "object" - required: - - "uris" - properties: - uris: - type: "array" - items: - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - metrics: - type: "object" - required: - - "commitCount" - - "recentCommit" - properties: - commitCount: - type: "integer" - recentCommit: - type: "object" - required: - - "identifier" - - "repositoryPHID" - - "author" - - "committer" - - "isUnreachable" - - "isImported" - - "auditStatus" - - "message" - properties: - author: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - message: - type: "string" - committer: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - identifier: - type: "string" - isImported: - type: "boolean" - auditStatus: - type: "object" - required: - - "value" - - "name" - - "closed" - - "color.ansi" - properties: - name: - type: "string" - value: - type: "string" - closed: - type: "boolean" - color.ansi: - type: "null" - isUnreachable: - type: "boolean" - repositoryPHID: - type: "string" - projects: - type: "object" - required: - - "projectPHIDs" - properties: - projectPHIDs: - type: "array" - items: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -9712,880 +667,11 @@ cursor_field: - "fields" - "dateModified" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "phid" - stream: name: "revisions" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - required: - - "id" - - "type" - - "phid" - - "fields" - - "attachments" - - "repository" - properties: - id: - type: "integer" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "title" - - "uri" - - "authorPHID" - - "status" - - "repositoryPHID" - - "diffPHID" - - "summary" - - "testPlan" - - "isDraft" - - "holdAsDraft" - - "dateCreated" - - "dateModified" - - "policy" - properties: - uri: - type: "string" - title: - type: "string" - policy: - type: "object" - required: - - "view" - - "edit" - properties: - edit: - type: "string" - view: - type: "string" - status: - type: "object" - required: - - "value" - - "name" - - "closed" - - "color.ansi" - properties: - name: - type: "string" - value: - type: "string" - closed: - type: "boolean" - color.ansi: - type: "string" - isDraft: - type: "boolean" - summary: - type: "string" - diffPHID: - type: "string" - testPlan: - type: "string" - authorPHID: - type: "string" - dateCreated: - type: "integer" - holdAsDraft: - type: "boolean" - dateModified: - type: "integer" - repositoryPHID: - type: "string" - repository: - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - - "attachments" - properties: - id: - type: "integer" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "name" - - "vcs" - - "callsign" - - "shortName" - - "status" - - "isImporting" - - "almanacServicePHID" - - "refRules" - - "defaultBranch" - - "description" - - "spacePHID" - - "dateCreated" - - "dateModified" - - "policy" - properties: - vcs: - type: "string" - name: - type: "string" - policy: - type: "object" - required: - - "view" - - "edit" - - "diffusion.push" - properties: - edit: - type: "string" - view: - type: "string" - diffusion.push: - type: "string" - status: - type: "string" - callsign: - type: "null" - refRules: - type: "object" - required: - - "fetchRules" - - "trackRules" - - "permanentRefRules" - properties: - fetchRules: - type: "array" - items: {} - trackRules: - type: "array" - items: {} - permanentRefRules: - type: "array" - items: {} - shortName: - type: "string" - spacePHID: - type: "null" - dateCreated: - type: "integer" - description: - type: "object" - required: - - "raw" - properties: - raw: - type: "string" - isImporting: - type: "boolean" - dateModified: - type: "integer" - defaultBranch: - type: "string" - almanacServicePHID: - type: "null" - attachments: - type: "object" - required: - - "projects" - - "uris" - - "metrics" - properties: - uris: - type: "object" - required: - - "uris" - properties: - uris: - type: "array" - items: - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - - type: "object" - required: - - "id" - - "type" - - "phid" - - "fields" - properties: - id: - type: "string" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "repositoryPHID" - - "uri" - - "io" - - "display" - - "credentialPHID" - - "disabled" - - "builtin" - - "dateCreated" - - "dateModified" - properties: - io: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - uri: - type: "object" - required: - - "raw" - - "display" - - "effective" - - "normalized" - properties: - raw: - type: "string" - display: - type: "string" - effective: - type: "string" - normalized: - type: "string" - builtin: - type: "object" - required: - - "protocol" - - "identifier" - properties: - protocol: - type: "string" - identifier: - type: "string" - display: - type: "object" - required: - - "raw" - - "default" - - "effective" - properties: - raw: - type: "string" - default: - type: "string" - effective: - type: "string" - disabled: - type: "boolean" - dateCreated: - type: "string" - dateModified: - type: "string" - credentialPHID: - type: "null" - repositoryPHID: - type: "string" - metrics: - type: "object" - required: - - "commitCount" - - "recentCommit" - properties: - commitCount: - type: "integer" - recentCommit: - type: "object" - required: - - "identifier" - - "repositoryPHID" - - "author" - - "committer" - - "isUnreachable" - - "isImported" - - "auditStatus" - - "message" - properties: - author: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - message: - type: "string" - committer: - type: "object" - required: - - "name" - - "email" - - "raw" - - "epoch" - - "identityPHID" - - "userPHID" - properties: - raw: - type: "string" - name: - type: "string" - email: - type: "string" - epoch: - type: "integer" - userPHID: - type: "string" - identityPHID: - type: "string" - identifier: - type: "string" - isImported: - type: "boolean" - auditStatus: - type: "object" - required: - - "value" - - "name" - - "closed" - - "color.ansi" - properties: - name: - type: "string" - value: - type: "string" - closed: - type: "boolean" - color.ansi: - type: "null" - isUnreachable: - type: "boolean" - repositoryPHID: - type: "string" - projects: - type: "object" - required: - - "projectPHIDs" - properties: - projectPHIDs: - type: "array" - items: - type: "string" - attachments: - type: "object" - required: - - "projects" - - "subscribers" - - "reviewers" - properties: - projects: - type: "object" - required: - - "projectPHIDs" - properties: - projectPHIDs: - type: "array" - items: - type: "string" - reviewers: - type: "object" - required: - - "reviewers" - properties: - reviewers: - type: "array" - items: - - type: "object" - required: - - "reviewerPHID" - - "status" - - "isBlocking" - - "actorPHID" - properties: - status: - type: "string" - actorPHID: - type: "string" - isBlocking: - type: "boolean" - reviewerPHID: - type: "string" - subscribers: - type: "object" - required: - - "subscriberPHIDs" - - "subscriberCount" - - "viewerIsSubscribed" - properties: - subscriberCount: - type: "integer" - subscriberPHIDs: - type: "array" - items: - type: "string" - viewerIsSubscribed: - type: "boolean" supported_sync_modes: - "full_refresh" - "incremental" @@ -10599,61 +685,11 @@ cursor_field: - "fields" - "dateModified" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "phid" - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - required: - - "id" - - "type" - - "phid" - - "fields" - - "attachments" - properties: - id: - type: "integer" - phid: - type: "string" - type: - type: "string" - fields: - type: "object" - required: - - "username" - - "realName" - - "roles" - - "dateCreated" - - "dateModified" - - "policy" - properties: - roles: - type: "array" - items: - type: "string" - policy: - type: "object" - required: - - "view" - - "edit" - properties: - edit: - type: "string" - view: - type: "string" - realName: - type: "string" - username: - type: "string" - dateCreated: - type: "integer" - dateModified: - type: "integer" - attachments: - type: "object" supported_sync_modes: - "full_refresh" - "incremental" @@ -10667,86 +703,18 @@ cursor_field: - "fields" - "dateModified" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "phid" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "jira_source__jira__" - sourceId: "22852029-670c-4296-958e-c581fa76ae98" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "577ceecf-a92a-4785-b385-c41112d7f537" - name: "default" +- prefix: "jira_source__jira__" + name: "Jira - Faros" catalog: streams: - stream: name: "board_issues" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - key: - type: "string" - self: - type: "string" - expand: - type: "string" - fields: - type: "object" - properties: - epic: - type: - - "null" - - "object" - sprint: - type: - - "null" - - "object" - comment: - type: - - "null" - - "array" - items: - type: "object" - flagged: - type: - - "null" - - "boolean" - project: - type: - - "null" - - "object" - updated: - type: - - "null" - - "string" - format: "date-time" - worklog: - type: - - "null" - - "array" - items: - type: "object" - description: - type: - - "null" - - "string" - timetracking: - type: - - "null" - - "object" - closedSprints: - type: - - "null" - - "object" - boardId: - type: "integer" supported_sync_modes: - "full_refresh" - "incremental" @@ -10760,27 +728,11 @@ cursor_field: - "fields" - "updated" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "boards" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "integer" - name: - type: "string" - self: - type: "string" - type: - type: "string" - projectId: - type: "string" - projectKey: - type: "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -10788,70 +740,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "epics" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - readOnly: true - description: "The ID of the epic." - key: - type: "string" - readOnly: true - description: "The key of the epic." - self: - type: "string" - format: "uri" - readOnly: true - description: "The URL of the epic details." - expand: - xml: - attribute: true - type: "string" - readOnly: true - description: "Expand options that include additional issue details in\ - \ the response." - fields: - type: "object" - properties: - status: - type: - - "string" - - "object" - description: "Epic status" - summary: - type: - - "string" - - "null" - description: "Epic summary" - updated: - type: - - "string" - - "null" - format: "date-time" - description: "This field is not shown in schema / swagger, but exists\ - \ in records and we use it as cursor fiekd." - description: - type: - - "string" - - "null" - description: "Epic description" - additionalProperties: {} - projectId: - type: "string" - readOnly: true - description: "The ID of the project containing the epic." - projectKey: - type: "string" - readOnly: true - description: "The key of the project containing the epic." - additionalProperties: false supported_sync_modes: - "full_refresh" - "incremental" @@ -10865,170 +758,11 @@ cursor_field: - "fields" - "updated" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "issue_fields" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - description: "The ID of the field." - key: - type: "string" - description: "The key of the field." - name: - type: "string" - description: "The name of the field." - scope: - type: - - "object" - - "null" - properties: - type: - enum: - - "PROJECT" - - "TEMPLATE" - type: "string" - readOnly: true - description: "The type of scope." - project: - type: "object" - readOnly: true - properties: - id: - type: "string" - description: "The ID of the project." - key: - type: "string" - readOnly: true - description: "The key of the project." - name: - type: "string" - readOnly: true - description: "The name of the project." - self: - type: "string" - readOnly: true - description: "The URL of the project details." - avatarUrls: - type: "object" - readOnly: true - properties: - "16x16": - type: "string" - format: "uri" - description: "The URL of the item's 16x16 pixel avatar." - "24x24": - type: "string" - format: "uri" - description: "The URL of the item's 24x24 pixel avatar." - "32x32": - type: "string" - format: "uri" - description: "The URL of the item's 32x32 pixel avatar." - "48x48": - type: "string" - format: "uri" - description: "The URL of the item's 48x48 pixel avatar." - description: "The URLs of the project's avatars." - simplified: - type: "boolean" - readOnly: true - description: "Whether or not the project is simplified." - projectTypeKey: - enum: - - "software" - - "service_desk" - - "business" - type: "string" - readOnly: true - description: "The [project type](https://confluence.atlassian.com/x/GwiiLQ#Jiraapplicationsoverview-Productfeaturesandprojecttypes)\ - \ of the project." - projectCategory: - type: "object" - readOnly: true - properties: - id: - type: "string" - readOnly: true - description: "The ID of the project category." - name: - type: "string" - readOnly: true - description: "The description of the project category." - self: - type: "string" - readOnly: true - description: "The URL of the project category." - description: - type: "string" - readOnly: true - description: "The name of the project category." - description: "The category the project belongs to." - description: "The project the item has scope in." - description: "The scope of the field." - custom: - type: "boolean" - description: "Whether the field is a custom field." - schema: - type: "object" - properties: - type: - type: "string" - readOnly: true - description: "The data type of the field." - items: - type: "string" - readOnly: true - description: "When the data type is an array, the name of the field\ - \ items within the array." - custom: - type: "string" - readOnly: true - description: "If the field is a custom field, the URI of the field." - system: - type: "string" - readOnly: true - description: "If the field is a system field, the name of the field." - customId: - type: "integer" - format: "int64" - readOnly: true - description: "If the field is a custom field, the custom ID of the\ - \ field." - configuration: - type: "object" - readOnly: true - description: "If the field is a custom field, the configuration\ - \ of the field." - additionalProperties: - readOnly: true - description: "The data schema for the field." - navigable: - type: "boolean" - description: "Whether the field can be used as a column on the issue\ - \ navigator." - orderable: - type: "boolean" - description: "Whether the content of the field can be used to order\ - \ lists." - searchable: - type: "boolean" - description: "Whether the content of the field can be searched." - clauseNames: - type: "array" - items: - type: "string" - description: "The names that can be used to reference the field in an\ - \ advanced search. For more information, see [Advanced searching -\ - \ fields reference](https://confluence.atlassian.com/x/gwORLQ)." - uniqueItems: true - description: "Details about a field." - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -11036,111 +770,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "issues" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - readOnly: true - description: "The ID of the issue." - key: - type: "string" - readOnly: true - description: "The key of the issue." - self: - type: "string" - format: "uri" - readOnly: true - description: "The URL of the issue details." - names: - type: "object" - readOnly: true - description: "The ID and name of each field present on the issue." - additionalProperties: - type: "string" - readOnly: true - expand: - xml: - attribute: true - type: "string" - readOnly: true - description: "Expand options that include additional issue details in\ - \ the response." - fields: - type: "object" - properties: - created: - type: - - "string" - - "null" - format: "date-time" - description: "This field is not shown in schema / swagger, but exists\ - \ in records and we use it as cursor fiekd. Updated may be absent.\ - \ Added to solve the #4341" - updated: - type: - - "string" - - "null" - format: "date-time" - description: "This field is not shown in schema / swagger, but exists\ - \ in records and we use it as cursor fiekd. Updated may be absent.\ - \ Added to solve the #4341" - additionalProperties: {} - schema: - type: "object" - readOnly: true - description: "The schema describing each field present on the issue." - editmeta: - readOnly: true - description: "The metadata for the fields on the issue that can be amended." - changelog: - readOnly: true - description: "Details of changelogs associated with the issue." - projectId: - type: "string" - readOnly: true - description: "The ID of the project containing the issue." - operations: - readOnly: true - description: "The operations that can be performed on the issue." - projectKey: - type: "string" - readOnly: true - description: "The key of the project containing the issue." - properties: - type: "object" - readOnly: true - description: "Details of the issue properties identified in the request." - additionalProperties: - readOnly: true - transitions: - type: "array" - readOnly: true - description: "The transitions that can be performed on the issue." - renderedFields: - type: "object" - readOnly: true - description: "The rendered value of each field present on the issue." - additionalProperties: - readOnly: true - fieldsToInclude: - type: "object" - versionedRepresentations: - type: "object" - readOnly: true - description: "The versions of each field on the issue." - additionalProperties: - type: "object" - readOnly: true - additionalProperties: - readOnly: true - additionalProperties: false supported_sync_modes: - "full_refresh" - "incremental" @@ -11154,151 +788,11 @@ cursor_field: - "fields" - "updated" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "project_versions" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - readOnly: true - properties: - id: - type: "string" - readOnly: true - description: "The ID of the version." - name: - type: "string" - description: "The unique name of the version. Required when creating\ - \ a version. Optional when updating a version. The maximum length\ - \ is 255 characters." - self: - type: "string" - format: "uri" - readOnly: true - description: "The URL of the version." - expand: - xml: - attribute: true - type: - - "string" - - "null" - description: "Use [expand](em>#expansion) to include additional information\ - \ about version in the response. This parameter accepts a comma-separated\ - \ list. Expand options include:\n\n * `operations` Returns the list\ - \ of operations available for this version.\n * `issuesstatus` Returns\ - \ the count of issues in this version for each of the status categories\ - \ *to do*, *in progress*, *done*, and *unmapped*. The *unmapped* property\ - \ contains a count of issues with a status other than *to do*, *in\ - \ progress*, and *done*.\n\nOptional for create and update." - overdue: - type: "boolean" - readOnly: true - description: "Indicates that the version is overdue." - project: - type: "string" - description: "Deprecated. Use `projectId`." - archived: - type: "boolean" - description: "Indicates that the version is archived. Optional when\ - \ creating or updating a version." - released: - type: "boolean" - description: "Indicates that the version is released. If the version\ - \ is released a request to release again is ignored. Not applicable\ - \ when creating a version. Optional when updating a version." - projectId: - type: "integer" - format: "int64" - description: "The ID of the project to which this version is attached.\ - \ Required when creating a version. Not applicable when updating a\ - \ version." - startDate: - type: "string" - format: "date" - description: "The start date of the version. Expressed in ISO 8601 format\ - \ (yyyy-mm-dd). Optional when creating or updating a version." - operations: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - href: - type: "string" - label: - type: "string" - title: - type: "string" - weight: - type: "integer" - format: "int32" - iconClass: - type: "string" - styleClass: - type: "string" - readOnly: true - description: "If the expand option `operations` is used, returns the\ - \ list of operations available for this version." - description: - type: "string" - description: "The description of the version. Optional when creating\ - \ or updating a version." - releaseDate: - type: "string" - format: "date" - description: "The release date of the version. Expressed in ISO 8601\ - \ format (yyyy-mm-dd). Optional when creating or updating a version." - userStartDate: - type: "string" - readOnly: true - description: "The date on which work on this version is expected to\ - \ start, expressed in the instance's *Day/Month/Year Format* date\ - \ format." - userReleaseDate: - type: "string" - readOnly: true - description: "The date on which work on this version is expected to\ - \ finish, expressed in the instance's *Day/Month/Year Format* date\ - \ format." - moveUnfixedIssuesTo: - type: "string" - format: "uri" - description: "The URL of the self link to the version to which all unfixed\ - \ issues are moved when a version is released. Not applicable when\ - \ creating a version. Optional when updating a version." - issuesStatusForFixVersion: - type: "object" - readOnly: true - properties: - done: - type: "integer" - format: "int64" - readOnly: true - description: "Count of issues with status *done*." - toDo: - type: "integer" - format: "int64" - readOnly: true - description: "Count of issues with status *to do*." - unmapped: - type: "integer" - format: "int64" - readOnly: true - description: "Count of issues with a status other than *to do*,\ - \ *in progress*, and *done*." - inProgress: - type: "integer" - format: "int64" - readOnly: true - description: "Count of issues with status *in progress*." - description: "If the expand option `issuesstatus` is used, returns the\ - \ count of issues in this version for each of the status categories\ - \ *to do*, *in progress*, *done*, and *unmapped*. The *unmapped* property\ - \ contains a count of issues with a status other than *to do*, *in\ - \ progress*, and *done*." supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -11306,166 +800,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "projects" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - description: "The ID of the project." - key: - type: "string" - readOnly: true - description: "The key of the project." - url: - type: "string" - readOnly: true - description: "A link to information about this project, such as project\ - \ documentation." - lead: - readOnly: true - description: "The username of the project lead." - name: - type: "string" - readOnly: true - description: "The name of the project." - self: - type: "string" - format: "uri" - readOnly: true - description: "The URL of the project details." - uuid: - type: "string" - format: "uuid" - readOnly: true - description: "Unique ID for next-gen projects." - email: - type: "string" - description: "An email address associated with the project." - roles: - type: "object" - readOnly: true - description: "The name and self URL for each role defined in the project.\ - \ For more information, see [Create project role](#api-rest-api-3-role-post)." - additionalProperties: - type: "string" - format: "uri" - readOnly: true - style: - enum: - - "classic" - - "next-gen" - type: "string" - readOnly: true - description: "The type of the project." - expand: - xml: - attribute: true - type: "string" - readOnly: true - description: "Expand options that include additional project details\ - \ in the response." - deleted: - type: "boolean" - readOnly: true - description: "Whether the project is marked as deleted." - insight: - readOnly: true - description: "Insights about the project." - archived: - type: "boolean" - readOnly: true - description: "Whether the project is archived." - versions: - type: "array" - readOnly: true - description: "The versions defined in the project. For more information,\ - \ see [Create version](#api-rest-api-3-version-post)." - deletedBy: - readOnly: true - description: "The user who marked the project as deleted." - favourite: - type: "boolean" - description: "Whether the project is selected as a favorite." - isPrivate: - type: "boolean" - readOnly: true - description: "Whether the project is private." - archivedBy: - readOnly: true - description: "The user who archived the project." - avatarUrls: - readOnly: true - description: "The URLs of the project's avatars." - components: - type: "array" - readOnly: true - description: "List of the components contained in the project." - issueTypes: - type: "array" - readOnly: true - description: "List of the issue types available in the project." - properties: - type: "object" - readOnly: true - description: "Map of project properties" - additionalProperties: - readOnly: true - simplified: - type: "boolean" - readOnly: true - description: "Whether the project is simplified." - deletedDate: - type: "string" - format: "date-time" - readOnly: true - description: "The date when the project was marked as deleted." - description: - type: "string" - readOnly: true - description: "A brief description of the project." - permissions: - readOnly: true - description: "User permissions on the project" - archivedDate: - type: "string" - format: "date-time" - readOnly: true - description: "The date when the project was archived." - assigneeType: - enum: - - "PROJECT_LEAD" - - "UNASSIGNED" - type: "string" - readOnly: true - description: "The default assignee when creating issues for this project." - projectTypeKey: - enum: - - "software" - - "service_desk" - - "business" - type: "string" - readOnly: true - description: "The [project type](https://confluence.atlassian.com/x/GwiiLQ#Jiraapplicationsoverview-Productfeaturesandprojecttypes)\ - \ of the project." - projectCategory: - readOnly: true - description: "The category the project belongs to." - retentionTillDate: - type: "string" - format: "date-time" - readOnly: true - description: "The date when the project is deleted permanently." - issueTypeHierarchy: - readOnly: true - description: "The issue type hierarchy for the project" - description: "Details about a project." - additionalProperties: false supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -11473,75 +812,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "pull_requests" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: - - "null" - - "string" - updated: - type: "string" - format: "date-time" - branches: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - _instance: - type: - - "null" - - "object" - properties: - id: - type: - - "null" - - "string" - name: - type: - - "null" - - "string" - type: - type: - - "null" - - "string" - baseUrl: - type: - - "null" - - "string" - typeName: - type: - - "null" - - "string" - singleInstance: - type: - - "null" - - "boolean" - pullRequests: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" - repositories: - type: - - "null" - - "array" - items: - type: - - "null" - - "object" supported_sync_modes: - "full_refresh" - "incremental" @@ -11553,75 +828,11 @@ sync_mode: "incremental" cursor_field: - "updated" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "sprint_issues" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - key: - type: "string" - self: - type: "string" - expand: - type: "string" - fields: - type: "object" - properties: - epic: - type: - - "null" - - "object" - sprint: - type: - - "null" - - "object" - comment: - type: - - "null" - - "array" - items: - type: "object" - flagged: - type: - - "null" - - "boolean" - project: - type: - - "null" - - "object" - updated: - type: - - "null" - - "string" - format: "date-time" - worklog: - type: - - "null" - - "array" - items: - type: "object" - description: - type: - - "null" - - "string" - timetracking: - type: - - "null" - - "object" - closedSprints: - type: - - "null" - - "object" - issueId: - type: "number" - sprintId: - type: "number" supported_sync_modes: - "full_refresh" - "incremental" @@ -11635,36 +846,11 @@ cursor_field: - "fields" - "updated" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "sprints" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "integer" - goal: - type: "string" - name: - type: "string" - self: - type: "string" - state: - type: "string" - endDate: - type: "string" - format: "date-time" - startDate: - type: "string" - format: "date-time" - completeDate: - type: "string" - format: "date-time" - originBoardId: - type: "integer" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -11672,284 +858,21 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "users" - json_schema: - xml: - name: "user" - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - key: - type: "string" - description: "This property is no longer available and will be removed\ - \ from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/)\ - \ for details." - name: - type: "string" - description: "This property is no longer available and will be removed\ - \ from the documentation soon. See the [deprecation notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-user-privacy-api-migration-guide/)\ - \ for details." - self: - type: "string" - format: "uri" - readOnly: true - description: "The URL of the user." - active: - type: "boolean" - readOnly: true - description: "Whether the user is active." - expand: - xml: - attribute: true - type: "string" - readOnly: true - description: "Expand options that include additional user details in\ - \ the response." - groups: - type: "object" - readOnly: true - properties: - size: - xml: - attribute: true - type: "integer" - format: "int32" - items: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - description: "The name of group." - self: - type: "string" - format: "uri" - readOnly: true - description: "The URL for these group details." - callback: - type: "object" - max-results: - xml: - name: "max-results" - attribute: true - type: "integer" - format: "int32" - pagingCallback: - type: "object" - description: "The groups that the user belongs to." - locale: - type: "string" - readOnly: true - description: "The locale of the user. Depending on the user’s privacy\ - \ setting, this may be returned as null." - timeZone: - type: "string" - readOnly: true - description: "The time zone specified in the user's profile. Depending\ - \ on the user’s privacy setting, this may be returned as null." - accountId: - type: "string" - maxLength: 128 - description: "The account ID of the user, which uniquely identifies\ - \ the user across all Atlassian products. For example, *5b10ac8d82e05b22cc7d4ef5*.\ - \ Required in requests." - avatarUrls: - type: "object" - readOnly: true - properties: - "16x16": - type: "string" - format: "uri" - description: "The URL of the item's 16x16 pixel avatar." - "24x24": - type: "string" - format: "uri" - description: "The URL of the item's 24x24 pixel avatar." - "32x32": - type: "string" - format: "uri" - description: "The URL of the item's 32x32 pixel avatar." - "48x48": - type: "string" - format: "uri" - description: "The URL of the item's 48x48 pixel avatar." - description: "The avatars of the user." - accountType: - enum: - - "atlassian" - - "app" - - "customer" - - "unknown" - type: "string" - readOnly: true - description: "The user account type. Can take the following values:\n\ - \n * `atlassian` regular Atlassian user account\n * `app` system\ - \ account used for Connect applications and OAuth to represent external\ - \ systems\n * `customer` Jira Service Desk account representing an\ - \ external service desk" - displayName: - type: "string" - readOnly: true - description: "The display name of the user. Depending on the user’s\ - \ privacy setting, this may return an alternative value." - emailAddress: - type: "string" - readOnly: true - description: "The email address of the user. Depending on the user’\ - s privacy setting, this may be returned as null." - applicationRoles: - type: "object" - readOnly: true - properties: - size: - xml: - attribute: true - type: "integer" - format: "int32" - items: - type: "array" - items: - type: "object" - properties: - key: - type: "string" - description: "The key of the application role." - name: - type: "string" - description: "The display name of the application role." - groups: - type: "array" - items: - type: "string" - description: "The groups associated with the application role." - uniqueItems: true - defined: - type: "boolean" - description: "Deprecated." - platform: - type: "boolean" - description: "Indicates if the application role belongs to\ - \ Jira platform (`jira-core`)." - userCount: - type: "integer" - format: "int32" - description: "The number of users counting against your license." - defaultGroups: - type: "array" - items: - type: "string" - description: "The groups that are granted default access for\ - \ this application role." - uniqueItems: true - numberOfSeats: - type: "integer" - format: "int32" - description: "The maximum count of users on your license." - remainingSeats: - type: "integer" - format: "int32" - description: "The count of users remaining on your license." - hasUnlimitedSeats: - type: "boolean" - selectedByDefault: - type: "boolean" - description: "Determines whether this application role should\ - \ be selected by default on user creation." - userCountDescription: - type: "string" - description: "The [type of users](https://confluence.atlassian.com/x/lRW3Ng)\ - \ being counted against your license." - callback: - type: "object" - max-results: - xml: - name: "max-results" - attribute: true - type: "integer" - format: "int32" - pagingCallback: - type: "object" - description: "The application roles the user is assigned to." - description: "A user with details as permitted by the user's Atlassian Account\ - \ privacy settings. However, be aware of these exceptions:\n\n * User\ - \ record deleted from Atlassian: This occurs as the result of a right\ - \ to be forgotten request. In this case, `displayName` provides an indication\ - \ and other parameters have default values or are blank (for example,\ - \ email is blank).\n * User record corrupted: This occurs as a results\ - \ of events such as a server import and can only happen to deleted users.\ - \ In this case, `accountId` returns *unknown* and all other parameters\ - \ have fallback values.\n * User record unavailable: This usually occurs\ - \ due to an internal service outage. In this case, all parameters have\ - \ fallback values." - additionalProperties: false supported_sync_modes: - "full_refresh" default_cursor_field: [] source_defined_primary_key: [] sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: [] - stream: name: "workflow_statuses" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - readOnly: true - description: "The ID of the status." - name: - type: "string" - readOnly: true - description: "The name of the status." - self: - type: "string" - readOnly: true - description: "The URL of the status." - iconUrl: - type: "string" - readOnly: true - description: "The URL of the icon used to represent the status." - description: - type: "string" - readOnly: true - description: "The description of the status." - statusCategory: - type: "object" - readOnly: true - properties: - id: - type: "integer" - format: "int64" - readOnly: true - description: "The ID of the status category." - key: - type: "string" - readOnly: true - description: "The key of the status category." - name: - type: "string" - readOnly: true - description: "The name of the status category." - self: - type: "string" - readOnly: true - description: "The URL of the status category." - colorName: - type: "string" - readOnly: true - description: "The name of the color used to represent the status\ - \ category." - description: "The category assigned to the status." - description: "A status." - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -11957,75 +880,18 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "buildkite_source__buildkite__" - sourceId: "407664f2-22b0-431d-bc36-b1be8a6aae96" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "0b3cbaa7-82b2-4171-a7c8-2d3ba8a014ce" - name: "default" +- prefix: "buildkite_source__buildkite__" + name: "Buildkite - Faros" catalog: streams: - stream: name: "builds" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - url: - type: "string" - uuid: - type: "string" - state: - type: "string" - commit: - type: "string" - cursor: - type: - - "string" - - "null" - number: - type: "number" - message: - type: "string" - pipeline: - type: "object" - properties: - slug: - type: "string" - repository: - type: "object" - properties: - url: - type: "string" - provider: - type: "object" - properties: - name: - type: "string" - organization: - type: "object" - properties: - slug: - type: "string" - createdAt: - type: "string" - startedAt: - type: - - "string" - - "null" - finishedAt: - type: - - "string" - - "null" - additionalProperties: true supported_sync_modes: - "full_refresh" - "incremental" @@ -12037,24 +903,11 @@ sync_mode: "incremental" cursor_field: - "createdAt" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "uuid" - stream: name: "organizations" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - name: - type: "string" - slug: - type: "string" - web_url: - type: "string" - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -12062,55 +915,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "pipelines" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - url: - type: "string" - name: - type: "string" - slug: - type: "string" - uuid: - type: "string" - cursor: - type: - - "string" - - "null" - createdAt: - type: - - "string" - - "null" - repository: - type: - - "object" - - "null" - properties: - url: - type: "string" - provider: - type: "object" - properties: - name: - type: "string" - description: - type: - - "string" - - "null" - organization: - type: "object" - properties: - slug: - type: "string" - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -12118,117 +927,18 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "circleci_source__circleci__" - sourceId: "81c73d90-cbff-465a-ab8d-8cde1be6e718" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "997e5c0f-f81b-45a9-8fbd-e7c514008f71" - name: "default" +- prefix: "circleci_source__circleci__" + name: "CircleCI - Faros" catalog: streams: - stream: name: "pipelines" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - vcs: - type: "object" - properties: - branch: - type: "string" - revision: - type: "string" - provider_name: - type: "string" - origin_repository_url: - type: "string" - target_repository_url: - type: "string" - state: - type: "string" - errors: - type: "array" - items: {} - number: - type: "number" - trigger: - type: "object" - properties: - type: - type: "string" - actor: - type: "object" - properties: - login: - type: "string" - avatar_url: - type: "string" - received_at: - type: "string" - workflows: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - jobs: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - name: - type: "string" - type: - type: "string" - status: - type: "string" - job_number: - type: "number" - started_at: - type: "string" - stopped_at: - type: "string" - dependencies: - type: "array" - items: {} - project_slug: - type: "string" - name: - type: "string" - status: - type: "string" - created_at: - type: "string" - started_by: - type: "string" - stopped_at: - type: "string" - pipeline_id: - type: "string" - project_slug: - type: "string" - pipeline_number: - type: "number" - created_at: - type: "string" - updated_at: - type: "string" - project_slug: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -12240,36 +950,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "projects" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - name: - type: "string" - slug: - type: "string" - vcs_info: - type: "object" - properties: - vcs_url: - type: "string" - provider: - type: "string" - default_branch: - type: "string" - organization_id: - type: "string" - organization_name: - type: "string" - organization_slug: - type: "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -12277,130 +962,18 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "harness_source__harness" - sourceId: "c78b0bb2-5fee-46f7-a043-63e498004776" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "1381ff92-865d-473c-9c97-20b7e81afa78" - name: "default" +- prefix: "harness_source__harness__" + name: "Harness - Faros" catalog: streams: - stream: name: "executions" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - status: - type: "string" - endedAt: - type: "number" - outcomes: - type: "object" - properties: - nodes: - type: "array" - items: - type: "object" - properties: - service: - type: "object" - properties: - id: - type: "string" - artifactType: - type: "string" - artifactSources: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - environment: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - artifacts: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - buildNo: - type: "string" - artifactSource: - type: "object" - properties: - id: - type: "string" - name: - type: "string" - createdAt: - type: "number" - startedAt: - type: "number" - application: - type: "object" - properties: - id: - type: "string" - name: - type: "string" - tags: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - value: - type: "string" - services: - type: "object" - properties: - nodes: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - artifactType: - type: "string" - artifactSources: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - environments: - type: "object" - properties: - nodes: - type: "array" - items: - type: "object" - properties: - name: - type: "string" - type: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -12412,75 +985,18 @@ sync_mode: "incremental" cursor_field: - "endedAt" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "jenkins_source__jenkins__" - sourceId: "55b0325e-4bc7-49b4-b230-36c5a67df068" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "bd8ead71-0911-4b72-9916-245624fc44c1" - name: "default" +- prefix: "jenkins_source__jenkins__" + name: "Jenkins - Faros" catalog: streams: - stream: name: "builds" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "_class" - - "building" - - "displayName" - - "duration" - - "fullDisplayName" - - "id" - - "number" - - "result" - - "timestamp" - - "url" - properties: - id: - type: "string" - url: - type: "string" - _class: - type: "string" - number: - type: "number" - result: - type: "string" - actions: - type: "array" - items: - type: "object" - properties: - _class: - type: "string" - lastBuiltRevision: - type: "object" - properties: - SHA1: - type: "string" - remoteUrls: - type: "array" - items: - type: "string" - building: - type: "boolean" - duration: - type: "number" - timestamp: - type: "number" - displayName: - type: "string" - fullDisplayName: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -12492,139 +1008,11 @@ sync_mode: "incremental" cursor_field: - "number" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "fullDisplayName" - stream: name: "jobs" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "_class" - - "fullName" - - "name" - - "url" - properties: - url: - type: "string" - name: - type: "string" - _class: - type: "string" - builds: - type: "array" - items: - type: "object" - required: - - "_class" - - "building" - - "displayName" - - "duration" - - "fullDisplayName" - - "id" - - "number" - - "result" - - "timestamp" - - "url" - properties: - id: - type: "string" - url: - type: "string" - _class: - type: "string" - number: - type: "number" - result: - type: "string" - actions: - type: "array" - items: - type: "object" - properties: - _class: - type: "string" - lastBuiltRevision: - type: "object" - properties: - SHA1: - type: "string" - remoteUrls: - type: "array" - items: - type: "string" - building: - type: "boolean" - duration: - type: "number" - timestamp: - type: "number" - displayName: - type: "string" - jobFullName: - type: "string" - fullName: - type: "string" - allBuilds: - type: "array" - items: - type: "object" - required: - - "_class" - - "building" - - "displayName" - - "duration" - - "fullDisplayName" - - "id" - - "number" - - "result" - - "timestamp" - - "url" - properties: - id: - type: "string" - url: - type: "string" - _class: - type: "string" - number: - type: "number" - result: - type: "string" - actions: - type: "array" - items: - type: "object" - properties: - _class: - type: "string" - lastBuiltRevision: - type: "object" - properties: - SHA1: - type: "string" - remoteUrls: - type: "array" - items: - type: "string" - building: - type: "boolean" - duration: - type: "number" - timestamp: - type: "number" - displayName: - type: "string" - jobFullName: - type: "string" - lastCompletedBuild: - type: "object" - required: - - "number" - properties: - number: - type: "number" supported_sync_modes: - "full_refresh" - "incremental" @@ -12636,142 +1024,18 @@ sync_mode: "incremental" cursor_field: - "url" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "fullName" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "datadog_source__datadog__" - sourceId: "7fc3063d-fb70-4cf7-bff3-69051b58e84c" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "d58ba853-baea-46b1-93e4-bf1771ee2e32" - name: "default" +- prefix: "datadog_source__datadog__" + name: "Datadog - Faros" catalog: streams: - stream: name: "incidents" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "id" - properties: - id: - type: "string" - description: "The incident's ID." - attributes: - type: "object" - required: - - "title" - properties: - title: - type: "string" - description: "The title of the incident, which summarizes what happened." - fields: - state: - type: "object" - properties: - type: - type: "string" - value: - type: - - "string" - - "null" - teams: - type: "object" - properties: - type: - type: "string" - value: - type: - - "string" - - "null" - summary: - type: "object" - properties: - type: - type: "string" - value: - type: - - "string" - - "null" - services: - type: "object" - properties: - type: - type: "string" - value: - type: - - "string" - - "null" - severity: - type: "object" - properties: - type: - type: "string" - value: - type: - - "string" - - "null" - root_cause: - type: "object" - properties: - type: - type: "string" - value: - type: - - "string" - - "null" - detection_method: - type: "object" - properties: - type: - type: "string" - value: - type: - - "string" - - "null" - created: - type: "string" - format: "date-time" - description: "Timestamp when the incident was created." - detected: - type: "string" - format: "date-time" - description: "Timestamp when the incident was detected." - modified: - type: "string" - format: "date-time" - description: "Timestamp when the incident was last modified." - publicId: - type: "number" - resolved: - type: "string" - format: "date-time" - description: "Timestamp when the incident's state was set to resolved." - relationships: - type: "object" - properties: - commanderUser: - type: "object" - properties: - id: - type: "string" - createdByUser: - type: "object" - properties: - id: - type: "string" - lastModifiedByUser: - type: "object" - properties: - properties: - id: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -12782,60 +1046,10 @@ sync_mode: "incremental" cursor_field: - "modified" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: [] - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "id" - properties: - id: - type: "string" - attributes: - type: "object" - properties: - icon: - type: "string" - name: - type: "string" - email: - type: "string" - title: - type: "string" - handle: - type: "string" - status: - type: "string" - disabled: - type: "boolean" - verified: - type: "boolean" - createdAt: - type: "string" - format: "date-time" - modifiedAt: - type: "string" - format: "date-time" - serviceAccount: - type: "boolean" - relationships: - type: "object" - properties: - org: - type: "object" - required: - - "data" - properties: - data: - type: "object" - required: - - "id" - properties: - id: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -12846,108 +1060,17 @@ sync_mode: "full_refresh" cursor_field: - "modifiedAt" - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: [] status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "opsgenie_source__opsgenie__" - sourceId: "6749ef2f-5dc7-4e5f-ad5d-74aac6ef49da" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "955e6bfc-3056-4b07-820d-0b5039ce69e4" - name: "default" +- prefix: "opsgenie_source__opsgenie__" + name: "OpsGenie - Faros" catalog: streams: - stream: name: "incidents" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - properties: - id: - type: "string" - tags: - type: "array" - items: - type: "string" - links: - type: "object" - properties: - api: - type: "string" - web: - type: "string" - status: - type: "string" - tinyId: - type: "string" - actions: - type: "array" - items: - type: "string" - message: - type: "string" - priority: - type: "string" - createdAt: - type: "string" - ownerTeam: - type: "string" - timelines: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - type: - type: "string" - actor: - type: "object" - properties: - name: - type: "string" - type: - type: "string" - group: - type: "string" - title: - type: "object" - properties: - type: - type: "string" - content: - type: "string" - hidden: - type: "boolean" - eventTime: - type: "string" - updatedAt: - type: "string" - responders: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - type: - type: "string" - description: - type: - - "string" - - "null" - extraProperties: - type: "object" - impactStartDate: - type: "string" - impactedServices: - type: "array" - items: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -12959,30 +1082,11 @@ sync_mode: "incremental" cursor_field: - "createdAt" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "teams" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - properties: - id: - type: "string" - name: - type: "string" - links: - type: "object" - properties: - api: - type: "string" - web: - type: "string" - description: - type: - - "string" - - "null" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -12990,65 +1094,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-04/schema#" - properties: - id: - type: "string" - role: - type: "object" - properties: - id: - type: "string" - name: - type: "string" - locale: - type: - - "string" - - "null" - blocked: - type: "boolean" - fullName: - type: "string" - timeZone: - type: - - "string" - - "null" - username: - type: "string" - verified: - type: "boolean" - createdAt: - type: "string" - userAddress: - type: "object" - properties: - city: - type: - - "string" - - "null" - line: - type: - - "string" - - "null" - state: - type: - - "string" - - "null" - country: - type: - - "string" - - "null" - zipCode: - type: - - "string" - - "null" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13056,78 +1106,18 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "pagerduty_source__pagerduty__" - sourceId: "5ed7412c-72d3-46a9-bc40-0e362ad437c6" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "8885a0f8-a0ae-4cce-84a4-4c8974691548" - name: "default" +- prefix: "pagerduty_source__pagerduty__" + name: "PagerDuty - Faros" catalog: streams: - stream: name: "incident_log_entries" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "id" - - "type" - - "summary" - - "self" - - "html_url" - - "created_at" - - "incident" - - "service" - - "event_details" - properties: - id: - type: "string" - self: - type: "string" - type: - type: "string" - service: - type: "object" - properties: - id: - type: "string" - self: - type: "string" - type: - type: "string" - summary: - type: "string" - html_url: - type: "string" - summary: - type: "string" - html_url: - type: "string" - incident: - type: "object" - properties: - id: - type: "string" - self: - type: "string" - type: - type: "string" - summary: - type: "string" - html_url: - type: "string" - created_at: - type: "string" - event_details: - type: "object" supported_sync_modes: - "full_refresh" - "incremental" @@ -13139,125 +1129,11 @@ sync_mode: "incremental" cursor_field: - "created_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "incidents" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "id" - - "type" - - "summary" - - "self" - - "html_url" - - "description" - - "status" - - "acknowledgements" - - "incident_key" - - "urgency" - - "title" - - "created_at" - - "service" - - "assignments" - - "last_status_change_at" - properties: - id: - type: "string" - self: - type: "string" - type: - type: "string" - title: - type: "string" - status: - type: "string" - service: - type: "object" - properties: - id: - type: "string" - self: - type: "string" - type: - type: "string" - summary: - type: "string" - html_url: - type: "string" - summary: - type: "string" - urgency: - type: "string" - html_url: - type: "string" - priority: - type: "object" - properties: - id: - type: "string" - name: - type: "string" - self: - type: "string" - type: - type: "string" - summary: - type: "string" - html_url: - type: "string" - description: - type: "string" - created_at: - type: "string" - assignments: - type: "array" - items: - type: "object" - properties: - at: - type: "string" - assignee: - type: "object" - properties: - id: - type: "string" - self: - type: "string" - type: - type: "string" - summary: - type: "string" - html_url: - type: "string" - description: - type: "string" - incident_key: - type: "string" - acknowledgements: - type: "array" - items: - type: "object" - properties: - at: - type: "string" - acknowledger: - type: "object" - properties: - id: - type: "string" - self: - type: "string" - type: - type: "string" - summary: - type: "string" - html_url: - type: "string" - last_status_change_at: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -13269,37 +1145,11 @@ sync_mode: "incremental" cursor_field: - "created_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "priorities_resource" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "id" - - "type" - - "summary" - - "self" - - "html_url" - - "description" - - "name" - properties: - id: - type: "string" - name: - type: "string" - self: - type: "string" - type: - type: "string" - summary: - type: "string" - html_url: - type: "string" - description: - type: "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13307,37 +1157,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "id" - - "type" - - "summary" - - "self" - - "html_url" - - "email" - - "name" - properties: - id: - type: "string" - name: - type: "string" - self: - type: "string" - type: - type: "string" - email: - type: "string" - summary: - type: "string" - html_url: - type: "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13345,116 +1169,18 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "squadcast_source__squadcast__" - sourceId: "4321e9f5-bb4d-4f6e-8625-0cb73d1cff16" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "7ca9314f-9654-460b-8b9a-87821f25f5ef" - name: "default" +- prefix: "squadcast_source__squadcast__" + name: "SquadCast - Faros" catalog: streams: - stream: name: "events" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - tags: - type: - - "object" - - "null" - patternProperties: - ".*": - color: - type: "string" - value: - type: "string" - additionalProperties: true - message: - type: "string" - payload: - type: "object" - properties: - tags: - type: - - "object" - - "null" - patternProperties: - ".": - color: - type: "string" - value: - type: "string" - metric: - type: "object" - properties: - pod: - type: "string" - time: - type: "string" - absolute: - type: "object" - properties: - unit: - type: "string" - threshold: - type: "number" - current_value: - type: "number" - relative: - type: "object" - properties: - threshold: - type: "number" - current_value: - type: "number" - additionalProperties: true - status: - type: "string" - message: - type: "string" - assignee: - type: "object" - properties: - id: - type: "string" - type: - type: "string" - event_id: - type: "string" - created_by: - type: "string" - description: - type: "string" - additionalProperties: true - description: - type: - - "string" - incident_id: - type: "string" - alert_source_id: - type: "string" - time_of_creation: - type: "string" - deduplication_reason: - type: "object" - properties: - time_window: - type: "string" - matched_event_id: - type: "string" - evaluated_expression: - type: "string" - additionalProperties: true - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13462,70 +1188,11 @@ - - "alert_source_id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "alert_source_id" - stream: name: "incidents" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - url: - type: "string" - logs: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - time: - type: "string" - action: - type: "string" - reason: - type: "string" - assignedTo: - type: "string" - additionalProperties: true - tags: - type: - - "object" - - "null" - patternProperties: - ".*": - color: - type: "string" - value: - type: "string" - title: - type: "string" - status: - type: "string" - service: - type: "string" - assignee: - type: "string" - tta (ms): - type: "number" - ttr (ms): - type: "number" - created_at: - type: "string" - description: - type: "string" - event_count: - type: "number" - resolved_at: - type: "string" - alert_source: - type: "string" - acknowledged_at: - type: "string" - additionalProperties: true supported_sync_modes: - "full_refresh" - "incremental" @@ -13537,52 +1204,11 @@ sync_mode: "incremental" cursor_field: - "created_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "services" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - name: - type: "string" - slug: - type: "string" - email: - type: "string" - owner: - type: "object" - properties: - id: - type: "string" - type: - type: "string" - api_key: - type: "string" - description: - type: "string" - on_maintenance: - type: "boolean" - organization_id: - type: "string" - escalation_policy: - type: "object" - properties: - id: - type: "string" - name: - type: "string" - slug: - type: "string" - description: - type: "string" - escalation_policy_id: - type: "string" - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13590,53 +1216,11 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - bio: - type: "string" - role: - type: "string" - email: - type: "string" - title: - type: "string" - contact: - type: "object" - properties: - dial_code: - type: "string" - phone_number: - type: "string" - role_id: - type: "string" - last_name: - type: "string" - time_zone: - type: "string" - first_name: - type: "string" - user_image: - type: "boolean" - email_verified: - type: "boolean" - phone_verified: - type: "boolean" - in_grace_period: - type: "boolean" - secondary_emails: - type: "array" - items: - type: "string" - additionalProperties: true supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13644,42 +1228,18 @@ - - "id" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "statuspage_source__statuspage__" - sourceId: "a778b201-49fe-43e4-a470-391b07115fab" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "3e16f930-5b66-4214-803e-39bc4a6413bb" - name: "default" +- prefix: "statuspage_source__statuspage__" + name: "Statuspage - Faros" catalog: streams: - stream: name: "incident_updates" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - body: - type: "string" - status: - type: "string" - created_at: - type: "string" - display_at: - type: "string" - updated_at: - type: "string" - incident_id: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -13691,58 +1251,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "incidents" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - name: - type: "string" - impact: - type: "string" - status: - type: "string" - page_id: - type: "string" - shortlink: - type: "string" - created_at: - type: "string" - updated_at: - type: "string" - resolved_at: - type: - - "string" - - "null" - monitoring_at: - type: - - "string" - - "null" - incident_updates: - type: "array" - items: - type: "object" - properties: - id: - type: "string" - body: - type: "string" - status: - type: "string" - created_at: - type: "string" - display_at: - type: "string" - updated_at: - type: "string" - incident_id: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -13754,29 +1267,11 @@ sync_mode: "incremental" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "id" - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - properties: - id: - type: "string" - email: - type: "string" - last_name: - type: "string" - created_at: - type: "string" - first_name: - type: "string" - updated_at: - type: "string" - organization_id: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -13788,102 +1283,18 @@ sync_mode: "full_refresh" cursor_field: - "updated_at" - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "id" status: "active" manual: true resourceRequirements: {} -- namespaceDefinition: "source" - namespaceFormat: "${SOURCE_NAMESPACE}" - prefix: "victorops_source__victorops__" - sourceId: "e1d032e0-3a8b-47dc-bac8-2786264b11f4" - destinationId: "ca307eaf-5bee-4b26-84d4-3a084e6ebbcd" - operationIds: [] - connectionId: "e039b3c9-3abf-4459-b69b-e177cb34873f" - name: "default" +- prefix: "victorops_source__victorops__" + name: "VictorOps - Faros" catalog: streams: - stream: name: "incidents" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "incidentNumber" - - "startTime" - - "transitions" - properties: - host: - type: "string" - service: - type: "string" - entityId: - type: "string" - startTime: - type: "string" - alertCount: - type: "number" - entityType: - type: "string" - pagedTeams: - type: "array" - items: - type: "string" - pagedUsers: - type: "array" - items: - type: "string" - entityState: - type: "string" - lastAlertId: - type: "string" - monitorName: - type: "string" - monitorType: - type: "string" - transitions: - type: "array" - items: - type: "object" - properties: - at: - type: "string" - by: - type: "string" - name: - type: "string" - message: - type: "string" - currentPhase: - type: "string" - incidentLink: - type: "string" - lastAlertTime: - type: "string" - pagedPolicies: - type: "array" - items: - type: "object" - properties: - team: - type: "object" - properties: - name: - type: "string" - slug: - type: "string" - policy: - type: "object" - properties: - name: - type: "string" - slug: - type: "string" - incidentNumber: - type: "string" - entityDisplayName: - type: "string" supported_sync_modes: - "full_refresh" - "incremental" @@ -13895,36 +1306,11 @@ sync_mode: "incremental" cursor_field: - "startTime" - destination_sync_mode: "append" + destination_sync_mode: "append_dedup" primary_key: - - "incidentNumber" - stream: name: "teams" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "name" - - "slug" - properties: - name: - type: "string" - slug: - type: "string" - version: - type: "number" - _selfUrl: - type: "string" - _adminsUrl: - type: "string" - _membersUrl: - type: "string" - memberCount: - type: "number" - _policiesUrl: - type: "string" - isDefaultTeam: - type: "boolean" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13932,35 +1318,11 @@ - - "slug" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "slug" - stream: name: "users" - json_schema: - type: "object" - $schema: "http://json-schema.org/draft-07/schema#" - required: - - "username" - properties: - email: - type: "string" - _selfUrl: - type: "string" - lastName: - type: "string" - username: - type: "string" - verified: - type: "boolean" - createdAt: - type: "string" - firstName: - type: "string" - displayName: - type: "string" - passwordLastUpdated: - type: "string" supported_sync_modes: - "full_refresh" default_cursor_field: [] @@ -13968,7 +1330,7 @@ - - "username" sync_mode: "full_refresh" cursor_field: [] - destination_sync_mode: "append" + destination_sync_mode: "overwrite" primary_key: - - "username" status: "active" diff --git a/init/resources/airbyte/workspace/airbyte_config/STANDARD_WORKSPACE.yaml b/init/resources/airbyte/workspace/airbyte_config/STANDARD_WORKSPACE.yaml deleted file mode 100644 index 30fefe65..00000000 --- a/init/resources/airbyte/workspace/airbyte_config/STANDARD_WORKSPACE.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- workspaceId: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - customerId: "65ebab11-e573-4735-9200-e518cb9179b1" - name: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - slug: "b740a9b4-048b-4500-b41b-038e14a3f2d5" - initialSetupComplete: true - anonymousDataCollection: false - news: false - securityUpdates: false - displaySetupWizard: false - tombstone: false - notifications: [] diff --git a/init/src/airbyte/init.ts b/init/src/airbyte/init.ts index 28e148d5..9a0aa2cd 100644 --- a/init/src/airbyte/init.ts +++ b/init/src/airbyte/init.ts @@ -2,20 +2,13 @@ import Analytics from 'analytics-node'; import retry from 'async-retry'; import axios, {AxiosInstance} from 'axios'; import {InvalidArgumentError, program} from 'commander'; -import fs from 'fs-extra'; -import handlebars from 'handlebars'; import {find} from 'lodash'; -import os from 'os'; -import pLimit from 'p-limit'; -import path from 'path'; import pino from 'pino'; -import tar from 'tar'; import util from 'util'; import {v4 as uuidv4, v5 as uuidv5} from 'uuid'; import {VError} from 'verror'; -import {BASE_RESOURCES_DIR} from '../config'; -import {DestinationDefinition, SourceDefinition} from './types'; +import {AirbyteInitV40} from './initv40'; const logger = pino({ name: 'airbyte-init', @@ -23,11 +16,6 @@ const logger = pino({ }); export const FAROS_DEST_REPO = 'farosai/airbyte-faros-destination'; -const WORKSPACE_TEMPLATE_DIR = path.join( - BASE_RESOURCES_DIR, - 'airbyte', - 'workspace' -); const UUID_NAMESPACE = 'bb229e18-eb5f-4309-a863-893cbec53758'; @@ -57,10 +45,6 @@ export class AirbyteInit { ); } - private static getDestinationTemplatePath(dir: string): string { - return path.join(dir, 'airbyte_config', 'DESTINATION_CONNECTION.yaml'); - } - static makeSegmentUser(): SegmentUser { const version = process.env.FAROS_INIT_VERSION || ''; const source = process.env.FAROS_START_SOURCE || ''; @@ -94,26 +78,35 @@ export class AirbyteInit { host, }); const fn = (callback: ((err: Error) => void) | undefined): void => { - analytics - .identify( - { - userId: segmentUser.userId, - traits: { - email: segmentUser.email, - version: segmentUser.version, - source: segmentUser.source, + try { + analytics + .identify( + { + userId: segmentUser.userId, + traits: { + email: segmentUser.email, + version: segmentUser.version, + source: segmentUser.source, + }, + }, + callback + ) + .track( + { + userId: segmentUser.userId, + event: 'Start', }, - }, - callback - ) - .track( - { - userId: segmentUser.userId, - event: 'Start', - }, - callback - ) - .flush(callback); + callback + ) + .flush(callback); + } catch (err) { + if (callback && err instanceof Error) { + callback(err); + logger.error( + `Failed to send identity and start event: ${err.message}` + ); + } + } }; return util @@ -141,48 +134,29 @@ export class AirbyteInit { if (workspace.initialSetupComplete) { logger.info(`Workspace ${workspaceId} is already set up`); + // TODO: force setup if (forceSetup) { - logger.info(`Re-setting up workspace ${workspaceId}`); - } else { - return; + throw new VError('Forced setup not supported'); } - } else { - logger.info(`Setting up workspace ${workspaceId}`); + return; // TODO: connector upgrades } + logger.info(`Setting up workspace ${workspaceId}`); - const tmpDir = await fs.mkdtemp( - path.join(os.tmpdir(), 'faros-ce-airbyte-init') + // TODO: connectors upgrades + const farosConnectorsVersion = await AirbyteInit.getLatestImageTag( + FAROS_DEST_REPO ); - await fs.copy(WORKSPACE_TEMPLATE_DIR, tmpDir, {recursive: true}); - const destTemplatePath = AirbyteInit.getDestinationTemplatePath(tmpDir); - const destTemplate = await fs.readFile(destTemplatePath, 'utf-8'); - await fs.writeFile( - destTemplatePath, - handlebars.compile(destTemplate)({ - hasura_admin_secret: hasuraAdminSecret, - hasura_url: airbyteDestinationHasuraUrl, - segment_user_id: segmentUser.userId, - }), - 'utf-8' - ); - const workspaceZipPath = path.join(tmpDir, 'workspace.tar.gz'); + logger.info('faros connectors version: ' + farosConnectorsVersion); + const airbyteInitV40: AirbyteInitV40 = new AirbyteInitV40(this.api); try { - await tar.create( - { - cwd: tmpDir, - file: workspaceZipPath, - gzip: true, - }, - ['VERSION', 'airbyte_config'] + await airbyteInitV40.init( + farosConnectorsVersion, + airbyteDestinationHasuraUrl, + hasuraAdminSecret, + segmentUser.userId ); - const buffer = await fs.readFile(workspaceZipPath); - await this.api.post('/deployment/import', buffer, { - headers: {'Content-Type': 'application/x-gzip'}, - }); } catch (error) { throw new VError(`Failed to set up workspace: ${error}`); - } finally { - await fs.remove(tmpDir); } } @@ -215,90 +189,6 @@ export class AirbyteInit { } return version; } - - async setupFarosDestinationDefinition(): Promise { - // const version = await AirbyteInit.getLatestImageTag(FAROS_DEST_REPO); - const version = '0.4.69'; // tmp for product hunt launch - const listResponse = await this.api.post('/destination_definitions/list'); - const farosDestDef = find( - listResponse.data.destinationDefinitions as DestinationDefinition[], - (dd) => dd.dockerRepository === FAROS_DEST_REPO - ); - - if (!farosDestDef) { - logger.info(`Adding Faros Destination ${version}`); - await this.api.post('/destination_definitions/create', { - name: 'Faros Destination', - dockerRepository: FAROS_DEST_REPO, - dockerImageTag: version, - documentationUrl: 'https://docs.faros.ai', - }); - } - // tmp for product hunt launch - /* else if (farosDestDef.dockerImageTag === version) { - logger.info('Faros Destination is already at %s', version); - } else { - logger.info( - 'Updating Faros Destination from %s to %s', - farosDestDef.dockerImageTag, - version - ); - await this.api.post('/destination_definitions/update', { - destinationDefinitionId: farosDestDef.destinationDefinitionId, - dockerImageTag: version, - }); - } - */ - } - - /** - * we update the sources we use in the CLI - * - * we do NOT update the other sources at startup - * those will use the version specified in the airbyte config - * and the binaries will be downloaded when source config is saved - * - * 12/20/22: we temporarily stop updating the Jira source after issues - * introduced by https://github.com/airbytehq/airbyte/pull/20128 - * - * 2/1/23: we temporarily stop updating all airbyte sources, as - * we are at the mercy of a incompatible spec change, e.g. - * gitlab spec from 0.1.6 to 1.0.2 (broke the CLI). - */ - async updateSelectSourceVersions(concurrency?: number): Promise { - const listResponse = await this.api.post('/source_definitions/list'); - const farosSourceDefs = ( - listResponse.data.sourceDefinitions as SourceDefinition[] - ).filter( - (sd) => sd.dockerRepository === 'farosai/airbyte-bitbucket-source' - ); - - const promises: Promise[] = []; - const limit = pLimit(concurrency || Number.POSITIVE_INFINITY); - - for (const sourceDef of farosSourceDefs) { - const version = await AirbyteInit.getLatestImageTag( - sourceDef.dockerRepository - ); - if (sourceDef.dockerImageTag !== version) { - logger.info( - 'Updating %s source from %s to %s', - sourceDef.dockerRepository, - sourceDef.dockerImageTag, - version - ); - promises.push( - limit(() => - this.api.post('/source_definitions/update', { - sourceDefinitionId: sourceDef.sourceDefinitionId, - dockerImageTag: version, - }) - ) - ); - } - } - await Promise.all(promises); - } } async function main(): Promise { @@ -328,17 +218,18 @@ async function main(): Promise { ); const segmentUser = AirbyteInit.makeSegmentUser(); + await AirbyteInit.sendIdentityAndStartEvent(segmentUser); await airbyte.waitUntilHealthy(); + await airbyte.setupWorkspace( segmentUser, options.hasuraAdminSecret, options.airbyteDestinationHasuraUrl, options.forceSetup ); - await airbyte.setupFarosDestinationDefinition(); - await airbyte.updateSelectSourceVersions(options.airbyteApiCallsConcurrency); + logger.info('Airbyte setup is complete'); } diff --git a/init/src/airbyte/initv40.ts b/init/src/airbyte/initv40.ts new file mode 100644 index 00000000..3c0c3fd3 --- /dev/null +++ b/init/src/airbyte/initv40.ts @@ -0,0 +1,429 @@ +import {AxiosInstance} from 'axios'; +import * as fs from 'fs'; +import * as yaml from 'js-yaml'; +import {Dictionary} from 'lodash'; +import path from 'path'; +import pino from 'pino'; + +const logger = pino({ + name: 'airbytev40-init', + level: process.env.LOG_LEVEL || 'info', +}); + +import {BASE_RESOURCES_DIR} from '../config'; +const CATALOGS = path.join( + BASE_RESOURCES_DIR, + 'airbyte', + 'workspace', + 'airbyte_config', + 'STANDARD_SYNC.yaml' +); +const SOURCES = path.join( + BASE_RESOURCES_DIR, + 'airbyte', + 'workspace', + 'airbyte_config', + 'SOURCE_CONNECTION.yaml' +); +// Function to load and parse a YAML file +function loadYamlFile(filePath: string): any { + try { + const fileContent = fs.readFileSync(filePath, 'utf-8'); + const data = yaml.load(fileContent); + return data; + } catch (error) { + console.error(`Error loading YAML file: ${error}`); + return null; + } +} + +// Function to find an entry with a specific attribute value +function findEntryWithAttributeValue( + data: any[], + attribute: string, + value: any +): any { + return data.find((entry) => entry[attribute] === value); +} + +function snakeCaseToCamelCase(snakeCaseStr: string): string { + return snakeCaseStr.replace(/_([a-z])/g, (match, letter) => + letter.toUpperCase() + ); +} + +function convertKeysToCamelCase(data: any): any { + if (Array.isArray(data)) { + return data.map(convertKeysToCamelCase); + } else if (data !== null && typeof data === 'object') { + const newData: {[key: string]: any} = {}; + for (const key in data) { + const camelCaseKey = snakeCaseToCamelCase(key); + newData[camelCaseKey] = convertKeysToCamelCase(data[key]); + } + return newData; + } + return data; +} + +export class AirbyteInitV40 { + private readonly api: AxiosInstance; + + constructor(api: AxiosInstance) { + this.api = api; + } + + async createWorkspace(params: {name: string}): Promise { + const response = await this.api.post('/workspaces/create', params); + return response.data.workspaceId as string; + } + + async getWorkspaceBySlug(params: {slug: string}): Promise { + const response = await this.api.post('/workspaces/get_by_slug', params); + return response.data.workspaceId as string; + } + + async getFirstWorkspace(): Promise { + const response = await this.api.post('/workspaces/list', {}); + return response.data.workspaces[0].workspaceId as string; + } + + // extra settings appear to be needed + async completeWorkspaceSetup(params: { + workspaceId: string; + initialSetupComplete: boolean; + anonymousDataCollection: boolean; + news: boolean; + securityUpdates: boolean; + }): Promise { + const response = await this.api.post('/workspaces/update', params); + return response.data.initialSetupComplete as string; + } + async createCustomSourceDefinition(params: { + workspaceId: string; + sourceDefinition: { + name: string; + dockerRepository: string; + dockerImageTag: string; + documentationUrl: string; + }; + }): Promise { + const response = await this.api.post( + '/source_definitions/create_custom', + params + ); + return response.data.sourceDefinitionId as string; + } + + async createCustomDestinationDefinition(params: { + workspaceId: string; + destinationDefinition: { + name: string; + dockerRepository: string; + dockerImageTag: string; + documentationUrl: string; + }; + }): Promise { + const response = await this.api.post( + '/destination_definitions/create_custom', + params + ); + return response.data.destinationDefinitionId as string; + } + + async createSource(params: { + sourceDefinitionId: string; + connectionConfiguration: Dictionary; + workspaceId: string; + name: string; + }): Promise { + const response = await this.api.post('/sources/create', params); + return response.data.sourceId as string; + } + + async createDestination(params: { + destinationDefinitionId: string; + connectionConfiguration: Dictionary; + workspaceId: string; + name: string; + }): Promise { + const response = await this.api.post('/destinations/create', params); + return response.data.destinationId as string; + } + + async listDestinationNames(params: {workspaceId: string}): Promise { + const response = await this.api.post('/destinations/list', params); + return response.data.destinations.map( + (destination: any) => destination.name + ) as string[]; + } + + async getCatalog(params: {sourceId: string}): Promise> { + const response = await this.api.post('/sources/discover_schema', params); + return response.data.catalog; + } + + async createConnection(params: { + name: string; + prefix: string; + sourceId: string; + destinationId: string; + syncCatalog: Dictionary; + status: string; + }): Promise { + const response = await this.api.post('/connections/create', params); + return response.data.connectionId as string; + } + + async completeFarosWorkspaceSetup(workspaceId: string): Promise { + return await this.completeWorkspaceSetup({ + workspaceId, + initialSetupComplete: true, + anonymousDataCollection: false, + news: false, + securityUpdates: false, + }); + } + + async createFarosDestinationDefinition( + workspaceId: string, + version: string + ): Promise { + return await this.createCustomDestinationDefinition({ + workspaceId, + destinationDefinition: { + name: 'Faros Destination', + dockerRepository: 'farosai/airbyte-faros-destination', + dockerImageTag: version, + documentationUrl: 'https://docs.faros.ai', + }, + }); + } + + async createFarosDestination( + workspaceId: string, + farosDestinationDefinitionId: string, + hasura_url: string, + hasura_admin_secret: string, + segment_user_id: string + ): Promise { + return await this.createDestination({ + workspaceId, + destinationDefinitionId: farosDestinationDefinitionId, + name: 'Faros Destination', + connectionConfiguration: { + dry_run: false, + jsonata_mode: 'FALLBACK', + edition_configs: { + edition: 'community', + hasura_url, + hasura_admin_secret, + segment_user_id, + }, + invalid_record_strategy: 'SKIP', + }, + }); + } + + async getFarosDestinationId(workspaceId: string): Promise { + return (await this.listDestinationNames({workspaceId})).filter( + (name: string) => name === 'Faros Destination' + )[0]; + } + + async createSourceFromYAML( + workspaceId: string, + yamlData: any, + sourceName: string, + sourceDefinitionId: string + ): Promise { + const source = findEntryWithAttributeValue(yamlData, 'name', sourceName); + + return await this.createSource({ + workspaceId, + sourceDefinitionId, + name: source.name, + connectionConfiguration: source.configuration, + }); + } + + async createConnectionToFaros( + sourceId: string, + farosDestinationId: string, + yamlData: any, + connectionName: string + ): Promise { + const connection = findEntryWithAttributeValue( + yamlData, + 'name', + connectionName + ); + const streams: any[] = connection.catalog.streams; + const streamsWithConfig = streams.map((stream) => { + const streamWithConfig = {...stream}; + streamWithConfig.config = { + syncMode: stream.syncMode, + cursorField: stream.cursorField, + destinationSyncMode: stream.destinationSyncMode, + primaryKey: stream.primaryKey, + selected: true, + }; + + delete streamWithConfig.syncMode; + delete streamWithConfig.cursorField; + delete streamWithConfig.destinationSyncMode; + delete streamWithConfig.primaryKey; + // removing it completely causes the sync to not start + streamWithConfig.stream.jsonSchema = {}; + return streamWithConfig; + }); + + return await this.createConnection({ + name: connection.name, + sourceId, + destinationId: farosDestinationId, + syncCatalog: { + streams: streamsWithConfig, + }, + prefix: connection.prefix, + status: connection.status, + }); + } + + async handleFarosSource( + name: string, + workspaceId: string, + farosDestinationId: string, + farosConnectorsVersion: string, + yamlSourceData: any, + yamlCatalogData: any + ): Promise { + const sourceDefinitionId = await this.createCustomSourceDefinition({ + workspaceId, + sourceDefinition: { + name, + dockerRepository: 'farosai/airbyte-' + name.toLowerCase() + '-source', + dockerImageTag: farosConnectorsVersion, + documentationUrl: 'https://docs.faros.ai', + }, + }); + logger.info('sourceDefinitionId for ' + name + ': ' + sourceDefinitionId); + + await this.createAndConnectSource( + name, + workspaceId, + farosDestinationId, + yamlSourceData, + yamlCatalogData, + sourceDefinitionId + ); + } + + async createAndConnectSource( + name: string, + workspaceId: string, + farosDestinationId: string, + yamlSourceData: any, + yamlCatalogData: any, + sourceDefinitionId: string + ): Promise { + const sourceId = await this.createSourceFromYAML( + workspaceId, + yamlSourceData, + name, + sourceDefinitionId + ); + logger.info('sourceId for ' + name + ': ' + sourceId); + + const connectionId = await this.createConnectionToFaros( + sourceId, + farosDestinationId, + yamlCatalogData, + name + ' - Faros' + ); + logger.info('connectionId for ' + name + ': ' + connectionId); + } + + async init( + farosConnectorsVersion: string, + hasuraUrl: string, + hasuraAdminSecret: string, + segmentUserId: string + ): Promise { + logger.info('init'); + + const workspaceId = await this.getFirstWorkspace(); + logger.info('workspaceId: ' + workspaceId); + await this.completeFarosWorkspaceSetup(workspaceId); + + const farosDestinationDefintionId = + await this.createFarosDestinationDefinition( + workspaceId, + farosConnectorsVersion + ); + logger.info('farosDestinationDefintionId: ' + farosDestinationDefintionId); + + const farosDestinationId = await this.createFarosDestination( + workspaceId, + farosDestinationDefintionId, + hasuraUrl, + hasuraAdminSecret, + segmentUserId + ); + logger.info('farosDestinationId: ' + farosDestinationId); + + // do NOT converstion to camel case + const yamlSourceData = loadYamlFile(SOURCES); + + // convert to camel case because of sync_mode (file) vs syncMode (API) + const yamlCatalogData = convertKeysToCamelCase(loadYamlFile(CATALOGS)); + + const communitySources = [ + ['GitHub', 'ef69ef6e-aa7f-4af1-a01d-ef775033524e'], + ['GitLab', '5e6175e5-68e1-4c17-bff9-56103bbb0d80'], + ['Jira', '68e63de2-bb83-4c7e-93fa-a8a9051e3993'], + ]; + for (const communitySource of communitySources) { + logger.info( + 'sourceDefinitionId for ' + + communitySource[0] + + ': ' + + communitySource[1] + + ' (community)' + ); + await this.createAndConnectSource( + communitySource[0], + workspaceId, + farosDestinationId, + yamlSourceData, + yamlCatalogData, + communitySource[1] + ); + } + + const farosSources = [ + 'Bitbucket', + 'Phabricator', + 'Buildkite', + 'CircleCI', + 'Harness', + 'Jenkins', + 'Datadog', + 'OpsGenie', + 'PagerDuty', + 'SquadCast', + 'Statuspage', + 'VictorOps', + ]; + for (const farosSource of farosSources) { + await this.handleFarosSource( + farosSource, + workspaceId, + farosDestinationId, + farosConnectorsVersion, + yamlSourceData, + yamlCatalogData + ); + } + } +} diff --git a/init/src/airbyte/types.ts b/init/src/airbyte/types.ts deleted file mode 100644 index 8f4f6f07..00000000 --- a/init/src/airbyte/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -interface ConnectorDefinition { - dockerImageTag: string; - dockerRepository: string; - name: string; -} - -export interface DestinationDefinition extends ConnectorDefinition { - destinationDefinitionId: string; -} - -export interface SourceDefinition extends ConnectorDefinition { - sourceDefinitionId: string; -} diff --git a/init/test/integration-tests/airbyte-client.ts b/init/test/integration-tests/airbyte-client.ts index b996655d..b035d1d3 100644 --- a/init/test/integration-tests/airbyte-client.ts +++ b/init/test/integration-tests/airbyte-client.ts @@ -50,24 +50,28 @@ export class AirbyteClient { ); } - async checkDestinationConnection(destinationId: string): Promise { - return await this.api - .post('/destinations/check_connection', { - destinationId, - }) - .then((response) => response.data.status === 'succeeded'); + async getFirstWorkspace(): Promise { + const response = await this.api.post('/workspaces/list', {}); + return response.data.workspaces[0].workspaceId as string; + } + + async checkDestinationConnection(destinationName: string): Promise { + const workspaceId = await this.getFirstWorkspace(); + const response = await this.api.post('/destinations/list', {workspaceId}); + return ( + response.data.destinations.filter( + (destination) => destination.name === destinationName + ).length > 0 + ); } async getDestinationConnectionConfiguration( - destinationId: string + destinationName: string ): Promise { - return await this.api - .post('/destinations/get', { - destinationId, - }) - .then( - (response) => - response.data.connectionConfiguration as ConnectionConfiguration - ); + const workspaceId = await this.getFirstWorkspace(); + const response = await this.api.post('/destinations/list', {workspaceId}); + return response.data.destinations.filter( + (destination) => destination.name === destinationName + )[0].connectionConfiguration as ConnectionConfiguration; } } diff --git a/init/test/integration-tests/integration.test.ts b/init/test/integration-tests/integration.test.ts index e768ad0a..3bd2a0bf 100644 --- a/init/test/integration-tests/integration.test.ts +++ b/init/test/integration-tests/integration.test.ts @@ -6,13 +6,11 @@ import path from 'path'; import {AirbyteClient, ConnectionConfiguration} from './airbyte-client'; import {HasuraClient} from './hasura-client'; -let destinationId: string; let hasuraAdminSecret: string; let hasuraClient: HasuraClient; let airbyteClient: AirbyteClient; beforeAll(async () => { - destinationId = process.env.DESTINATION_ID; hasuraAdminSecret = process.env.HASURA_GRAPHQL_ADMIN_SECRET; airbyteClient = new AirbyteClient('http://localhost:8000'); @@ -29,7 +27,7 @@ describe('integration tests', () => { 'check connection to the Faros destination', async () => { expect( - await airbyteClient.checkDestinationConnection(destinationId) + await airbyteClient.checkDestinationConnection('Faros Destination') ).toBe(true); }, 60 * 1000 @@ -40,7 +38,7 @@ describe('integration tests', () => { async () => { const connectionConfiguration: ConnectionConfiguration = await airbyteClient.getDestinationConnectionConfiguration( - destinationId + 'Faros Destination' ); connectionConfiguration.edition_configs.hasura_admin_secret = diff --git a/kube/base/faros/config/.env b/kube/base/faros/config/.env index 870fe03e..7fc7d30e 100644 --- a/kube/base/faros/config/.env +++ b/kube/base/faros/config/.env @@ -25,7 +25,7 @@ FAROS_AIRBYTE_FORCE_SETUP=false FAROS_INIT_IMAGE=farosai/faros-ce-init:latest ############################## Airbyte ######################################## -VERSION=0.39.37-alpha +VERSION=0.40.9 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_USER=docker DATABASE_PASSWORD=docker @@ -74,7 +74,7 @@ N8N_DOCKER_MOUNT=n8n_data # # # Contributors - please organise this env file according to the above linked file. -# Source: https://github.com/airbytehq/airbyte/blob/v0.39.37-alpha/.env +# Source: https://github.com/airbytehq/airbyte/blob/v0.40.9/.env ### SHARED ### @@ -125,6 +125,10 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= ### LOGGING/MONITORING/TRACKING ### TRACKING_STRATEGY=logging @@ -135,6 +139,7 @@ LOG_LEVEL=INFO ### APPLICATIONS ### # Worker # +WORKERS_MICRONAUT_ENVIRONMENTS=control # Relevant to scaling. MAX_SYNC_WORKERS=5 MAX_SPEC_WORKERS=5 @@ -158,13 +163,15 @@ METRIC_CLIENT= # Useful only when metric client is set to be otel. Must start with http:// or https://. OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317" -USE_STREAM_CAPABLE_STATE=false +USE_STREAM_CAPABLE_STATE=true -# extra settings to limit warnings during CE startup -PAPERCUPS_STORYTIME=disabled -RUN_DATABASE_MIGRATION_ON_STARTUP=true -SECRET_PERSISTENCE=NONE -WORKER_ENVIRONMENT=docker -NEW_SCHEDULER= +# extra settings to limit warnings during CE setup +RUN_DATABASE_MIGRATION_ON_STARTUP=true +SECRET_PERSISTENCE=NONE +WORKER_ENVIRONMENT=docker +NEW_SCHEDULER= DEPLOYMENT_MODE= +JOB_ERROR_REPORTING_STRATEGY= JOB_ERROR_REPORTING_SENTRY_DSN= +LOG_CONNECTOR_MESSAGES= +TEMPORAL_HISTORY_RETENTION_IN_DAYS= diff --git a/kube/base/kustomization.yaml b/kube/base/kustomization.yaml index a253b931..870500ae 100644 --- a/kube/base/kustomization.yaml +++ b/kube/base/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: # Airbyte - - https://github.com/airbytehq/airbyte/kube/overlays/stable?ref=v0.39.37-alpha + - https://github.com/airbytehq/airbyte/kube/overlays/stable?ref=v0.40.9 # Metabase - ./metabase # Hasura