From c3b05e2fc33256bd1b68e90b32af96ea56d286a4 Mon Sep 17 00:00:00 2001 From: Igor Sechyn Date: Thu, 3 Nov 2022 09:23:56 +0200 Subject: [PATCH 1/3] chore: added explicit json response parsing to catch an error --- src/client/LinearClient.ts | 20 +++++++++++++++++++- src/index.ts | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/client/LinearClient.ts b/src/client/LinearClient.ts index 3acfded..0afb3a6 100644 --- a/src/client/LinearClient.ts +++ b/src/client/LinearClient.ts @@ -1,4 +1,5 @@ import axios from "axios"; +import { AirbyteLogger } from "faros-airbyte-cdk/lib"; import { Attachment, AuditEntry, @@ -57,7 +58,10 @@ export type EntityType = * Thin client on top of the rest export api to fetch different resources. */ export class LinearClient { - public constructor(private readonly config: Config) {} + public constructor( + private readonly config: Config, + private readonly logger: AirbyteLogger + ) {} /** * @returns List of all issues in organization. @@ -209,6 +213,20 @@ export class LinearClient { method: "GET", baseURL: LINEAR_API_BASE_URL, url: entityType, + transformResponse: (data) => { + try { + return JSON.parse(data); + } catch (error) { + this.logger.error( + `Failed to parse data as JSON. Error: ${JSON.stringify( + error, + null, + 2 + )}` + ); + throw error; + } + }, headers: { Authorization: this.config.apiKey, }, diff --git a/src/index.ts b/src/index.ts index 919725e..3223f50 100644 --- a/src/index.ts +++ b/src/index.ts @@ -49,7 +49,7 @@ class LinearSource extends AirbyteSourceBase { public async checkConnection( config: AirbyteConfig ): Promise<[boolean, VError]> { - const client = new LinearClient({ apiKey: config.apiKey }); + const client = new LinearClient({ apiKey: config.apiKey }, this.logger); try { await client.checkConnection(); return [true, null]; @@ -59,7 +59,7 @@ class LinearSource extends AirbyteSourceBase { } public streams(config: AirbyteConfig): AirbyteStreamBase[] { - const client = new LinearClient({ apiKey: config.apiKey }); + const client = new LinearClient({ apiKey: config.apiKey }, this.logger); return [ new Issue(this.logger, client), new Organization(this.logger, client), From 525e1ca334a34af04ec7fca70318b2d4f46385fa Mon Sep 17 00:00:00 2001 From: Igor Sechyn Date: Thu, 3 Nov 2022 09:30:39 +0200 Subject: [PATCH 2/3] chore: added more logging --- src/client/LinearClient.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/LinearClient.ts b/src/client/LinearClient.ts index 0afb3a6..6cd4bb2 100644 --- a/src/client/LinearClient.ts +++ b/src/client/LinearClient.ts @@ -218,11 +218,9 @@ export class LinearClient { return JSON.parse(data); } catch (error) { this.logger.error( - `Failed to parse data as JSON. Error: ${JSON.stringify( - error, - null, - 2 - )}` + `Failed to parse data as JSON. + Data:${data}, + Error: ${JSON.stringify(error, null, 2)}` ); throw error; } From 00f373d479576861eb3318caf93935abce69fe71 Mon Sep 17 00:00:00 2001 From: Igor Sechyn Date: Thu, 3 Nov 2022 09:39:37 +0200 Subject: [PATCH 3/3] chore: removed milestone as entity --- readme.md | 1 - resources/schemas/milestone.json | 26 --------------------- resources/schemas/project.json | 3 --- src/client/LinearClient.ts | 9 -------- src/client/types.ts | 11 --------- src/index.ts | 2 -- src/streams/milestone.ts | 30 ------------------------- test_files/full_configured_catalog.json | 17 -------------- 8 files changed, 99 deletions(-) delete mode 100644 resources/schemas/milestone.json delete mode 100644 src/streams/milestone.ts diff --git a/readme.md b/readme.md index b0ded3c..0bd12f7 100644 --- a/readme.md +++ b/readme.md @@ -12,7 +12,6 @@ This Source is capable of syncing the following Streams: * [Organization](https://github.com/linear/linear/blob/master/packages/sdk/src/schema.graphql#L7098) * [Teams](https://github.com/linear/linear/blob/master/packages/sdk/src/schema.graphql#L10249) * [User](https://github.com/linear/linear/blob/master/packages/sdk/src/schema.graphql#L11949) -* [Milestone](https://github.com/linear/linear/blob/master/packages/sdk/src/schema.graphql#L4205) * [Project](https://github.com/linear/linear/blob/master/packages/sdk/src/schema.graphql#L7667) * [Integration Resource](https://github.com/linear/linear/blob/master/packages/sdk/src/schema.graphql#L2297) * [Attachment](https://github.com/linear/linear/blob/master/packages/sdk/src/schema.graphql#L144) diff --git a/resources/schemas/milestone.json b/resources/schemas/milestone.json deleted file mode 100644 index 351a838..0000000 --- a/resources/schemas/milestone.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "additionalProperties": true, - "properties": { - "id": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "archivedAt": { - "type": ["null", "string"] - }, - "name": { - "type": "string" - }, - "organizationId": { - "type": "string" - }, - "sortOrder": { - "type": "number" - } - } -} diff --git a/resources/schemas/project.json b/resources/schemas/project.json index b73cbda..3f76c6e 100644 --- a/resources/schemas/project.json +++ b/resources/schemas/project.json @@ -52,9 +52,6 @@ "organizationId": { "type": "string" }, - "milestoneId": { - "type": ["null", "string"] - }, "lastProjectUpdatePromptAt": { "type": ["null", "string"] }, diff --git a/src/client/LinearClient.ts b/src/client/LinearClient.ts index 6cd4bb2..b1ffea7 100644 --- a/src/client/LinearClient.ts +++ b/src/client/LinearClient.ts @@ -9,7 +9,6 @@ import { IssueHistory, IssueLabel, IssueRelation, - Milestone, Organization, Project, ProjectLink, @@ -41,7 +40,6 @@ export type EntityType = | "teamkey" | "teammembership" | "user" - | "milestone" | "project" | "projectupdate" | "projectlink" @@ -98,13 +96,6 @@ export class LinearClient { return await this.fetchEntities("teammembership"); } - /** - * @returns List of all milestones in organization. - */ - public async milestones(): Promise { - return await this.fetchEntities("milestone"); - } - /** * @returns List of all projects in organization. */ diff --git a/src/client/types.ts b/src/client/types.ts index 1343f19..b3d0d56 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -160,16 +160,6 @@ export interface IssueLabel { creatorId?: null | string; } -export interface Milestone { - id?: string; - createdAt?: string; - updatedAt?: string; - archivedAt?: null | string; - name?: string; - organizationId?: string; - sortOrder?: number; -} - export interface Project { id?: string; createdAt?: string; @@ -186,7 +176,6 @@ export interface Project { leadId?: null | string; memberIds?: string[]; organizationId?: string; - milestoneId?: string; lastProjectUpdatePromptAt?: null | string; startDate?: string; targetDate?: string; diff --git a/src/index.ts b/src/index.ts index 3223f50..f9fe1bf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,6 @@ import { Document } from "./streams/document"; import { IssueHistory } from "./streams/issueHistory"; import { IssueLabel } from "./streams/issueLabel"; import { IssueRelation } from "./streams/issueRelation"; -import { Milestone } from "./streams/milestone"; import { Organization } from "./streams/organization"; import { Project } from "./streams/project"; import { ProjectLink } from "./streams/projectLink"; @@ -67,7 +66,6 @@ class LinearSource extends AirbyteSourceBase { new TeamKey(this.logger, client), new TeamMembership(this.logger, client), new User(this.logger, client), - new Milestone(this.logger, client), new Project(this.logger, client), new ProjectUpdate(this.logger, client), new ProjectLink(this.logger, client), diff --git a/src/streams/milestone.ts b/src/streams/milestone.ts deleted file mode 100644 index 0f1a513..0000000 --- a/src/streams/milestone.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { AirbyteLogger, AirbyteStreamBase, StreamKey } from "faros-airbyte-cdk"; -import { Dictionary } from "ts-essentials"; -import { LinearClient } from "../client/LinearClient"; -import { Milestone as MilestoneModel } from "../client/types"; - -export class Milestone extends AirbyteStreamBase { - public constructor( - protected readonly logger: AirbyteLogger, - private readonly client: LinearClient - ) { - super(logger); - } - - public getJsonSchema(): Dictionary { - return require("../../resources/schemas/milestone.json"); - } - public get primaryKey(): StreamKey { - return ["id"]; - } - public get cursorField(): string | string[] { - return []; - } - - public async *readRecords(): AsyncGenerator { - const result = await this.client.milestones(); - for (const record of result) { - yield record; - } - } -} diff --git a/test_files/full_configured_catalog.json b/test_files/full_configured_catalog.json index c154cba..8e46aad 100644 --- a/test_files/full_configured_catalog.json +++ b/test_files/full_configured_catalog.json @@ -136,23 +136,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "milestone", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "project",