From 0609173c76664fcfd021450796d388217e5d39d1 Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Tue, 19 Nov 2024 16:44:18 +0100 Subject: [PATCH 1/5] Preload datasets for partial pipeline runs --- .../src/bundle/BundlePipelinesManager.test.ts | 4 + .../src/bundle/BundlePipelinesManager.ts | 318 +++++++++++++----- .../src/bundle/run/PipelineRunStatus.ts | 16 +- packages/databricks-vscode/src/extension.ts | 1 + .../databricks-vscode/src/locking/Barrier.ts | 10 +- .../PipelineRunEventsTreeNode.ts | 2 +- .../PipelineRunStatusTreeNode.ts | 2 +- 7 files changed, 254 insertions(+), 99 deletions(-) diff --git a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts index 14fda5419..81ab22f9a 100644 --- a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts +++ b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts @@ -5,8 +5,10 @@ import {mock, instance, when} from "ts-mockito"; import assert from "assert"; import {EventEmitter} from "vscode"; import {install, InstalledClock} from "@sinonjs/fake-timers"; +import {ConnectionManager} from "../configuration/ConnectionManager"; describe(__filename, () => { + let connectionManager: ConnectionManager; let runStatusManager: BundleRunStatusManager; let configModel: ConfigModel; let manager: BundlePipelinesManager; @@ -18,6 +20,7 @@ describe(__filename, () => { eventEmitter = new EventEmitter(); runStatusManager = mock(); configModel = mock(); + connectionManager = mock(); when(runStatusManager.onDidChange).thenReturn(eventEmitter.event); when(configModel.onDidChangeKey("remoteStateConfig")).thenReturn( new EventEmitter().event @@ -26,6 +29,7 @@ describe(__filename, () => { new EventEmitter().event ); manager = new BundlePipelinesManager( + instance(connectionManager), instance(runStatusManager), instance(configModel) ); diff --git a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts index 905a6f077..8f8571d0e 100644 --- a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts +++ b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts @@ -13,155 +13,231 @@ import {PipelineRunStatus} from "./run/PipelineRunStatus"; import {BundleRunStatusManager} from "./run/BundleRunStatusManager"; import {ConfigModel} from "../configuration/models/ConfigModel"; import { + ListPipelineEventsRequest, PipelineEvent, UpdateInfo, } from "@databricks/databricks-sdk/dist/apis/pipelines"; +import {ConnectionManager} from "../configuration/ConnectionManager"; +import {Barrier} from "../locking/Barrier"; +import {WorkspaceClient} from "@databricks/databricks-sdk"; +import {BundlePreValidateState} from "./models/BundlePreValidateModel"; + +type RunState = { + data: UpdateInfo | undefined; + events: PipelineEvent[] | undefined; +}; type PipelineState = { key: string; datasets: Set; - runs: Set; + runs: Set; }; -type Pick = QuickPickItem & {isDataset?: boolean; isDefault?: boolean}; +type PreloadedPipelineState = Promise | undefined>; + +type Pick = QuickPickItem & {isDataset?: boolean}; export class BundlePipelinesManager { private disposables: Disposable[] = []; - private readonly pipelines: Map = new Map(); + private readonly triggeredState: Map = new Map(); + private readonly preloadedState: Map = + new Map(); constructor( + private readonly connectionManager: ConnectionManager, private readonly runStatusManager: BundleRunStatusManager, private readonly configModel: ConfigModel ) { this.disposables.push( this.configModel.onDidChangeTarget(() => { - this.updatePipelines(); + this.updateTriggeredPipelinesState(); }), this.configModel.onDidChangeKey("remoteStateConfig")(async () => { - this.updatePipelines(); + this.updateTriggeredPipelinesState(); }), this.runStatusManager.onDidChange(() => { - this.updatePipelines(); + this.updateTriggeredPipelinesState(); }) ); - this.updatePipelines(); + this.updateTriggeredPipelinesState(); } - private async updatePipelines() { + private async updateTriggeredPipelinesState() { const remoteState = await this.configModel.get("remoteStateConfig"); if (!remoteState) { - this.pipelines.clear(); + this.triggeredState.clear(); return; } const pipelines = remoteState.resources?.pipelines ?? {}; Object.keys(pipelines).map((pipelineKey) => { - if (!this.pipelines.has(pipelineKey)) { - this.pipelines.set(pipelineKey, { + if (!this.triggeredState.has(pipelineKey)) { + this.triggeredState.set(pipelineKey, { key: pipelineKey, datasets: new Set(), runs: new Set(), }); } - const state = this.pipelines.get(pipelineKey)!; + const state = this.triggeredState.get(pipelineKey)!; const runStatus = this.runStatusManager.runStatuses.get( `pipelines.${pipelineKey}` ); if (runStatus) { state.runs.add(runStatus as PipelineRunStatus); - state.datasets = this.updatePipelineDatasets(state.runs); + state.datasets = extractPipelineDatasets(state.runs); } }); } public getDatasets(pipelineKey: string) { - return this.pipelines.get(pipelineKey)?.datasets ?? new Set(); + return this.triggeredState.get(pipelineKey)?.datasets ?? new Set(); } - private updatePipelineDatasets(runs: Set) { - const datasets = new Set(); - const runsByStartTimeDesc = Array.from(runs).sort( - (a, b) => - (b.data?.update?.creation_time ?? 0) - - (a.data?.update?.creation_time ?? 0) - ); - for (const run of runsByStartTimeDesc) { - for (const event of run.events ?? []) { - const datasetName = extractDatasetName(event); - if (datasetName) { - datasets.add(datasetName); - } + async preloadDatasets(pipelineKey: string): PreloadedPipelineState { + const remoteState = await this.configModel.get("remoteStateConfig"); + if (!remoteState) { + return undefined; + } + + const pipelines = remoteState.resources?.pipelines ?? {}; + const pipelineId = pipelines[pipelineKey]?.id; + if (!pipelineId) { + return undefined; + } + + const client = this.connectionManager.workspaceClient; + if (!client) { + return undefined; + } + + const preloaded = this.preloadedState.get(pipelineKey); + if (preloaded) { + return preloaded; + } + + const barrier = new Barrier>(); + this.preloadedState.set(pipelineKey, barrier.promise); + + try { + const runs = await this.preloadUpdates(client, pipelineId); + if (!runs) { + barrier.resolve(new Set()); + return barrier.promise; } - if (isFullGraphUpdate(run.data?.update)) { - break; + const listing = this.createPreloadEventsRequest( + client, + pipelineId, + runs + ); + for await (const event of listing) { + const runState = runs.get(event.origin?.update_id ?? ""); + if (runState?.events) { + runState.events.push(event); + } } + const datasets = extractPipelineDatasets(new Set(runs.values())); + barrier.resolve(datasets); + } catch (e) { + barrier.reject(e); } - return datasets; + + return barrier.promise; } - async showTableSelectionQuickPick(pipelineKey: string) { - const key = pipelineKey.split(".")[1]; - const datasets = this.getDatasets(key); - const defaultsSeparatorPick: Pick = { - label: "Defaults", - kind: QuickPickItemKind.Separator, - alwaysShow: true, - }; - const datasetPicks: Pick[] = Array.from(datasets).map((dataset) => ({ - label: dataset, - isDataset: true, - alwaysShow: true, - isDefault: true, - })); - const optionsSeparatorPick: Pick = { - label: "Options", - kind: QuickPickItemKind.Separator, - alwaysShow: true, - }; - const fullRefreshPick: Pick = { - label: "Full Refresh", - description: "Reset tables before the update", - alwaysShow: true, - isDefault: true, + private async preloadUpdates(client: WorkspaceClient, pipelineId: string) { + const latestUpdates = await client.pipelines.listUpdates({ + // eslint-disable-next-line @typescript-eslint/naming-convention + pipeline_id: pipelineId, + // eslint-disable-next-line @typescript-eslint/naming-convention + max_results: 3, + }); + if (!latestUpdates.updates) { + return undefined; + } + const runs: Map = latestUpdates.updates.reduce( + (map, update) => { + map.set(update.update_id, {data: update, events: []}); + return map; + }, + new Map() + ); + return runs; + } + + private createPreloadEventsRequest( + client: WorkspaceClient, + pipelineId: string, + runs: Map + ) { + const listEventsOptions: ListPipelineEventsRequest = { + // eslint-disable-next-line @typescript-eslint/naming-convention + pipeline_id: pipelineId, + // eslint-disable-next-line @typescript-eslint/naming-convention + order_by: ["timestamp asc"], }; - const ui = window.createQuickPick(); - ui.canSelectMany = true; - const defaultItems = [ - defaultsSeparatorPick, - ...datasetPicks, - optionsSeparatorPick, - fullRefreshPick, - ]; - ui.items = defaultItems; - if (datasets.size > 0) { - ui.title = "Select or type in tables to update"; + const oldestUpdateTime = Array.from(runs.values()).sort( + (a, b) => a.data?.creation_time ?? 0 - (b.data?.creation_time ?? 0) + )[0].data?.creation_time; + if (oldestUpdateTime) { + const timestamp = new Date(oldestUpdateTime).toISOString(); + listEventsOptions.filter = `timestamp >= '${timestamp}'`; } else { - ui.title = "Provide table names to update"; + listEventsOptions.max_results = 100; } - ui.placeholder = "Comma separated list of table names"; + return client.pipelines.listPipelineEvents(listEventsOptions); + } + + public async showTableSelectionQuickPick(pipelineKey: string) { + const key = pipelineKey.split(".")[1]; + const knownDatasets = this.getDatasets(key); + const mode = await this.configModel.get("mode"); + const {allPicks, fullRefreshPick} = createPicks(mode, knownDatasets); + const ui = window.createQuickPick(); + ui.title = "Select tables to update"; + ui.placeholder = + "Comma-separated list of tables to extend the selection"; + ui.canSelectMany = true; + ui.busy = true; + ui.items = allPicks; ui.show(); + let isUIVisible = true; const disposables: Disposable[] = []; ui.onDidChangeValue( - () => { - const manualItems = stringToPicks(ui.value); - ui.items = manualItems.concat(defaultItems); - ui.selectedItems = manualItems.concat( - ui.selectedItems.filter((item) => item.isDefault) - ); - }, + () => updateItems(ui, mode, knownDatasets), null, disposables ); + this.preloadDatasets(key) + .then((preloadedDatasets) => { + if (preloadedDatasets && isUIVisible) { + for (const dataset of preloadedDatasets) { + knownDatasets.add(dataset); + } + updateItems(ui, mode, knownDatasets); + } + }) + .catch((e) => { + window.showErrorMessage( + "Failed to load datasets from previous pipeline runs", + {detail: e.message} + ); + }) + .finally(() => { + if (isUIVisible) { + ui.busy = false; + } + }); const picks = await waitForPicks(ui, disposables); const selectedTables = picksToString(picks); disposables.forEach((d) => d.dispose()); - ui.hide(); + ui.dispose(); + isUIVisible = false; return { tables: selectedTables, fullRefresh: ui.selectedItems.includes(fullRefreshPick), }; } - dispose() { + public dispose() { this.disposables.forEach((d) => d.dispose()); } } @@ -191,6 +267,63 @@ function extractDatasetName( return event.origin.dataset_name; } +function extractPipelineDatasets(runs: Set) { + const datasets = new Set(); + const runsByStartTimeDesc = Array.from(runs).sort( + (a, b) => (b.data?.creation_time ?? 0) - (a.data?.creation_time ?? 0) + ); + for (const run of runsByStartTimeDesc) { + for (const event of run.events ?? []) { + const datasetName = extractDatasetName(event); + if (datasetName) { + datasets.add(datasetName); + } + } + if (isFullGraphUpdate(run.data)) { + break; + } + } + return datasets; +} + +function createPicks( + mode: BundlePreValidateState["mode"], + datasets: Set, + manualValue?: string +) { + const defaultsSeparatorPick: Pick = { + label: "Defaults", + kind: QuickPickItemKind.Separator, + alwaysShow: true, + }; + if (manualValue) { + const manualDatasets = stringToDatasets(manualValue); + datasets = new Set([...manualDatasets, ...datasets]); + } + const datasetPicks: Pick[] = setToDatasetPicks(datasets); + const optionsSeparatorPick: Pick = { + label: "Options", + kind: QuickPickItemKind.Separator, + alwaysShow: true, + }; + const fullRefreshPick: Pick = { + label: "Full Refresh", + description: "Reset tables before the update", + alwaysShow: true, + }; + const ui = window.createQuickPick(); + ui.canSelectMany = true; + const allPicks = [ + defaultsSeparatorPick, + ...datasetPicks, + optionsSeparatorPick, + ]; + if (mode === "development") { + allPicks.push(fullRefreshPick); + } + return {allPicks, fullRefreshPick}; +} + function picksToString(picks?: readonly Pick[]): string | undefined { return picks ?.filter((p) => p.isDataset) @@ -198,18 +331,20 @@ function picksToString(picks?: readonly Pick[]): string | undefined { .join(","); } -function stringToPicks(str: string): Pick[] { - return str +function stringToDatasets(str: string): Set { + const list = str .split(",") .map((item) => item.trim()) - .filter(Boolean) - .map((item) => { - return { - label: item, - alwaysShow: true, - isDataset: true, - }; - }); + .filter(Boolean); + return new Set(list); +} + +function setToDatasetPicks(datasets: Set): Pick[] { + return Array.from(datasets).map((dataset) => ({ + label: dataset, + isDataset: true, + alwaysShow: true, + })); } async function waitForPicks(ui: QuickPick, disposables: Disposable[]) { @@ -218,3 +353,14 @@ async function waitForPicks(ui: QuickPick, disposables: Disposable[]) { ui.onDidHide(() => resolve(undefined), null, disposables); }); } + +function updateItems( + ui: QuickPick, + mode: BundlePreValidateState["mode"], + knownDatasets: Set +) { + ui.items = createPicks(mode, knownDatasets, ui.value).allPicks; + ui.selectedItems = ui.items.filter((i) => + ui.selectedItems.some((s) => s.label === i.label) + ); +} diff --git a/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts b/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts index 368aac0c5..1667efaae 100644 --- a/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts +++ b/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts @@ -13,7 +13,7 @@ function isRunning(status?: pipelines.UpdateInfoState) { export class PipelineRunStatus extends BundleRunStatus { public readonly type = "pipelines"; - public data: pipelines.GetUpdateResponse | undefined; + public data: pipelines.UpdateInfo | undefined; public events: pipelines.PipelineEvent[] | undefined; private interval?: NodeJS.Timeout; @@ -59,21 +59,22 @@ export class PipelineRunStatus extends BundleRunStatus { if (this.runId === undefined) { throw new Error("No update id"); } - this.data = await client.pipelines.getUpdate({ + const getUpdateResponse = await client.pipelines.getUpdate({ pipeline_id: this.pipelineId, update_id: this.runId, }); + this.data = getUpdateResponse.update; - if (this.data.update?.creation_time !== undefined) { + if (this.data?.creation_time !== undefined) { this.events = await this.fetchUpdateEvents( client, - this.data.update.creation_time, - this.data.update.update_id + this.data?.creation_time, + this.data?.update_id ); } // If update is completed, we stop polling. - if (!isRunning(this.data.update?.state)) { + if (!isRunning(this.data?.state)) { this.markCompleted(); } else { this.onDidChangeEmitter.fire(); @@ -141,10 +142,11 @@ export class PipelineRunStatus extends BundleRunStatus { }) ).wait(); } - this.data = await client.pipelines.getUpdate({ + const getUpdateResponse = await client.pipelines.getUpdate({ pipeline_id: this.pipelineId, update_id: this.runId, }); + this.data = getUpdateResponse.update; this.markCancelled(); } } diff --git a/packages/databricks-vscode/src/extension.ts b/packages/databricks-vscode/src/extension.ts index 8ddb7d5d5..ea49a0b07 100644 --- a/packages/databricks-vscode/src/extension.ts +++ b/packages/databricks-vscode/src/extension.ts @@ -635,6 +635,7 @@ export async function activate( ); const bundlePipelinesManager = new BundlePipelinesManager( + connectionManager, bundleRunStatusManager, configModel ); diff --git a/packages/databricks-vscode/src/locking/Barrier.ts b/packages/databricks-vscode/src/locking/Barrier.ts index 4c741a4b2..8e06a2d46 100644 --- a/packages/databricks-vscode/src/locking/Barrier.ts +++ b/packages/databricks-vscode/src/locking/Barrier.ts @@ -1,9 +1,11 @@ -export class Barrier { - public promise: Promise; - public resolve: () => void = () => {}; +export class Barrier { + public promise: Promise; + public resolve: (value: T) => void = () => {}; + public reject: (error: any) => void = () => {}; constructor() { - this.promise = new Promise((resolve) => { + this.promise = new Promise((resolve, reject) => { this.resolve = resolve; + this.reject = reject; }); } } diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts index 9a9457287..70b6e77a1 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts @@ -15,7 +15,7 @@ export class PipelineRunEventsTreeNode readonly type = "pipeline_run_events"; private get update() { - return this.runMonitor?.data?.update; + return this.runMonitor?.data; } private get events() { diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts index 099b58188..4a053e795 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts @@ -46,7 +46,7 @@ export class PipelineRunStatusTreeNode readonly type = "pipeline_run_status"; private get update() { - return this.runMonitor?.data?.update; + return this.runMonitor?.data; } public get url() { From f0206d3a561e05f17a0efa0868433e468108ae00 Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Wed, 20 Nov 2024 11:04:56 +0100 Subject: [PATCH 2/5] Improve full refresh confirmation --- .../src/bundle/BundlePipelinesManager.ts | 60 ++++++++++++------- .../BundleCommands.ts | 2 +- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts index 8f8571d0e..707784987 100644 --- a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts +++ b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts @@ -189,8 +189,7 @@ export class BundlePipelinesManager { public async showTableSelectionQuickPick(pipelineKey: string) { const key = pipelineKey.split(".")[1]; const knownDatasets = this.getDatasets(key); - const mode = await this.configModel.get("mode"); - const {allPicks, fullRefreshPick} = createPicks(mode, knownDatasets); + const {allPicks, fullRefreshPick} = createPicks(knownDatasets); const ui = window.createQuickPick(); ui.title = "Select tables to update"; ui.placeholder = @@ -202,7 +201,7 @@ export class BundlePipelinesManager { let isUIVisible = true; const disposables: Disposable[] = []; ui.onDidChangeValue( - () => updateItems(ui, mode, knownDatasets), + () => updateItems(ui, knownDatasets), null, disposables ); @@ -212,7 +211,7 @@ export class BundlePipelinesManager { for (const dataset of preloadedDatasets) { knownDatasets.add(dataset); } - updateItems(ui, mode, knownDatasets); + updateItems(ui, knownDatasets); } }) .catch((e) => { @@ -231,10 +230,16 @@ export class BundlePipelinesManager { disposables.forEach((d) => d.dispose()); ui.dispose(); isUIVisible = false; - return { - tables: selectedTables, - fullRefresh: ui.selectedItems.includes(fullRefreshPick), - }; + if (isPickSelected(ui, fullRefreshPick)) { + switch (await confirmFullRefresh()) { + case "Yes": + return {tables: selectedTables, fullRefresh: true}; + default: + return {tables: undefined, fullRefresh: false}; + } + } else { + return {tables: selectedTables, fullRefresh: false}; + } } public dispose() { @@ -242,6 +247,19 @@ export class BundlePipelinesManager { } } +async function confirmFullRefresh() { + return await window.showWarningMessage( + "Are you sure you want to full refresh?", + { + modal: true, + // The same warning we show in the workspace + detail: "Full refresh will truncate and recompute ALL tables in this pipeline from scratch. This can lead to data loss for non-idempotent sources.", + }, + "Yes", + "No" + ); +} + function isFullGraphUpdate(update?: UpdateInfo) { if (!update) { return false; @@ -286,11 +304,7 @@ function extractPipelineDatasets(runs: Set) { return datasets; } -function createPicks( - mode: BundlePreValidateState["mode"], - datasets: Set, - manualValue?: string -) { +function createPicks(datasets: Set, manualValue?: string) { const defaultsSeparatorPick: Pick = { label: "Defaults", kind: QuickPickItemKind.Separator, @@ -308,7 +322,7 @@ function createPicks( }; const fullRefreshPick: Pick = { label: "Full Refresh", - description: "Reset tables before the update", + description: "Truncate and recopmute tables", alwaysShow: true, }; const ui = window.createQuickPick(); @@ -317,10 +331,8 @@ function createPicks( defaultsSeparatorPick, ...datasetPicks, optionsSeparatorPick, + fullRefreshPick, ]; - if (mode === "development") { - allPicks.push(fullRefreshPick); - } return {allPicks, fullRefreshPick}; } @@ -354,13 +366,15 @@ async function waitForPicks(ui: QuickPick, disposables: Disposable[]) { }); } -function updateItems( - ui: QuickPick, - mode: BundlePreValidateState["mode"], - knownDatasets: Set -) { - ui.items = createPicks(mode, knownDatasets, ui.value).allPicks; +function updateItems(ui: QuickPick, knownDatasets: Set) { + ui.items = createPicks(knownDatasets, ui.value).allPicks; ui.selectedItems = ui.items.filter((i) => ui.selectedItems.some((s) => s.label === i.label) ); } + +function isPickSelected(ui: QuickPick, pick: Pick) { + return ui.selectedItems.some( + (i) => i.label === pick.label && i.description === pick.description + ); +} diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/BundleCommands.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/BundleCommands.ts index db82ca3f6..811b85846 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/BundleCommands.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/BundleCommands.ts @@ -217,7 +217,7 @@ export class BundleCommands implements Disposable { const key = treeNode.resourceKey; const result = await this.bundlePipelinesManager.showTableSelectionQuickPick(key); - if (!result.tables) { + if (!result.tables || result.tables.length === 0) { return; } return this.deployAndRun( From 28457ceae1bf805eecf226ccfd83207882dc046d Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Wed, 20 Nov 2024 11:12:47 +0100 Subject: [PATCH 3/5] Remove unnecessary imports --- packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts index 707784987..9005eb294 100644 --- a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts +++ b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts @@ -20,7 +20,6 @@ import { import {ConnectionManager} from "../configuration/ConnectionManager"; import {Barrier} from "../locking/Barrier"; import {WorkspaceClient} from "@databricks/databricks-sdk"; -import {BundlePreValidateState} from "./models/BundlePreValidateModel"; type RunState = { data: UpdateInfo | undefined; From a6a3a047103cf24ab660e2f209ec2651e6485c98 Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Wed, 20 Nov 2024 11:20:43 +0100 Subject: [PATCH 4/5] Fix run state UI --- .../PipelineRunEventsTreeNode.ts | 10 +++++++++- .../PipelineRunStatusTreeNode.ts | 12 ++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts index 70b6e77a1..ae54e4f24 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts @@ -61,8 +61,16 @@ export class PipelineRunEventsTreeNode return children; } + isLoading(): boolean { + return ( + (this.events === undefined || this.events.length === 0) && + (this.runMonitor.runState === "running" || + this.runMonitor.runState === "unknown") + ); + } + getTreeItem(): BundleResourceExplorerTreeItem { - if (this.events === undefined || this.events.length === 0) { + if (this.isLoading()) { return { label: "Event Log", iconPath: new ThemeIcon("loading~spin"), diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts index 4a053e795..fe020c118 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts @@ -102,6 +102,14 @@ export class PipelineRunStatusTreeNode return children; } + isLoading(): boolean { + return ( + this.update === undefined && + (this.runMonitor.runState === "running" || + this.runMonitor.runState === "unknown") + ); + } + getTreeItem(): BundleResourceExplorerTreeItem { const runMonitorRunStateTreeItem = RunStateUtils.getTreeItemFromRunMonitorStatus( @@ -114,7 +122,7 @@ export class PipelineRunStatusTreeNode return runMonitorRunStateTreeItem; } - if (this.update === undefined) { + if (this.isLoading()) { return { label: "Run Status", iconPath: new ThemeIcon("loading~spin"), @@ -131,7 +139,7 @@ export class PipelineRunStatusTreeNode return { label: "Run Status", iconPath: icon, - description: sentenceCase(this.update.state), + description: sentenceCase(this.update?.state), contextValue: ContextUtils.getContextString({ nodeType: this.type, hasUrl: this.url !== undefined, From ea9501c72cbd16de1e1ff94c6c6db9078d3fbbf6 Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Wed, 20 Nov 2024 13:11:50 +0100 Subject: [PATCH 5/5] Fix unit test --- .../src/bundle/BundlePipelinesManager.test.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts index 81ab22f9a..9310dd9ce 100644 --- a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts +++ b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts @@ -48,9 +48,7 @@ describe(__filename, () => { /* eslint-disable @typescript-eslint/naming-convention */ const firstRun = { - data: { - update: {creation_time: 10}, - }, + data: {creation_time: 10}, events: [ {origin: {dataset_name: "table1"}}, {origin: {not_a_dataset_name: "table1.5"}}, @@ -71,10 +69,8 @@ describe(__filename, () => { /* eslint-disable @typescript-eslint/naming-convention */ const secondPartialRun = { data: { - update: { - creation_time: 100, - refresh_selection: ["table3", "table4"], - }, + creation_time: 100, + refresh_selection: ["table3", "table4"], }, events: [ {origin: {dataset_name: "table3"}}, @@ -98,10 +94,8 @@ describe(__filename, () => { /* eslint-disable @typescript-eslint/naming-convention */ const finalFullRefreshRun = { data: { - update: { - creation_time: 200, - refresh_selection: [], - }, + creation_time: 200, + refresh_selection: [], }, events: [ {origin: {dataset_name: "table_new"}},