From 8e72b6c701f8c8826dc337e584d7f89e61a5eb09 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Wed, 4 Dec 2024 15:28:39 -0500 Subject: [PATCH 01/22] feat(amazonq): setting toggle for using the amazonq language server (#6142) ## Problem We want to experiment with the amazon q language server ## Solution Set up the structure for the eventual amazon q language srever --- packages/amazonq/src/extensionNode.ts | 20 ++++++++++++++++--- packages/amazonq/src/lsp/activation.ts | 12 +++++++++++ packages/core/src/shared/index.ts | 2 +- .../core/src/shared/settings-toolkit.gen.ts | 3 ++- packages/toolkit/package.json | 4 ++++ 5 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 packages/amazonq/src/lsp/activation.ts diff --git a/packages/amazonq/src/extensionNode.ts b/packages/amazonq/src/extensionNode.ts index 6fe0c28ad8f..39e0a80643f 100644 --- a/packages/amazonq/src/extensionNode.ts +++ b/packages/amazonq/src/extensionNode.ts @@ -7,7 +7,15 @@ import * as vscode from 'vscode' import { activateAmazonQCommon, amazonQContextPrefix, deactivateCommon } from './extension' import { DefaultAmazonQAppInitContext } from 'aws-core-vscode/amazonq' import { activate as activateQGumby } from 'aws-core-vscode/amazonqGumby' -import { ExtContext, globals, CrashMonitoring, getLogger, isNetworkError, isSageMaker } from 'aws-core-vscode/shared' +import { + ExtContext, + globals, + CrashMonitoring, + getLogger, + isNetworkError, + isSageMaker, + Experiments, +} from 'aws-core-vscode/shared' import { filetypes, SchemaService } from 'aws-core-vscode/sharedNode' import { updateDevMode } from 'aws-core-vscode/dev' import { CommonAuthViewProvider } from 'aws-core-vscode/login' @@ -21,6 +29,7 @@ import { beta } from 'aws-core-vscode/dev' import { activate as activateNotifications, NotificationsController } from 'aws-core-vscode/notifications' import { AuthState, AuthUtil } from 'aws-core-vscode/codewhisperer' import { telemetry, AuthUserState } from 'aws-core-vscode/telemetry' +import { activate as activateAmazonqLsp } from './lsp/activation' export async function activate(context: vscode.ExtensionContext) { // IMPORTANT: No other code should be added to this function. Place it in one of the following 2 functions where appropriate. @@ -42,8 +51,13 @@ async function activateAmazonQNode(context: vscode.ExtensionContext) { const extContext = { extensionContext: context, } - await activateCWChat(context) - await activateQGumby(extContext as ExtContext) + + if (Experiments.instance.get('amazonqLSP', false)) { + await activateAmazonqLsp(context) + } else { + await activateCWChat(context) + await activateQGumby(extContext as ExtContext) + } const authProvider = new CommonAuthViewProvider( context, diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts new file mode 100644 index 00000000000..7924096107d --- /dev/null +++ b/packages/amazonq/src/lsp/activation.ts @@ -0,0 +1,12 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import vscode from 'vscode' + +export async function activate(ctx: vscode.ExtensionContext): Promise { + /** + * download install and run the language server + */ +} diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index a4f35d525c6..5942f6ba6c2 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -13,7 +13,7 @@ export { activate as activateLogger } from './logger/activation' export { activate as activateTelemetry } from './telemetry/activation' export { DefaultAwsContext } from './awsContext' export { DefaultAWSClientBuilder, ServiceOptions } from './awsClientBuilder' -export { Settings, DevSettings } from './settings' +export { Settings, Experiments, DevSettings } from './settings' export * from './extensionUtilities' export * from './extensionStartup' export { RegionProvider } from './regions/regionProvider' diff --git a/packages/core/src/shared/settings-toolkit.gen.ts b/packages/core/src/shared/settings-toolkit.gen.ts index ea291352701..8e4cc453d03 100644 --- a/packages/core/src/shared/settings-toolkit.gen.ts +++ b/packages/core/src/shared/settings-toolkit.gen.ts @@ -41,7 +41,8 @@ export const toolkitSettings = { "ssoCacheError": {} }, "aws.experiments": { - "jsonResourceModification": {} + "jsonResourceModification": {}, + "amazonqLSP": {} }, "aws.resources.enabledResources": {}, "aws.lambda.recentlyUploaded": {}, diff --git a/packages/toolkit/package.json b/packages/toolkit/package.json index 0ea90663c9f..13dc18cc91d 100644 --- a/packages/toolkit/package.json +++ b/packages/toolkit/package.json @@ -246,6 +246,10 @@ "jsonResourceModification": { "type": "boolean", "default": false + }, + "amazonqLSP": { + "type": "boolean", + "default": false } }, "additionalProperties": false From e193d44575104f345b6eff6fed291a9a55a144a4 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:38:34 -0500 Subject: [PATCH 02/22] feat(amazonq): Create a common langauge server downloader (#6148) ## Problem - we need a way to download the manifest and install the correct mynah ui/language server code ## Solution - create a common lsp downloader that flare/workspace lsps can use --- packages/amazonq/src/lsp/activation.ts | 11 +- packages/amazonq/src/lsp/download.ts | 77 +++++ .../unit/amazonq/lsp/lspController.test.ts | 4 +- packages/core/src/amazonq/index.ts | 2 +- .../core/src/amazonq/lsp/lspController.ts | 284 ++++-------------- packages/core/src/shared/fetchLsp.ts | 214 +++++++++++++ packages/core/src/shared/index.ts | 1 + packages/core/src/shared/logger/logger.ts | 2 +- .../src/testInteg/perf/tryInstallLsp.test.ts | 11 +- 9 files changed, 375 insertions(+), 231 deletions(-) create mode 100644 packages/amazonq/src/lsp/download.ts create mode 100644 packages/core/src/shared/fetchLsp.ts diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts index 7924096107d..44465f8659a 100644 --- a/packages/amazonq/src/lsp/activation.ts +++ b/packages/amazonq/src/lsp/activation.ts @@ -4,9 +4,18 @@ */ import vscode from 'vscode' +import { AmazonQLSPDownloader } from './download' export async function activate(ctx: vscode.ExtensionContext): Promise { + const serverPath = ctx.asAbsolutePath('resources/qdeveloperserver') + const clientPath = ctx.asAbsolutePath('resources/qdeveloperclient') + await new AmazonQLSPDownloader(serverPath, clientPath).tryInstallLsp() + /** - * download install and run the language server + * at this point the language server should be installed and available + * at serverPath and mynah ui should be available and serveable at + * clientPath + * + * TODO: actually hook up the language server */ } diff --git a/packages/amazonq/src/lsp/download.ts b/packages/amazonq/src/lsp/download.ts new file mode 100644 index 00000000000..29966b21495 --- /dev/null +++ b/packages/amazonq/src/lsp/download.ts @@ -0,0 +1,77 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + LspDownloader, + getLogger, + makeTemporaryToolkitFolder, + tryRemoveFolder, + fs, + Manifest, +} from 'aws-core-vscode/shared' + +const manifestURL = 'https://aws-toolkit-language-servers.amazonaws.com/codewhisperer/0/manifest.json' + +export class AmazonQLSPDownloader extends LspDownloader { + constructor( + private readonly serverPath: string, + private readonly clientPath: string + ) { + super(manifestURL) + } + + async isLspInstalled(): Promise { + return (await fs.exists(this.serverPath)) && (await fs.exists(this.clientPath)) + } + + async cleanup(): Promise { + if (await fs.exists(this.serverPath)) { + await tryRemoveFolder(this.serverPath) + } + + if (await fs.exists(this.clientPath)) { + await tryRemoveFolder(this.clientPath) + } + + return true + } + + async install(manifest: Manifest) { + const server = this.getDependency(manifest, 'servers') + const clients = this.getDependency(manifest, 'clients') + if (!server || !clients) { + getLogger('lsp').info(`Did not find LSP URL for ${process.platform} ${process.arch}`) + return false + } + + let tempFolder = undefined + + try { + tempFolder = await makeTemporaryToolkitFolder() + + // download and extract the business logic + await this.downloadAndExtractServer({ + content: server, + installLocation: this.serverPath, + name: 'qdeveloperserver', + tempFolder, + }) + + // download and extract mynah ui + await this.downloadAndExtractServer({ + content: clients, + installLocation: this.clientPath, + name: 'qdeveloperclient', + tempFolder, + }) + } finally { + if (tempFolder) { + await tryRemoveFolder(tempFolder) + } + } + + return true + } +} diff --git a/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts b/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts index d54551e433f..87111b97f1a 100644 --- a/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts +++ b/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts @@ -4,9 +4,9 @@ */ import assert from 'assert' import sinon from 'sinon' -import { Content, LspController } from 'aws-core-vscode/amazonq' +import { LspController } from 'aws-core-vscode/amazonq' import { createTestFile } from 'aws-core-vscode/test' -import { fs } from 'aws-core-vscode/shared' +import { fs, Content } from 'aws-core-vscode/shared' describe('Amazon Q LSP controller', function () { it('Download mechanism checks against hash, when hash matches', async function () { diff --git a/packages/core/src/amazonq/index.ts b/packages/core/src/amazonq/index.ts index 5bd20e4dfd0..c44878a18b6 100644 --- a/packages/core/src/amazonq/index.ts +++ b/packages/core/src/amazonq/index.ts @@ -15,7 +15,7 @@ export { walkthroughInlineSuggestionsExample, walkthroughSecurityScanExample, } from './onboardingPage/walkthrough' -export { LspController, Content } from './lsp/lspController' +export { LspController } from './lsp/lspController' export { LspClient } from './lsp/lspClient' export { api } from './extApi' export { AmazonQChatViewProvider } from './webview/webView' diff --git a/packages/core/src/amazonq/lsp/lspController.ts b/packages/core/src/amazonq/lsp/lspController.ts index 7a74318dd14..4c784eab163 100644 --- a/packages/core/src/amazonq/lsp/lspController.ts +++ b/packages/core/src/amazonq/lsp/lspController.ts @@ -5,23 +5,18 @@ import * as vscode from 'vscode' import * as path from 'path' -import * as crypto from 'crypto' -import { createWriteStream } from 'fs' // eslint-disable-line no-restricted-imports import { getLogger } from '../../shared/logger/logger' import { CurrentWsFolders, collectFilesForIndex } from '../../shared/utilities/workspaceUtils' -import fetch from 'node-fetch' -import request from '../../shared/request' import { LspClient } from './lspClient' -import AdmZip from 'adm-zip' import { RelevantTextDocument } from '@amzn/codewhisperer-streaming' -import { makeTemporaryToolkitFolder, tryRemoveFolder } from '../../shared/filesystemUtilities' import { activate as activateLsp } from './lspClient' import { telemetry } from '../../shared/telemetry' import { isCloud9 } from '../../shared/extensionUtilities' -import { fs, globals, ToolkitError } from '../../shared' -import { isWeb } from '../../shared/extensionGlobals' -import { getUserAgent } from '../../shared/telemetry/util' +import globals, { isWeb } from '../../shared/extensionGlobals' import { isAmazonInternalOs } from '../../shared/vscode/env' +import { LspDownloader, Manifest } from '../../shared/fetchLsp' +import { fs } from '../../shared/fs/fs' +import { makeTemporaryToolkitFolder, tryRemoveFolder } from '../../shared/filesystemUtilities' export interface Chunk { readonly filePath: string @@ -31,31 +26,6 @@ export interface Chunk { readonly programmingLanguage?: string } -export interface Content { - filename: string - url: string - hashes: string[] - bytes: number - serverVersion?: string -} - -export interface Target { - platform: string - arch: string - contents: Content[] -} - -export interface Manifest { - manifestSchemaVersion: string - artifactId: string - artifactDescription: string - isManifestDeprecated: boolean - versions: { - serverVersion: string - isDelisted: boolean - targets: Target[] - }[] -} const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json' // this LSP client in Q extension is only going to work with these LSP server versions const supportedLspServerVersions = ['0.1.29'] @@ -79,202 +49,24 @@ export interface BuildIndexConfig { * Pre-process the input to Index Files API * Post-process the output from Query API */ -export class LspController { +export class LspController extends LspDownloader { static #instance: LspController private _isIndexingInProgress = false + private serverPath: string + private nodePath: string public static get instance() { return (this.#instance ??= new this()) } - constructor() {} - - isIndexingInProgress() { - return this._isIndexingInProgress - } - - async _download(localFile: string, remoteUrl: string) { - const res = await fetch(remoteUrl, { - headers: { - 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), - }, - }) - if (!res.ok) { - throw new ToolkitError(`Failed to download. Error: ${JSON.stringify(res)}`) - } - return new Promise((resolve, reject) => { - const file = createWriteStream(localFile) - res.body.pipe(file) - res.body.on('error', (err) => { - reject(err) - }) - file.on('finish', () => { - file.close(resolve) - }) - }) - } - async fetchManifest() { - try { - const resp = await request.fetch('GET', manifestUrl, { - headers: { - 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), - }, - }).response - if (!resp.ok) { - throw new ToolkitError(`Failed to fetch manifest. Error: ${resp.statusText}`) - } - return resp.json() - } catch (e: any) { - throw new ToolkitError(`Failed to fetch manifest. Error: ${JSON.stringify(e)}`) - } - } - - async getFileSha384(filePath: string): Promise { - const fileBuffer = await fs.readFileBytes(filePath) - const hash = crypto.createHash('sha384') - hash.update(fileBuffer) - return hash.digest('hex') - } - - async isLspInstalled(context: vscode.ExtensionContext) { - const localQServer = context.asAbsolutePath(path.join('resources', 'qserver')) - const localNodeRuntime = context.asAbsolutePath(path.join('resources', nodeBinName)) - return (await fs.exists(localQServer)) && (await fs.exists(localNodeRuntime)) - } - - getQserverFromManifest(manifest: Manifest): Content | undefined { - if (manifest.isManifestDeprecated) { - return undefined - } - for (const version of manifest.versions) { - if (version.isDelisted) { - continue - } - if (!supportedLspServerVersions.includes(version.serverVersion)) { - continue - } - for (const t of version.targets) { - if ( - (t.platform === process.platform || (t.platform === 'windows' && process.platform === 'win32')) && - t.arch === process.arch - ) { - for (const content of t.contents) { - if (content.filename.startsWith('qserver') && content.hashes.length > 0) { - content.serverVersion = version.serverVersion - return content - } - } - } - } - } - return undefined - } - - getNodeRuntimeFromManifest(manifest: Manifest): Content | undefined { - if (manifest.isManifestDeprecated) { - return undefined - } - for (const version of manifest.versions) { - if (version.isDelisted) { - continue - } - if (!supportedLspServerVersions.includes(version.serverVersion)) { - continue - } - for (const t of version.targets) { - if ( - (t.platform === process.platform || (t.platform === 'windows' && process.platform === 'win32')) && - t.arch === process.arch - ) { - for (const content of t.contents) { - if (content.filename.startsWith('node') && content.hashes.length > 0) { - content.serverVersion = version.serverVersion - return content - } - } - } - } - } - return undefined - } - - private async hashMatch(filePath: string, content: Content) { - const sha384 = await this.getFileSha384(filePath) - if ('sha384:' + sha384 !== content.hashes[0]) { - getLogger().error( - `LspController: Downloaded file sha ${sha384} does not match manifest ${content.hashes[0]}.` - ) - await fs.delete(filePath) - return false - } - return true - } - - async downloadAndCheckHash(filePath: string, content: Content) { - await this._download(filePath, content.url) - const match = await this.hashMatch(filePath, content) - if (!match) { - return false - } - return true + constructor() { + super(manifestUrl, supportedLspServerVersions) + this.serverPath = globals.context.asAbsolutePath(path.join('resources', 'qserver')) + this.nodePath = globals.context.asAbsolutePath(path.join('resources', nodeBinName)) } - async tryInstallLsp(context: vscode.ExtensionContext): Promise { - let tempFolder = undefined - try { - if (await this.isLspInstalled(context)) { - getLogger().info(`LspController: LSP already installed`) - return true - } - // clean up previous downloaded LSP - const qserverPath = context.asAbsolutePath(path.join('resources', 'qserver')) - if (await fs.exists(qserverPath)) { - await tryRemoveFolder(qserverPath) - } - // clean up previous downloaded node runtime - const nodeRuntimePath = context.asAbsolutePath(path.join('resources', nodeBinName)) - if (await fs.exists(nodeRuntimePath)) { - await fs.delete(nodeRuntimePath) - } - // fetch download url for qserver and node runtime - const manifest: Manifest = (await this.fetchManifest()) as Manifest - const qserverContent = this.getQserverFromManifest(manifest) - const nodeRuntimeContent = this.getNodeRuntimeFromManifest(manifest) - if (!qserverContent || !nodeRuntimeContent) { - getLogger().info(`LspController: Did not find LSP URL for ${process.platform} ${process.arch}`) - return false - } - - tempFolder = await makeTemporaryToolkitFolder() - - // download lsp to temp folder - const qserverZipTempPath = path.join(tempFolder, 'qserver.zip') - const downloadOk = await this.downloadAndCheckHash(qserverZipTempPath, qserverContent) - if (!downloadOk) { - return false - } - const zip = new AdmZip(qserverZipTempPath) - zip.extractAllTo(tempFolder) - await fs.rename(path.join(tempFolder, 'qserver'), qserverPath) - - // download node runtime to temp folder - const nodeRuntimeTempPath = path.join(tempFolder, nodeBinName) - const downloadNodeOk = await this.downloadAndCheckHash(nodeRuntimeTempPath, nodeRuntimeContent) - if (!downloadNodeOk) { - return false - } - await fs.chmod(nodeRuntimeTempPath, 0o755) - await fs.rename(nodeRuntimeTempPath, nodeRuntimePath) - return true - } catch (e) { - getLogger().error(`LspController: Failed to setup LSP server ${e}`) - return false - } finally { - // clean up temp folder - if (tempFolder) { - await tryRemoveFolder(tempFolder) - } - } + isIndexingInProgress() { + return this._isIndexingInProgress } async query(s: string): Promise { @@ -378,6 +170,54 @@ export class LspController { } } + async isLspInstalled(): Promise { + return (await fs.exists(this.serverPath)) && (await fs.exists(this.nodePath)) + } + + async cleanup(): Promise { + if (await fs.exists(this.serverPath)) { + await tryRemoveFolder(this.serverPath) + } + + if (await fs.exists(this.nodePath)) { + await fs.delete(this.nodePath) + } + + return true + } + + async install(manifest: Manifest) { + const server = this.getDependency(manifest, 'qserver') + const runtime = this.getDependency(manifest, 'node') + if (!server || !runtime) { + getLogger('lsp').info(`Did not find LSP URL for ${process.platform} ${process.arch}`) + return false + } + + let tempFolder = undefined + + try { + tempFolder = await makeTemporaryToolkitFolder() + await this.downloadAndExtractServer({ + content: server, + installLocation: this.serverPath, + name: 'qserver', + tempFolder, + extractToTempFolder: true, + }) + + const runtimeTempPath = path.join(tempFolder, nodeBinName) + await this.installRuntime(runtime, this.nodePath, runtimeTempPath) + } finally { + // clean up temp folder + if (tempFolder) { + await tryRemoveFolder(tempFolder) + } + } + + return true + } + async trySetupLsp(context: vscode.ExtensionContext, buildIndexConfig: BuildIndexConfig) { if (isCloud9() || isWeb() || isAmazonInternalOs()) { getLogger().warn('LspController: Skipping LSP setup. LSP is not compatible with the current environment. ') @@ -385,7 +225,7 @@ export class LspController { return } setImmediate(async () => { - const ok = await LspController.instance.tryInstallLsp(context) + const ok = await LspController.instance.tryInstallLsp() if (!ok) { return } diff --git a/packages/core/src/shared/fetchLsp.ts b/packages/core/src/shared/fetchLsp.ts new file mode 100644 index 00000000000..d57b05dabdc --- /dev/null +++ b/packages/core/src/shared/fetchLsp.ts @@ -0,0 +1,214 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'path' +import * as crypto from 'crypto' +import fs from './fs/fs' +import { getLogger } from './logger/logger' +import request from './request' +import { getUserAgent } from './telemetry/util' +import { ToolkitError } from './errors' +import fetch from 'node-fetch' +// TODO remove +// eslint-disable-next-line no-restricted-imports +import { createWriteStream } from 'fs' +import AdmZip from 'adm-zip' + +export interface Content { + filename: string + url: string + hashes: string[] + bytes: number + serverVersion?: string +} + +export interface Target { + platform: string + arch: string + contents: Content[] +} + +export interface Manifest { + manifestSchemaVersion: string + artifactId: string + artifactDescription: string + isManifestDeprecated: boolean + versions: { + serverVersion: string + isDelisted: boolean + targets: Target[] + }[] +} + +export abstract class LspDownloader { + constructor( + private readonly manifestURL: string, + private readonly supportedLspServerVersions?: string[] + ) {} + + async fetchManifest() { + try { + const resp = await request.fetch('GET', this.manifestURL, { + headers: { + 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), + }, + }).response + if (!resp.ok) { + throw new ToolkitError(`Failed to fetch manifest. Error: ${resp.statusText}`) + } + return resp.json() + } catch (e: any) { + throw new ToolkitError(`Failed to fetch manifest. Error: ${JSON.stringify(e)}`) + } + } + + async _download(localFile: string, remoteUrl: string) { + const res = await fetch(remoteUrl, { + headers: { + 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), + }, + }) + if (!res.ok) { + throw new ToolkitError(`Failed to download. Error: ${JSON.stringify(res)}`) + } + return new Promise((resolve, reject) => { + const file = createWriteStream(localFile) + res.body.pipe(file) + res.body.on('error', (err) => { + reject(err) + }) + file.on('finish', () => { + file.close(resolve) + }) + }) + } + + async getFileSha384(filePath: string): Promise { + const fileBuffer = await fs.readFileBytes(filePath) + const hash = crypto.createHash('sha384') + hash.update(fileBuffer) + return hash.digest('hex') + } + + private async hashMatch(filePath: string, content: Content) { + const sha384 = await this.getFileSha384(filePath) + if ('sha384:' + sha384 !== content.hashes[0]) { + getLogger('lsp').error(`Downloaded file sha ${sha384} does not match manifest ${content.hashes[0]}.`) + await fs.delete(filePath) + return false + } + return true + } + + async downloadAndCheckHash(filePath: string, content: Content) { + await this._download(filePath, content.url) + const match = await this.hashMatch(filePath, content) + if (!match) { + return false + } + return true + } + + getDependency(manifest: Manifest, name: string): Content | undefined { + if (manifest.isManifestDeprecated) { + return undefined + } + for (const version of manifest.versions) { + if (version.isDelisted) { + continue + } + if (this.supportedLspServerVersions && !this.supportedLspServerVersions.includes(version.serverVersion)) { + continue + } + for (const t of version.targets) { + if ( + (t.platform === process.platform || (t.platform === 'windows' && process.platform === 'win32')) && + t.arch === process.arch + ) { + for (const content of t.contents) { + if (content.filename.startsWith(name) && content.hashes.length > 0) { + content.serverVersion = version.serverVersion + return content + } + } + } + } + } + return undefined + } + + async downloadAndExtractServer({ + content, + installLocation, + name, + tempFolder, + extractToTempFolder = false, + }: { + content: Content + installLocation: string + name: string + tempFolder: string + extractToTempFolder?: boolean + }) { + const serverZipTempPath = path.join(tempFolder, `${name}.zip`) + const downloadOk = await this.downloadAndCheckHash(serverZipTempPath, content) + if (!downloadOk) { + return false + } + + // load the zip contents + const extractPath = extractToTempFolder ? tempFolder : path.join(tempFolder, name) + new AdmZip(serverZipTempPath).extractAllTo(extractPath) + + await fs.rename(path.join(tempFolder, name), installLocation) + } + + async installRuntime(runtime: Content, installLocation: string, tempPath: string) { + const downloadNodeOk = await this.downloadAndCheckHash(tempPath, runtime) + if (!downloadNodeOk) { + return false + } + await fs.chmod(tempPath, 0o755) + await fs.rename(tempPath, installLocation) + } + + /** + * Detect if the lsps already exist on the filesystem + */ + abstract isLspInstalled(): Promise + + /** + * Cleanup any old LSPs or runtimes if they exist + */ + abstract cleanup(): Promise + + /** + * Given a manifest install any servers and runtimes that are required + */ + abstract install(manifest: Manifest): Promise + + async tryInstallLsp(): Promise { + try { + if (await this.isLspInstalled()) { + getLogger('lsp').info(`LSP already installed`) + return true + } + + const clean = await this.cleanup() + if (!clean) { + getLogger('lsp').error(`Failed to clean up old LSPs`) + return false + } + + // fetch download url for server and runtime + const manifest: Manifest = (await this.fetchManifest()) as Manifest + + return await this.install(manifest) + } catch (e) { + getLogger().error(`LspController: Failed to setup LSP server ${e}`) + return false + } + } +} diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index 5942f6ba6c2..f68221e7a4c 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -59,3 +59,4 @@ export { i18n } from './i18n-helper' export * from './icons' export * as textDocumentUtil from './utilities/textDocumentUtilities' export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants' +export * from './fetchLsp' diff --git a/packages/core/src/shared/logger/logger.ts b/packages/core/src/shared/logger/logger.ts index 9cc04aa6585..a4f0ec2d6c5 100644 --- a/packages/core/src/shared/logger/logger.ts +++ b/packages/core/src/shared/logger/logger.ts @@ -5,7 +5,7 @@ import * as vscode from 'vscode' -export type LogTopic = 'crashMonitoring' | 'dev/beta' | 'notifications' | 'test' | 'unknown' +export type LogTopic = 'crashMonitoring' | 'dev/beta' | 'notifications' | 'test' | 'unknown' | 'lsp' class ErrorLog { constructor( diff --git a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts index d84da8626d3..9fdd5ee8bdd 100644 --- a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts +++ b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts @@ -8,7 +8,7 @@ import { Content } from 'aws-sdk/clients/codecommit' import AdmZip from 'adm-zip' import path from 'path' import { LspController } from '../../amazonq' -import { fs, getRandomString, globals } from '../../shared' +import { fs, getRandomString } from '../../shared' import { createTestWorkspace } from '../../test/testUtil' import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' import { getFsCallsUpperBound } from './utilities' @@ -37,8 +37,11 @@ function createStubs(numberOfFiles: number, fileSize: number): sinon.SinonSpiedI // Avoid making HTTP request or mocking giant manifest, stub what we need directly from request. sinon.stub(LspController.prototype, 'fetchManifest') // Directly feed the runtime specifications. - sinon.stub(LspController.prototype, 'getQserverFromManifest').returns(fakeQServerContent) - sinon.stub(LspController.prototype, 'getNodeRuntimeFromManifest').returns(fakeNodeContent) + sinon + .stub(LspController.prototype, 'getDependency') + .withArgs(sinon.match.any, 'qserver') + .returns(fakeQServerContent) + sinon.stub(LspController.prototype, 'getDependency').withArgs(sinon.match.any, 'node').returns(fakeNodeContent) // avoid fetch call. sinon.stub(LspController.prototype, '_download').callsFake(getFakeDownload(numberOfFiles, fileSize)) // Hard code the hash since we are creating files on the spot, whose hashes can't be predicted. @@ -86,7 +89,7 @@ function performanceTestWrapper(numFiles: number, fileSize: number, message: str return createStubs(numFiles, fileSize) }, execute: async () => { - return await LspController.instance.tryInstallLsp(globals.context) + return await LspController.instance.tryInstallLsp() }, verify: async (fsSpy: sinon.SinonSpiedInstance, result: boolean) => { assert.ok(result) From 5d6512f458cad105178671c0e76f59424fc71ba7 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:23:52 -0500 Subject: [PATCH 03/22] feat(amazonq): Use codewhisperer language server for completions (#6158) ## Problem - vscode doesn't have the codewhisperer lsp for completion requests ## Solution - use the codewhisperer lsp when `"aws.experiments": { "amazonqLSP": true },` --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- package-lock.json | 103 ++++++++- packages/amazonq/src/chat/activation.ts | 94 ++++++++ packages/amazonq/src/chat/handler.ts | 260 ++++++++++++++++++++++ packages/amazonq/src/inline/completion.ts | 117 ++++++++++ packages/amazonq/src/lsp/activation.ts | 11 +- packages/amazonq/src/lsp/auth.ts | 98 ++++++++ packages/amazonq/src/lsp/client.ts | 110 +++++++++ packages/core/package.json | 2 + 8 files changed, 785 insertions(+), 10 deletions(-) create mode 100644 packages/amazonq/src/chat/activation.ts create mode 100644 packages/amazonq/src/chat/handler.ts create mode 100644 packages/amazonq/src/inline/completion.ts create mode 100644 packages/amazonq/src/lsp/auth.ts create mode 100644 packages/amazonq/src/lsp/client.ts diff --git a/package-lock.json b/package-lock.json index 8bf434177b1..b67c9a5aa3c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5163,6 +5163,15 @@ "node": ">=14.14" } }, + "node_modules/@aws/chat-client-ui-types": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@aws/chat-client-ui-types/-/chat-client-ui-types-0.0.8.tgz", + "integrity": "sha512-aU8r0FaCKIhMiTWvr/yuWYZmVWPgE2vBAPsVcafhlu7ucubiH/+YodqDw+0Owk0R0kxxZDdjdZghPZSyy0G84A==", + "dev": true, + "dependencies": { + "@aws/language-server-runtimes-types": "^0.0.7" + } + }, "node_modules/@aws/fully-qualified-names": { "version": "2.1.4", "dev": true, @@ -5171,6 +5180,93 @@ "web-tree-sitter": "^0.20.8" } }, + "node_modules/@aws/language-server-runtimes": { + "version": "0.2.27", + "resolved": "https://registry.npmjs.org/@aws/language-server-runtimes/-/language-server-runtimes-0.2.27.tgz", + "integrity": "sha512-qWog7upRVc09xLcuL0HladoxO3JbkgdtgkI/RUWRDcr6YB8hBvmSCADGWjUGbOyvK4CpaXqHIr883PAqnosoXg==", + "dev": true, + "dependencies": { + "@aws/language-server-runtimes-types": "^0.0.7", + "jose": "^5.9.6", + "rxjs": "^7.8.1", + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-protocol": "^3.17.5" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws/language-server-runtimes-types": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/@aws/language-server-runtimes-types/-/language-server-runtimes-types-0.0.7.tgz", + "integrity": "sha512-P83YkgWITcUGHaZvYFI0N487nWErgRpejALKNm/xs8jEcHooDfjigOpliN8TgzfF9BGvGeQnnAzIG16UBXc9ig==", + "dev": true, + "dependencies": { + "vscode-languageserver-textdocument": "^1.0.12", + "vscode-languageserver-types": "^3.17.5" + } + }, + "node_modules/@aws/language-server-runtimes-types/node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "dev": true + }, + "node_modules/@aws/language-server-runtimes/node_modules/jose": { + "version": "5.9.6", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.9.6.tgz", + "integrity": "sha512-AMlnetc9+CV9asI19zHmrgS/WYsWUwCn2R7RzlbJWD7F9eWYUTGyBmU9o6PxngtLGOiDGPRu+Uc4fhKzbpteZQ==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/@aws/language-server-runtimes/node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "dev": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/@aws/language-server-runtimes/node_modules/vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "dev": true, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws/language-server-runtimes/node_modules/vscode-languageserver": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz", + "integrity": "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==", + "dev": true, + "dependencies": { + "vscode-languageserver-protocol": "3.17.5" + }, + "bin": { + "installServerIntoExtension": "bin/installServerIntoExtension" + } + }, + "node_modules/@aws/language-server-runtimes/node_modules/vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "dev": true, + "dependencies": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + } + }, + "node_modules/@aws/language-server-runtimes/node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "dev": true + }, "node_modules/@aws/mynah-ui": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/@aws/mynah-ui/-/mynah-ui-4.21.0.tgz", @@ -18861,8 +18957,9 @@ "license": "MIT" }, "node_modules/vscode-languageserver-textdocument": { - "version": "1.0.8", - "license": "MIT" + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.12.tgz", + "integrity": "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==" }, "node_modules/vscode-languageserver-types": { "version": "3.17.3", @@ -20086,7 +20183,9 @@ }, "devDependencies": { "@aws-sdk/types": "^3.13.1", + "@aws/chat-client-ui-types": "^0.0.8", "@aws/fully-qualified-names": "^2.1.4", + "@aws/language-server-runtimes": "^0.2.27", "@cspotcode/source-map-support": "^0.8.1", "@sinonjs/fake-timers": "^10.0.2", "@types/adm-zip": "^0.4.34", diff --git a/packages/amazonq/src/chat/activation.ts b/packages/amazonq/src/chat/activation.ts new file mode 100644 index 00000000000..8a7bb8c9736 --- /dev/null +++ b/packages/amazonq/src/chat/activation.ts @@ -0,0 +1,94 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + CancellationToken, + Uri, + Webview, + WebviewView, + WebviewViewProvider, + WebviewViewResolveContext, + window, +} from 'vscode' +import { LanguageClient } from 'vscode-languageclient' +import { globals } from 'aws-core-vscode/shared' +import { handle } from './handler' + +export class AmazonQChatViewProvider implements WebviewViewProvider { + public static readonly viewType = 'aws.AmazonQChatView' + + constructor(private readonly client: LanguageClient) {} + + public async resolveWebviewView( + webviewView: WebviewView, + context: WebviewViewResolveContext, + _token: CancellationToken + ) { + webviewView.webview.options = { + enableScripts: true, + localResourceRoots: [Uri.joinPath(globals.context.extensionUri, 'resources', 'qdeveloperclient')], + } + + webviewView.webview.html = this.getWebviewContent(webviewView.webview, globals.context.extensionUri) + handle(this.client, webviewView.webview) + } + + private getWebviewContent(webView: Webview, extensionUri: Uri) { + return ` + + + + + + Chat UI + ${this.generateCss()} + + + ${this.generateJS(webView, extensionUri)} + + ` + } + + private generateCss() { + return ` + ` + } + + private generateJS(webView: Webview, extensionUri: Uri): string { + const assetsPath = Uri.joinPath(extensionUri) + const chatUri = Uri.joinPath(assetsPath, 'resources', 'qdeveloperclient', 'amazonq-ui.js') + + const entrypoint = webView.asWebviewUri(chatUri) + + return ` + + + ` + } +} + +export function registerChat(client: LanguageClient) { + const panel = new AmazonQChatViewProvider(client) + window.registerWebviewViewProvider(AmazonQChatViewProvider.viewType, panel, { + webviewOptions: { + retainContextWhenHidden: true, + }, + }) +} diff --git a/packages/amazonq/src/chat/handler.ts b/packages/amazonq/src/chat/handler.ts new file mode 100644 index 00000000000..eedf976016d --- /dev/null +++ b/packages/amazonq/src/chat/handler.ts @@ -0,0 +1,260 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import { + isValidAuthFollowUpType, + INSERT_TO_CURSOR_POSITION, + AUTH_FOLLOW_UP_CLICKED, + CHAT_OPTIONS, + COPY_TO_CLIPBOARD, +} from '@aws/chat-client-ui-types' +import { + ChatResult, + chatRequestType, + ChatParams, + followUpClickNotificationType, + quickActionRequestType, + QuickActionResult, + QuickActionParams, + insertToCursorPositionNotificationType, +} from '@aws/language-server-runtimes/protocol' +import { v4 as uuidv4 } from 'uuid' +import { Webview, window } from 'vscode' +import { Disposable, LanguageClient, Position, State, TextDocumentIdentifier } from 'vscode-languageclient' +import * as jose from 'jose' +import { encryptionKey } from '../lsp/auth' +import { Commands } from 'aws-core-vscode/shared' + +export function handle(client: LanguageClient, webview: Webview) { + // Listen for Initialize handshake from LSP server to register quick actions dynamically + client.onDidChangeState(({ oldState, newState }) => { + if (oldState === State.Starting && newState === State.Running) { + client.info( + 'Language client received initializeResult from server:', + JSON.stringify(client.initializeResult) + ) + + const chatOptions = client.initializeResult?.awsServerCapabilities?.chatOptions + + void webview.postMessage({ + command: CHAT_OPTIONS, + params: chatOptions, + }) + } + }) + + client.onTelemetry((e) => { + client.info(`[VSCode Client] Received telemetry event from server ${JSON.stringify(e)}`) + }) + + webview.onDidReceiveMessage(async (message) => { + client.info(`[VSCode Client] Received ${JSON.stringify(message)} from chat`) + + switch (message.command) { + case COPY_TO_CLIPBOARD: + client.info('[VSCode Client] Copy to clipboard event received') + break + case INSERT_TO_CURSOR_POSITION: { + const editor = window.activeTextEditor + let textDocument: TextDocumentIdentifier | undefined = undefined + let cursorPosition: Position | undefined = undefined + if (editor) { + cursorPosition = editor.selection.active + textDocument = { uri: editor.document.uri.toString() } + } + + client.sendNotification(insertToCursorPositionNotificationType.method, { + ...message.params, + cursorPosition, + textDocument, + }) + break + } + case AUTH_FOLLOW_UP_CLICKED: + client.info('[VSCode Client] AuthFollowUp clicked') + break + case chatRequestType.method: { + const partialResultToken = uuidv4() + const chatDisposable = client.onProgress(chatRequestType, partialResultToken, (partialResult) => + handlePartialResult(partialResult, encryptionKey, message.params.tabId, webview) + ) + + const editor = + window.activeTextEditor || + window.visibleTextEditors.find((editor) => editor.document.languageId !== 'Log') + if (editor) { + message.params.cursorPosition = [editor.selection.active] + message.params.textDocument = { uri: editor.document.uri.toString() } + } + + const chatRequest = await encryptRequest(message.params, encryptionKey) + const chatResult = (await client.sendRequest(chatRequestType.method, { + ...chatRequest, + partialResultToken, + })) as string | ChatResult + void handleCompleteResult( + chatResult, + encryptionKey, + message.params.tabId, + chatDisposable, + webview + ) + break + } + case quickActionRequestType.method: { + const quickActionPartialResultToken = uuidv4() + const quickActionDisposable = client.onProgress( + quickActionRequestType, + quickActionPartialResultToken, + (partialResult) => + handlePartialResult( + partialResult, + encryptionKey, + message.params.tabId, + webview + ) + ) + + const quickActionRequest = await encryptRequest(message.params, encryptionKey) + const quickActionResult = (await client.sendRequest(quickActionRequestType.method, { + ...quickActionRequest, + partialResultToken: quickActionPartialResultToken, + })) as string | ChatResult + void handleCompleteResult( + quickActionResult, + encryptionKey, + message.params.tabId, + quickActionDisposable, + webview + ) + break + } + case followUpClickNotificationType.method: + if (!isValidAuthFollowUpType(message.params.followUp.type)) { + client.sendNotification(followUpClickNotificationType.method, message.params) + } + break + default: + if (isServerEvent(message.command)) { + client.sendNotification(message.command, message.params) + } + break + } + }, undefined) + + registerGenericCommand('aws.amazonq.explainCode', 'Explain', webview) + registerGenericCommand('aws.amazonq.refactorCode', 'Refactor', webview) + registerGenericCommand('aws.amazonq.fixCode', 'Fix', webview) + registerGenericCommand('aws.amazonq.optimizeCode', 'Optimize', webview) + + Commands.register('aws.amazonq.sendToPrompt', (data) => { + const triggerType = getCommandTriggerType(data) + const selection = getSelectedText() + + void webview.postMessage({ + command: 'sendToPrompt', + params: { selection: selection, triggerType }, + }) + }) +} + +function getSelectedText(): string { + const editor = window.activeTextEditor + if (editor) { + const selection = editor.selection + const selectedText = editor.document.getText(selection) + return selectedText + } + + return ' ' +} + +function getCommandTriggerType(data: any): string { + // data is undefined when commands triggered from keybinding or command palette. Currently no + // way to differentiate keybinding and command palette, so both interactions are recorded as keybinding + return data === undefined ? 'hotkeys' : 'contextMenu' +} + +function registerGenericCommand(commandName: string, genericCommand: string, webview?: Webview) { + Commands.register(commandName, (data) => { + const triggerType = getCommandTriggerType(data) + const selection = getSelectedText() + + void webview?.postMessage({ + command: 'genericCommand', + params: { genericCommand, selection, triggerType }, + }) + }) +} + +function isServerEvent(command: string) { + return command.startsWith('aws/chat/') || command === 'telemetry/event' +} + +// Encrypt the provided request if encryption key exists otherwise do nothing +async function encryptRequest(params: T, encryptionKey: Buffer | undefined): Promise<{ message: string } | T> { + if (!encryptionKey) { + return params + } + + const payload = new TextEncoder().encode(JSON.stringify(params)) + + const encryptedMessage = await new jose.CompactEncrypt(payload) + .setProtectedHeader({ alg: 'dir', enc: 'A256GCM' }) + .encrypt(encryptionKey) + + return { message: encryptedMessage } +} + +async function decodeRequest(request: string, key: Buffer): Promise { + const result = await jose.jwtDecrypt(request, key, { + clockTolerance: 60, // Allow up to 60 seconds to account for clock differences + contentEncryptionAlgorithms: ['A256GCM'], + keyManagementAlgorithms: ['dir'], + }) + + if (!result.payload) { + throw new Error('JWT payload not found') + } + return result.payload as T +} + +async function handlePartialResult( + partialResult: string | T, + encryptionKey: Buffer | undefined, + tabId: string, + webview: Webview +) { + const decryptedMessage = + typeof partialResult === 'string' && encryptionKey + ? await decodeRequest(partialResult, encryptionKey) + : (partialResult as T) + + if (decryptedMessage.body) { + void webview?.postMessage({ + command: chatRequestType.method, + params: decryptedMessage, + isPartialResult: true, + tabId: tabId, + }) + } +} + +async function handleCompleteResult( + result: string | T, + encryptionKey: Buffer | undefined, + tabId: string, + disposable: Disposable, + webview: Webview +) { + const decryptedMessage = + typeof result === 'string' && encryptionKey ? await decodeRequest(result, encryptionKey) : result + + void webview?.postMessage({ + command: chatRequestType.method, + params: decryptedMessage, + tabId: tabId, + }) + disposable.dispose() +} diff --git a/packages/amazonq/src/inline/completion.ts b/packages/amazonq/src/inline/completion.ts new file mode 100644 index 00000000000..c4ff07b9648 --- /dev/null +++ b/packages/amazonq/src/inline/completion.ts @@ -0,0 +1,117 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + CancellationToken, + InlineCompletionContext, + InlineCompletionItem, + InlineCompletionItemProvider, + InlineCompletionList, + Position, + TextDocument, + commands, + languages, +} from 'vscode' +import { LanguageClient } from 'vscode-languageclient' +import { + InlineCompletionItemWithReferences, + InlineCompletionListWithReferences, + InlineCompletionWithReferencesParams, + inlineCompletionWithReferencesRequestType, + logInlineCompletionSessionResultsNotificationType, + LogInlineCompletionSessionResultsParams, +} from '@aws/language-server-runtimes/protocol' + +export const CodewhispererInlineCompletionLanguages = [ + { scheme: 'file', language: 'typescript' }, + { scheme: 'file', language: 'javascript' }, + { scheme: 'file', language: 'json' }, + { scheme: 'file', language: 'yaml' }, + { scheme: 'file', language: 'java' }, + { scheme: 'file', language: 'go' }, + { scheme: 'file', language: 'php' }, + { scheme: 'file', language: 'rust' }, + { scheme: 'file', language: 'kotlin' }, + { scheme: 'file', language: 'terraform' }, + { scheme: 'file', language: 'ruby' }, + { scheme: 'file', language: 'shellscript' }, + { scheme: 'file', language: 'dart' }, + { scheme: 'file', language: 'lua' }, + { scheme: 'file', language: 'powershell' }, + { scheme: 'file', language: 'r' }, + { scheme: 'file', language: 'swift' }, + { scheme: 'file', language: 'systemverilog' }, + { scheme: 'file', language: 'scala' }, + { scheme: 'file', language: 'vue' }, + { scheme: 'file', language: 'csharp' }, +] + +export function registerInlineCompletion(languageClient: LanguageClient) { + const inlineCompletionProvider = new AmazonQInlineCompletionItemProvider(languageClient) + languages.registerInlineCompletionItemProvider(CodewhispererInlineCompletionLanguages, inlineCompletionProvider) + + const onInlineAcceptance = async ( + sessionId: string, + itemId: string, + requestStartTime: number, + firstCompletionDisplayLatency?: number + ) => { + const params: LogInlineCompletionSessionResultsParams = { + sessionId: sessionId, + completionSessionResult: { + [itemId]: { + seen: true, + accepted: true, + discarded: false, + }, + }, + totalSessionDisplayTime: Date.now() - requestStartTime, + firstCompletionDisplayLatency: firstCompletionDisplayLatency, + } + languageClient.sendNotification(logInlineCompletionSessionResultsNotificationType as any, params) + } + commands.registerCommand('aws.sample-vscode-ext-amazonq.accept', onInlineAcceptance) +} + +export class AmazonQInlineCompletionItemProvider implements InlineCompletionItemProvider { + constructor(private readonly languageClient: LanguageClient) {} + + async provideInlineCompletionItems( + document: TextDocument, + position: Position, + context: InlineCompletionContext, + token: CancellationToken + ): Promise { + const requestStartTime = Date.now() + const request: InlineCompletionWithReferencesParams = { + textDocument: { + uri: document.uri.toString(), + }, + position, + context, + } + + const response = await this.languageClient.sendRequest( + inlineCompletionWithReferencesRequestType as any, + request, + token + ) + + const list: InlineCompletionListWithReferences = response as InlineCompletionListWithReferences + this.languageClient.info(`Client: Received ${list.items.length} suggestions`) + const firstCompletionDisplayLatency = Date.now() - requestStartTime + + // Add completion session tracking and attach onAcceptance command to each item to record used decision + list.items.forEach((item: InlineCompletionItemWithReferences) => { + item.command = { + command: 'aws.sample-vscode-ext-amazonq.accept', + title: 'On acceptance', + arguments: [list.sessionId, item.itemId, requestStartTime, firstCompletionDisplayLatency], + } + }) + + return list as InlineCompletionList + } +} diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts index 44465f8659a..c1aa78c5854 100644 --- a/packages/amazonq/src/lsp/activation.ts +++ b/packages/amazonq/src/lsp/activation.ts @@ -4,18 +4,13 @@ */ import vscode from 'vscode' +import path from 'path' import { AmazonQLSPDownloader } from './download' +import { startLanguageServer } from './client' export async function activate(ctx: vscode.ExtensionContext): Promise { const serverPath = ctx.asAbsolutePath('resources/qdeveloperserver') const clientPath = ctx.asAbsolutePath('resources/qdeveloperclient') await new AmazonQLSPDownloader(serverPath, clientPath).tryInstallLsp() - - /** - * at this point the language server should be installed and available - * at serverPath and mynah ui should be available and serveable at - * clientPath - * - * TODO: actually hook up the language server - */ + await startLanguageServer(ctx, path.join(serverPath, 'aws-lsp-codewhisperer.js')) } diff --git a/packages/amazonq/src/lsp/auth.ts b/packages/amazonq/src/lsp/auth.ts new file mode 100644 index 00000000000..70753e75c6b --- /dev/null +++ b/packages/amazonq/src/lsp/auth.ts @@ -0,0 +1,98 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + ConnectionMetadata, + NotificationType, + RequestType, + ResponseMessage, +} from '@aws/language-server-runtimes/protocol' +import * as jose from 'jose' +import * as crypto from 'crypto' +import { LanguageClient } from 'vscode-languageclient' +import { AuthUtil } from 'aws-core-vscode/codewhisperer' +import { Writable } from 'stream' + +export const encryptionKey = crypto.randomBytes(32) + +/** + * Sends a json payload to the language server, who is waiting to know what the encryption key is. + * Code reference: https://github.com/aws/language-servers/blob/7da212185a5da75a72ce49a1a7982983f438651a/client/vscode/src/credentialsActivation.ts#L77 + */ +export function writeEncryptionInit(stream: Writable): void { + const request = { + version: '1.0', + mode: 'JWT', + key: encryptionKey.toString('base64'), + } + stream.write(JSON.stringify(request)) + stream.write('\n') +} + +/** + * Request for custom notifications that Update Credentials and tokens. + * See core\aws-lsp-core\src\credentials\updateCredentialsRequest.ts for details + */ +export interface UpdateCredentialsRequest { + /** + * Encrypted token (JWT or PASETO) + * The token's contents differ whether IAM or Bearer token is sent + */ + data: string + /** + * Used by the runtime based language servers. + * Signals that this client will encrypt its credentials payloads. + */ + encrypted: boolean +} + +export const notificationTypes = { + updateBearerToken: new RequestType( + 'aws/credentials/token/update' + ), + deleteBearerToken: new NotificationType('aws/credentials/token/delete'), + getConnectionMetadata: new RequestType( + 'aws/credentials/getConnectionMetadata' + ), +} + +/** + * Facade over our VSCode Auth that does crud operations on the language server auth + */ +export class AmazonQLspAuth { + constructor(private readonly client: LanguageClient) {} + + async init() { + const activeConnection = AuthUtil.instance.auth.activeConnection + if (activeConnection?.type === 'sso') { + // send the token to the language server + const token = await AuthUtil.instance.getBearerToken() + await this.updateBearerToken(token) + } + } + + private async updateBearerToken(token: string) { + const request = await this.createUpdateCredentialsRequest({ + token, + }) + + await this.client.sendRequest(notificationTypes.updateBearerToken.method, request) + + this.client.info(`UpdateBearerToken: ${JSON.stringify(request)}`) + } + + private async createUpdateCredentialsRequest(data: any) { + const payload = new TextEncoder().encode(JSON.stringify({ data })) + + const jwt = await new jose.CompactEncrypt(payload) + .setProtectedHeader({ alg: 'dir', enc: 'A256GCM' }) + .encrypt(encryptionKey) + + return { + data: jwt, + encrypted: true, + } + } +} diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts new file mode 100644 index 00000000000..bf29bc6dddd --- /dev/null +++ b/packages/amazonq/src/lsp/client.ts @@ -0,0 +1,110 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import vscode, { env, version } from 'vscode' +import * as nls from 'vscode-nls' +import * as cp from 'child_process' +import * as crypto from 'crypto' +import { LanguageClient, LanguageClientOptions, ServerOptions, TransportKind } from 'vscode-languageclient' +import { registerInlineCompletion } from '../inline/completion' +import { AmazonQLspAuth, notificationTypes, writeEncryptionInit } from './auth' +import { AuthUtil } from 'aws-core-vscode/codewhisperer' +import { ConnectionMetadata } from '@aws/language-server-runtimes/protocol' +import { registerChat } from '../chat/activation' + +const localize = nls.loadMessageBundle() + +export function startLanguageServer(extensionContext: vscode.ExtensionContext, serverPath: string) { + const toDispose = extensionContext.subscriptions + + // The debug options for the server + // --inspect=6009: runs the server in Node's Inspector mode so VS Code can attach to the server for debugging + const debugOptions = { + execArgv: [ + '--nolazy', + '--preserve-symlinks', + '--stdio', + '--pre-init-encryption', + '--set-credentials-encryption-key', + ], + } + + // If the extension is launch in debug mode the debug server options are use + // Otherwise the run options are used + let serverOptions: ServerOptions = { + run: { module: serverPath, transport: TransportKind.ipc }, + debug: { module: serverPath, transport: TransportKind.ipc, options: debugOptions }, + } + + const child = cp.spawn('node', [serverPath, ...debugOptions.execArgv]) + writeEncryptionInit(child.stdin) + + serverOptions = () => Promise.resolve(child) + + const documentSelector = [{ scheme: 'file', language: '*' }] + + // Options to control the language client + const clientOptions: LanguageClientOptions = { + // Register the server for json documents + documentSelector, + initializationOptions: { + aws: { + clientInfo: { + name: env.appName, + version: version, + extension: { + name: `AWS IDE Extensions for VSCode`, // TODO change this to C9/Amazon + version: '0.0.1', + }, + clientId: crypto.randomUUID(), + }, + awsClientCapabilities: { + window: { + notifications: true, + }, + }, + }, + credentials: { + providesBearerToken: true, + }, + }, + } + + const client = new LanguageClient( + 'amazonq', + localize('amazonq.server.name', 'Amazon Q Language Server'), + serverOptions, + clientOptions + ) + + const disposable = client.start() + toDispose.push(disposable) + + const auth = new AmazonQLspAuth(client) + + return client.onReady().then(async () => { + await auth.init() + registerInlineCompletion(client) + registerChat(client) + + // Request handler for when the server wants to know about the clients auth connnection + client.onRequest(notificationTypes.getConnectionMetadata.method, () => { + return { + sso: { + startUrl: AuthUtil.instance.auth.startUrl, + }, + } + }) + + toDispose.push( + AuthUtil.instance.auth.onDidChangeActiveConnection(async () => { + await auth.init() + }), + AuthUtil.instance.auth.onDidDeleteConnection(async () => { + client.sendNotification(notificationTypes.deleteBearerToken.method) + }) + ) + }) +} diff --git a/packages/core/package.json b/packages/core/package.json index 3a3fc432a9d..db401e0383c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -438,6 +438,8 @@ "serveVue": "Local server for Vue.js code for development purposes. Provides faster iteration when updating Vue files" }, "devDependencies": { + "@aws/language-server-runtimes": "^0.2.27", + "@aws/chat-client-ui-types": "^0.0.8", "@aws-sdk/types": "^3.13.1", "@aws/fully-qualified-names": "^2.1.4", "@cspotcode/source-map-support": "^0.8.1", From 20efa947a1e8716f9679fcd082d70e8c5e1ebefc Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:07:01 -0500 Subject: [PATCH 04/22] feat(amazonq): Auto update language servers when new versions are available (#6310) ## Problem: - Only one version of a language server is installed right now ## Solution: - Automatically update when a new version is available in the manifest --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/activation.ts | 16 +- packages/amazonq/src/lsp/download.ts | 77 ---- packages/amazonq/src/lsp/lspInstaller.ts | 37 ++ .../unit/amazonq/lsp/lspController.test.ts | 51 --- packages/core/src/amazonq/lsp/lspClient.ts | 4 +- .../core/src/amazonq/lsp/lspController.ts | 76 +--- .../src/amazonq/lsp/workspaceInstaller.ts | 28 ++ packages/core/src/shared/crypto.ts | 15 +- packages/core/src/shared/fetchLsp.ts | 214 ----------- packages/core/src/shared/globalState.ts | 2 + packages/core/src/shared/index.ts | 4 +- .../src/shared/languageServer/lspResolver.ts | 352 ++++++++++++++++++ .../shared/languageServer/manifestResolver.ts | 111 ++++++ .../core/src/shared/languageServer/types.ts | 53 +++ .../resourcefetcher/httpResourceFetcher.ts | 45 +++ packages/core/src/shared/vscode/env.ts | 12 + .../src/testInteg/perf/getFileSha384.test.ts | 60 --- .../src/testInteg/perf/tryInstallLsp.test.ts | 111 ------ 18 files changed, 671 insertions(+), 597 deletions(-) delete mode 100644 packages/amazonq/src/lsp/download.ts create mode 100644 packages/amazonq/src/lsp/lspInstaller.ts delete mode 100644 packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts create mode 100644 packages/core/src/amazonq/lsp/workspaceInstaller.ts delete mode 100644 packages/core/src/shared/fetchLsp.ts create mode 100644 packages/core/src/shared/languageServer/lspResolver.ts create mode 100644 packages/core/src/shared/languageServer/manifestResolver.ts create mode 100644 packages/core/src/shared/languageServer/types.ts delete mode 100644 packages/core/src/testInteg/perf/getFileSha384.test.ts delete mode 100644 packages/core/src/testInteg/perf/tryInstallLsp.test.ts diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts index c1aa78c5854..ceed3cef624 100644 --- a/packages/amazonq/src/lsp/activation.ts +++ b/packages/amazonq/src/lsp/activation.ts @@ -4,13 +4,17 @@ */ import vscode from 'vscode' -import path from 'path' -import { AmazonQLSPDownloader } from './download' import { startLanguageServer } from './client' +import { AmazonQLSPResolver } from './lspInstaller' +import { ToolkitError } from 'aws-core-vscode/shared' +import path from 'path' export async function activate(ctx: vscode.ExtensionContext): Promise { - const serverPath = ctx.asAbsolutePath('resources/qdeveloperserver') - const clientPath = ctx.asAbsolutePath('resources/qdeveloperclient') - await new AmazonQLSPDownloader(serverPath, clientPath).tryInstallLsp() - await startLanguageServer(ctx, path.join(serverPath, 'aws-lsp-codewhisperer.js')) + try { + const installResult = await new AmazonQLSPResolver().resolve() + await startLanguageServer(ctx, path.join(installResult.assetDirectory, 'servers/aws-lsp-codewhisperer.js')) + } catch (err) { + const e = err as ToolkitError + void vscode.window.showInformationMessage(`Unable to launch amazonq language server: ${e.message}`) + } } diff --git a/packages/amazonq/src/lsp/download.ts b/packages/amazonq/src/lsp/download.ts deleted file mode 100644 index 29966b21495..00000000000 --- a/packages/amazonq/src/lsp/download.ts +++ /dev/null @@ -1,77 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - LspDownloader, - getLogger, - makeTemporaryToolkitFolder, - tryRemoveFolder, - fs, - Manifest, -} from 'aws-core-vscode/shared' - -const manifestURL = 'https://aws-toolkit-language-servers.amazonaws.com/codewhisperer/0/manifest.json' - -export class AmazonQLSPDownloader extends LspDownloader { - constructor( - private readonly serverPath: string, - private readonly clientPath: string - ) { - super(manifestURL) - } - - async isLspInstalled(): Promise { - return (await fs.exists(this.serverPath)) && (await fs.exists(this.clientPath)) - } - - async cleanup(): Promise { - if (await fs.exists(this.serverPath)) { - await tryRemoveFolder(this.serverPath) - } - - if (await fs.exists(this.clientPath)) { - await tryRemoveFolder(this.clientPath) - } - - return true - } - - async install(manifest: Manifest) { - const server = this.getDependency(manifest, 'servers') - const clients = this.getDependency(manifest, 'clients') - if (!server || !clients) { - getLogger('lsp').info(`Did not find LSP URL for ${process.platform} ${process.arch}`) - return false - } - - let tempFolder = undefined - - try { - tempFolder = await makeTemporaryToolkitFolder() - - // download and extract the business logic - await this.downloadAndExtractServer({ - content: server, - installLocation: this.serverPath, - name: 'qdeveloperserver', - tempFolder, - }) - - // download and extract mynah ui - await this.downloadAndExtractServer({ - content: clients, - installLocation: this.clientPath, - name: 'qdeveloperclient', - tempFolder, - }) - } finally { - if (tempFolder) { - await tryRemoveFolder(tempFolder) - } - } - - return true - } -} diff --git a/packages/amazonq/src/lsp/lspInstaller.ts b/packages/amazonq/src/lsp/lspInstaller.ts new file mode 100644 index 00000000000..120cdebe39c --- /dev/null +++ b/packages/amazonq/src/lsp/lspInstaller.ts @@ -0,0 +1,37 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as vscode from 'vscode' +import { Range } from 'semver' +import { ManifestResolver, LanguageServerResolver, LspResolver, LspResult } from 'aws-core-vscode/shared' + +const manifestURL = 'https://aws-toolkit-language-servers.amazonaws.com/codewhisperer/0/manifest.json' +const supportedLspServerVersions = '^2.3.1' + +export class AmazonQLSPResolver implements LspResolver { + async resolve(): Promise { + const overrideLocation = process.env.AWS_LANGUAGE_SERVER_OVERRIDE + if (overrideLocation) { + void vscode.window.showInformationMessage(`Using language server override location: ${overrideLocation}`) + return { + assetDirectory: overrideLocation, + location: 'override', + version: '0.0.0', + } + } + + // "AmazonQ" is shared across toolkits to provide a common access point, don't change it + const name = 'AmazonQ' + const manifest = await new ManifestResolver(manifestURL, name).resolve() + const installationResult = await new LanguageServerResolver( + manifest, + name, + new Range(supportedLspServerVersions) + ).resolve() + + // TODO Cleanup old versions of language servers + return installationResult + } +} diff --git a/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts b/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts deleted file mode 100644 index 87111b97f1a..00000000000 --- a/packages/amazonq/test/unit/amazonq/lsp/lspController.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -import assert from 'assert' -import sinon from 'sinon' -import { LspController } from 'aws-core-vscode/amazonq' -import { createTestFile } from 'aws-core-vscode/test' -import { fs, Content } from 'aws-core-vscode/shared' - -describe('Amazon Q LSP controller', function () { - it('Download mechanism checks against hash, when hash matches', async function () { - const content = { - filename: 'qserver-linux-x64.zip', - url: 'https://x/0.0.6/qserver-linux-x64.zip', - hashes: [ - 'sha384:768412320f7b0aa5812fce428dc4706b3cae50e02a64caa16a782249bfe8efc4b7ef1ccb126255d196047dfedf17a0a9', - ], - bytes: 512, - } as Content - const lspController = new LspController() - sinon.stub(lspController, '_download') - const mockFileName = 'test_case_1.zip' - const mockDownloadFile = await createTestFile(mockFileName) - await fs.writeFile(mockDownloadFile.fsPath, 'test') - const result = await lspController.downloadAndCheckHash(mockDownloadFile.fsPath, content) - assert.strictEqual(result, true) - }) - - it('Download mechanism checks against hash, when hash does not match', async function () { - const content = { - filename: 'qserver-linux-x64.zip', - url: 'https://x/0.0.6/qserver-linux-x64.zip', - hashes: [ - 'sha384:38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b', - ], - bytes: 512, - } as Content - const lspController = new LspController() - sinon.stub(lspController, '_download') - const mockFileName = 'test_case_2.zip' - const mockDownloadFile = await createTestFile(mockFileName) - await fs.writeFile(mockDownloadFile.fsPath, 'file_content') - const result = await lspController.downloadAndCheckHash(mockDownloadFile.fsPath, content) - assert.strictEqual(result, false) - }) - - afterEach(() => { - sinon.restore() - }) -}) diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts index 359d8d24256..e637edfc647 100644 --- a/packages/core/src/amazonq/lsp/lspClient.ts +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -172,13 +172,11 @@ export class LspClient { * It will create a output channel named Amazon Q Language Server. * This function assumes the LSP server has already been downloaded. */ -export async function activate(extensionContext: ExtensionContext) { +export async function activate(extensionContext: ExtensionContext, serverModule: string) { LspClient.instance const toDispose = extensionContext.subscriptions let rangeFormatting: Disposable | undefined - // The server is implemented in node - const serverModule = path.join(extensionContext.extensionPath, 'resources/qserver/lspServer.js') // The debug options for the server // --inspect=6009: runs the server in Node's Inspector mode so VS Code can attach to the server for debugging const debugOptions = { execArgv: ['--nolazy', '--preserve-symlinks', '--stdio'] } diff --git a/packages/core/src/amazonq/lsp/lspController.ts b/packages/core/src/amazonq/lsp/lspController.ts index 8001c0c43dc..439158a0dec 100644 --- a/packages/core/src/amazonq/lsp/lspController.ts +++ b/packages/core/src/amazonq/lsp/lspController.ts @@ -14,9 +14,7 @@ import { telemetry } from '../../shared/telemetry' import { isCloud9 } from '../../shared/extensionUtilities' import globals, { isWeb } from '../../shared/extensionGlobals' import { isAmazonInternalOs } from '../../shared/vscode/env' -import { LspDownloader, Manifest } from '../../shared/fetchLsp' -import { fs } from '../../shared/fs/fs' -import { makeTemporaryToolkitFolder, tryRemoveFolder } from '../../shared/filesystemUtilities' +import { WorkspaceLSPResolver } from './workspaceInstaller' export interface Chunk { readonly filePath: string @@ -25,13 +23,6 @@ export interface Chunk { readonly relativePath?: string readonly programmingLanguage?: string } - -const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json' -// this LSP client in Q extension is only going to work with these LSP server versions -const supportedLspServerVersions = ['0.1.32'] - -const nodeBinName = process.platform === 'win32' ? 'node.exe' : 'node' - export interface BuildIndexConfig { startUrl?: string maxIndexSize: number @@ -49,22 +40,14 @@ export interface BuildIndexConfig { * Pre-process the input to Index Files API * Post-process the output from Query API */ -export class LspController extends LspDownloader { +export class LspController { static #instance: LspController private _isIndexingInProgress = false - private serverPath: string - private nodePath: string public static get instance() { return (this.#instance ??= new this()) } - constructor() { - super(manifestUrl, supportedLspServerVersions) - this.serverPath = globals.context.asAbsolutePath(path.join('resources', 'qserver')) - this.nodePath = globals.context.asAbsolutePath(path.join('resources', nodeBinName)) - } - isIndexingInProgress() { return this._isIndexingInProgress } @@ -170,54 +153,6 @@ export class LspController extends LspDownloader { } } - async isLspInstalled(): Promise { - return (await fs.exists(this.serverPath)) && (await fs.exists(this.nodePath)) - } - - async cleanup(): Promise { - if (await fs.exists(this.serverPath)) { - await tryRemoveFolder(this.serverPath) - } - - if (await fs.exists(this.nodePath)) { - await fs.delete(this.nodePath) - } - - return true - } - - async install(manifest: Manifest) { - const server = this.getDependency(manifest, 'qserver') - const runtime = this.getDependency(manifest, 'node') - if (!server || !runtime) { - getLogger('lsp').info(`Did not find LSP URL for ${process.platform} ${process.arch}`) - return false - } - - let tempFolder = undefined - - try { - tempFolder = await makeTemporaryToolkitFolder() - await this.downloadAndExtractServer({ - content: server, - installLocation: this.serverPath, - name: 'qserver', - tempFolder, - extractToTempFolder: true, - }) - - const runtimeTempPath = path.join(tempFolder, nodeBinName) - await this.installRuntime(runtime, this.nodePath, runtimeTempPath) - } finally { - // clean up temp folder - if (tempFolder) { - await tryRemoveFolder(tempFolder) - } - } - - return true - } - async trySetupLsp(context: vscode.ExtensionContext, buildIndexConfig: BuildIndexConfig) { if (isCloud9() || isWeb() || isAmazonInternalOs()) { getLogger().warn('LspController: Skipping LSP setup. LSP is not compatible with the current environment. ') @@ -225,12 +160,9 @@ export class LspController extends LspDownloader { return } setImmediate(async () => { - const ok = await LspController.instance.tryInstallLsp() - if (!ok) { - return - } try { - await activateLsp(context) + const installResult = await new WorkspaceLSPResolver().resolve() + await activateLsp(context, path.join(installResult.assetDirectory, 'resources/qserver/lspServer.js')) getLogger().info('LspController: LSP activated') void LspController.instance.buildIndex(buildIndexConfig) // log the LSP server CPU and Memory usage per 30 minutes. diff --git a/packages/core/src/amazonq/lsp/workspaceInstaller.ts b/packages/core/src/amazonq/lsp/workspaceInstaller.ts new file mode 100644 index 00000000000..fda843fa063 --- /dev/null +++ b/packages/core/src/amazonq/lsp/workspaceInstaller.ts @@ -0,0 +1,28 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { LspResolver, LspResult } from '../../shared/languageServer/types' +import { ManifestResolver } from '../../shared/languageServer/manifestResolver' +import { LanguageServerResolver } from '../../shared/languageServer/lspResolver' +import { Range } from 'semver' + +const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json' +// this LSP client in Q extension is only going to work with these LSP server versions +const supportedLspServerVersions = '0.1.32' + +export class WorkspaceLSPResolver implements LspResolver { + async resolve(): Promise { + const name = 'AmazonQ-Workspace' + const manifest = await new ManifestResolver(manifestUrl, name).resolve() + const installationResult = await new LanguageServerResolver( + manifest, + name, + new Range(supportedLspServerVersions) + ).resolve() + + // TODO Cleanup old versions of language servers + return installationResult + } +} diff --git a/packages/core/src/shared/crypto.ts b/packages/core/src/shared/crypto.ts index 808f5730d81..647525098a7 100644 --- a/packages/core/src/shared/crypto.ts +++ b/packages/core/src/shared/crypto.ts @@ -24,11 +24,15 @@ import { isWeb } from './extensionGlobals' export function randomUUID(): `${string}-${string}-${string}-${string}-${string}` { + return getCrypto().randomUUID() +} + +function getCrypto() { if (isWeb()) { - return globalThis.crypto.randomUUID() + return globalThis.crypto } - return require('crypto').randomUUID() + return require('crypto') } /** @@ -54,3 +58,10 @@ export function truncateUuid(uuid: string) { const cleanedUUID = uuid.replace(/-/g, '') return `${cleanedUUID.substring(0, 4)}...${cleanedUUID.substring(cleanedUUID.length - 4)}` } + +export function createHash(algorithm: string, contents: string | Buffer): string { + const crypto = getCrypto() + const hash = crypto.createHash(algorithm) + hash.update(contents) + return `${algorithm}:${hash.digest('hex')}` +} diff --git a/packages/core/src/shared/fetchLsp.ts b/packages/core/src/shared/fetchLsp.ts deleted file mode 100644 index d57b05dabdc..00000000000 --- a/packages/core/src/shared/fetchLsp.ts +++ /dev/null @@ -1,214 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import path from 'path' -import * as crypto from 'crypto' -import fs from './fs/fs' -import { getLogger } from './logger/logger' -import request from './request' -import { getUserAgent } from './telemetry/util' -import { ToolkitError } from './errors' -import fetch from 'node-fetch' -// TODO remove -// eslint-disable-next-line no-restricted-imports -import { createWriteStream } from 'fs' -import AdmZip from 'adm-zip' - -export interface Content { - filename: string - url: string - hashes: string[] - bytes: number - serverVersion?: string -} - -export interface Target { - platform: string - arch: string - contents: Content[] -} - -export interface Manifest { - manifestSchemaVersion: string - artifactId: string - artifactDescription: string - isManifestDeprecated: boolean - versions: { - serverVersion: string - isDelisted: boolean - targets: Target[] - }[] -} - -export abstract class LspDownloader { - constructor( - private readonly manifestURL: string, - private readonly supportedLspServerVersions?: string[] - ) {} - - async fetchManifest() { - try { - const resp = await request.fetch('GET', this.manifestURL, { - headers: { - 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), - }, - }).response - if (!resp.ok) { - throw new ToolkitError(`Failed to fetch manifest. Error: ${resp.statusText}`) - } - return resp.json() - } catch (e: any) { - throw new ToolkitError(`Failed to fetch manifest. Error: ${JSON.stringify(e)}`) - } - } - - async _download(localFile: string, remoteUrl: string) { - const res = await fetch(remoteUrl, { - headers: { - 'User-Agent': getUserAgent({ includePlatform: true, includeClientId: true }), - }, - }) - if (!res.ok) { - throw new ToolkitError(`Failed to download. Error: ${JSON.stringify(res)}`) - } - return new Promise((resolve, reject) => { - const file = createWriteStream(localFile) - res.body.pipe(file) - res.body.on('error', (err) => { - reject(err) - }) - file.on('finish', () => { - file.close(resolve) - }) - }) - } - - async getFileSha384(filePath: string): Promise { - const fileBuffer = await fs.readFileBytes(filePath) - const hash = crypto.createHash('sha384') - hash.update(fileBuffer) - return hash.digest('hex') - } - - private async hashMatch(filePath: string, content: Content) { - const sha384 = await this.getFileSha384(filePath) - if ('sha384:' + sha384 !== content.hashes[0]) { - getLogger('lsp').error(`Downloaded file sha ${sha384} does not match manifest ${content.hashes[0]}.`) - await fs.delete(filePath) - return false - } - return true - } - - async downloadAndCheckHash(filePath: string, content: Content) { - await this._download(filePath, content.url) - const match = await this.hashMatch(filePath, content) - if (!match) { - return false - } - return true - } - - getDependency(manifest: Manifest, name: string): Content | undefined { - if (manifest.isManifestDeprecated) { - return undefined - } - for (const version of manifest.versions) { - if (version.isDelisted) { - continue - } - if (this.supportedLspServerVersions && !this.supportedLspServerVersions.includes(version.serverVersion)) { - continue - } - for (const t of version.targets) { - if ( - (t.platform === process.platform || (t.platform === 'windows' && process.platform === 'win32')) && - t.arch === process.arch - ) { - for (const content of t.contents) { - if (content.filename.startsWith(name) && content.hashes.length > 0) { - content.serverVersion = version.serverVersion - return content - } - } - } - } - } - return undefined - } - - async downloadAndExtractServer({ - content, - installLocation, - name, - tempFolder, - extractToTempFolder = false, - }: { - content: Content - installLocation: string - name: string - tempFolder: string - extractToTempFolder?: boolean - }) { - const serverZipTempPath = path.join(tempFolder, `${name}.zip`) - const downloadOk = await this.downloadAndCheckHash(serverZipTempPath, content) - if (!downloadOk) { - return false - } - - // load the zip contents - const extractPath = extractToTempFolder ? tempFolder : path.join(tempFolder, name) - new AdmZip(serverZipTempPath).extractAllTo(extractPath) - - await fs.rename(path.join(tempFolder, name), installLocation) - } - - async installRuntime(runtime: Content, installLocation: string, tempPath: string) { - const downloadNodeOk = await this.downloadAndCheckHash(tempPath, runtime) - if (!downloadNodeOk) { - return false - } - await fs.chmod(tempPath, 0o755) - await fs.rename(tempPath, installLocation) - } - - /** - * Detect if the lsps already exist on the filesystem - */ - abstract isLspInstalled(): Promise - - /** - * Cleanup any old LSPs or runtimes if they exist - */ - abstract cleanup(): Promise - - /** - * Given a manifest install any servers and runtimes that are required - */ - abstract install(manifest: Manifest): Promise - - async tryInstallLsp(): Promise { - try { - if (await this.isLspInstalled()) { - getLogger('lsp').info(`LSP already installed`) - return true - } - - const clean = await this.cleanup() - if (!clean) { - getLogger('lsp').error(`Failed to clean up old LSPs`) - return false - } - - // fetch download url for server and runtime - const manifest: Manifest = (await this.fetchManifest()) as Manifest - - return await this.install(manifest) - } catch (e) { - getLogger().error(`LspController: Failed to setup LSP server ${e}`) - return false - } - } -} diff --git a/packages/core/src/shared/globalState.ts b/packages/core/src/shared/globalState.ts index 5cce9ff6f84..a26437f0951 100644 --- a/packages/core/src/shared/globalState.ts +++ b/packages/core/src/shared/globalState.ts @@ -46,6 +46,8 @@ export type globalKey = | 'aws.amazonq.workspaceIndexToggleOn' | 'aws.toolkit.separationPromptCommand' | 'aws.toolkit.separationPromptDismissed' + | 'aws.toolkit.lsp.versions' + | 'aws.toolkit.lsp.manifest' // Deprecated/legacy names. New keys should start with "aws.". | '#sessionCreationDates' // Legacy name from `ssoAccessTokenProvider.ts`. | 'CODECATALYST_RECONNECT' diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index f68221e7a4c..a77f45d9a91 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -59,4 +59,6 @@ export { i18n } from './i18n-helper' export * from './icons' export * as textDocumentUtil from './utilities/textDocumentUtilities' export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants' -export * from './fetchLsp' +export * from './languageServer/manifestResolver' +export * from './languageServer/lspResolver' +export * from './languageServer/types' diff --git a/packages/core/src/shared/languageServer/lspResolver.ts b/packages/core/src/shared/languageServer/lspResolver.ts new file mode 100644 index 00000000000..636163327ab --- /dev/null +++ b/packages/core/src/shared/languageServer/lspResolver.ts @@ -0,0 +1,352 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from '../fs/fs' +import { ToolkitError } from '../errors' +import * as semver from 'semver' +import * as path from 'path' +import { FileType } from 'vscode' +import AdmZip from 'adm-zip' +import { TargetContent, logger, LspResult, LspVersion, Manifest } from './types' +import { getApplicationSupportFolder } from '../vscode/env' +import { createHash } from '../crypto' +import request from '../request' + +export class LanguageServerResolver { + constructor( + private readonly manifest: Manifest, + private readonly lsName: string, + private readonly versionRange: semver.Range, + private readonly _defaultDownloadFolder?: string + ) {} + + /** + * Downloads and sets up the Language Server, attempting different locations in order: + * 1. Local cache + * 2. Remote download + * 3. Fallback version + * @throws ToolkitError if no compatible version can be found + */ + async resolve() { + const result: LspResult = { + location: 'unknown', + version: '', + assetDirectory: '', + } + + const latestVersion = this.latestCompatibleLspVersion() + const targetContents = this.getLSPTargetContents(latestVersion) + const cacheDirectory = this.getDownloadDirectory(latestVersion.serverVersion) + + if (await this.hasValidLocalCache(cacheDirectory, targetContents)) { + result.location = 'cache' + result.version = latestVersion.serverVersion + result.assetDirectory = cacheDirectory + return result + } else { + // Delete the cached directory since it's invalid + if (await fs.existsDir(cacheDirectory)) { + await fs.delete(cacheDirectory, { + recursive: true, + }) + } + } + + if (await this.downloadRemoteTargetContent(targetContents, latestVersion.serverVersion)) { + result.location = 'remote' + result.version = latestVersion.serverVersion + result.assetDirectory = cacheDirectory + return result + } else { + // clean up any leftover content that may have been downloaded + if (await fs.existsDir(cacheDirectory)) { + await fs.delete(cacheDirectory, { + recursive: true, + }) + } + } + + logger.info( + `Unable to download language server version ${latestVersion.serverVersion}. Attempting to fetch from fallback location` + ) + + const fallbackDirectory = await this.getFallbackDir(latestVersion.serverVersion) + if (!fallbackDirectory) { + throw new ToolkitError('Unable to find a compatible version of the Language Server') + } + + const version = path.basename(cacheDirectory) + logger.info( + `Unable to install language server ${latestVersion.serverVersion}. Launching a previous version from ${fallbackDirectory}` + ) + + result.location = 'fallback' + result.version = version + result.assetDirectory = fallbackDirectory + + return result + } + + /** + * Get all of the compatible language server versions from the manifest + */ + private compatibleManifestLspVersion() { + return this.manifest.versions.filter((x) => this.isCompatibleVersion(x)) + } + + /** + * Returns the path to the most compatible cached LSP version that can serve as a fallback + **/ + private async getFallbackDir(version: string) { + const compatibleLspVersions = this.compatibleManifestLspVersion() + + // determine all folders containing lsp versions in the fallback parent folder + const cachedVersions = (await fs.readdir(this.defaultDownloadFolder())) + .filter(([_, filetype]) => filetype === FileType.Directory) + .map(([pathName, _]) => semver.parse(pathName)) + .filter((ver): ver is semver.SemVer => ver !== null) + + const expectedVersion = semver.parse(version) + if (!expectedVersion) { + return undefined + } + + const sortedCachedLspVersions = compatibleLspVersions + .filter((v) => this.isValidCachedVersion(v, cachedVersions, expectedVersion)) + .sort((a, b) => semver.compare(a.serverVersion, b.serverVersion)) + + const fallbackDir = ( + await Promise.all(sortedCachedLspVersions.map((ver) => this.getValidLocalCacheDirectory(ver))) + ).filter((v) => v !== undefined) + return fallbackDir.length > 0 ? fallbackDir[0] : undefined + } + + /** + * Validate the local cache directory of the given lsp version (matches expected hash) + * If valid return cache directory, else return undefined + */ + private async getValidLocalCacheDirectory(version: LspVersion) { + const targetContents = this.getTargetContents(version) + if (targetContents === undefined || targetContents.length === 0) { + return undefined + } + + const cacheDir = this.getDownloadDirectory(version.serverVersion) + const hasValidCache = await this.hasValidLocalCache(cacheDir, targetContents) + + return hasValidCache ? cacheDir : undefined + } + + /** + * Determines if a cached LSP version is valid for use as a fallback. + * A version is considered valid if it exists in the cache and is less than + * or equal to the expected version. + */ + private isValidCachedVersion(version: LspVersion, cachedVersions: semver.SemVer[], expectedVersion: semver.SemVer) { + const serverVersion = semver.parse(version.serverVersion) as semver.SemVer + return cachedVersions.find((x) => x === serverVersion) && serverVersion <= expectedVersion + } + + /** + * Download and unzip all of the contents into the download directory + * + * @returns + * true, if all of the contents were successfully downloaded and unzipped + * false, if any of the contents failed to download or unzip + */ + private async downloadRemoteTargetContent(contents: TargetContent[], version: string) { + const downloadDirectory = this.getDownloadDirectory(version) + + if (!(await fs.existsDir(downloadDirectory))) { + await fs.mkdir(downloadDirectory) + } + + const downloadTasks = contents.map(async (content) => { + // TODO This should be using the retryable http library but it doesn't seem to support zips right now + const res = await request.fetch('GET', content.url).response + if (!res.ok || !res.body) { + return false + } + + const arrBuffer = await res.arrayBuffer() + const data = Buffer.from(arrBuffer) + + const hash = createHash('sha384', data) + if (hash === content.hashes[0]) { + await fs.writeFile(`${downloadDirectory}/${content.filename}`, data) + return true + } + return false + }) + const downloadResults = await Promise.all(downloadTasks) + const downloadResult = downloadResults.every(Boolean) + return downloadResult && this.extractZipFilesFromRemote(downloadDirectory) + } + + private async extractZipFilesFromRemote(downloadDirectory: string) { + // Find all the zips + const zips = (await fs.readdir(downloadDirectory)) + .filter(([fileName, _]) => fileName.endsWith('.zip')) + .map(([fileName, _]) => `${downloadDirectory}/${fileName}`) + + if (zips.length === 0) { + return true + } + + return this.copyZipContents(zips) + } + + private async hasValidLocalCache(localCacheDirectory: string, targetContents: TargetContent[]) { + // check if the zips are still at the present location + const results = await Promise.all( + targetContents.map((content) => { + const path = `${localCacheDirectory}/${content.filename}` + return fs.existsFile(path) + }) + ) + + const allFilesExist = results.every(Boolean) + return allFilesExist && this.ensureUnzippedFoldersMatchZip(localCacheDirectory, targetContents) + } + + /** + * Ensures zip files in cache have an unzipped folder of the same name + * with the same content files (by name) + * + * @returns + * false, if any of the unzipped folder don't match zip contents (by name) + */ + private ensureUnzippedFoldersMatchZip(localCacheDirectory: string, targetContents: TargetContent[]) { + const zipPaths = targetContents + .filter((x) => x.filename.endsWith('.zip')) + .map((y) => `${localCacheDirectory}/${y.filename}`) + + if (zipPaths.length === 0) { + return true + } + + return this.copyZipContents(zipPaths) + } + + /** + * Copies all the contents from zip into the directory + * + * @returns + * false, if any of the unzips fails + */ + private copyZipContents(zips: string[]) { + const unzips = zips.map((zip) => { + try { + // attempt to unzip + const zipFile = new AdmZip(zip) + const extractPath = zip.replace('.zip', '') + zipFile.extractAllTo(extractPath, true) + } catch (e) { + return false + } + return true + }) + + // make sure every one completed successfully + return unzips.every(Boolean) + } + + /** + * Parses the toolkit lsp version object retrieved from the version manifest to determine + * lsp contents + */ + private getLSPTargetContents(version: LspVersion) { + const lspTarget = this.getCompatibleLspTarget(version) + if (!lspTarget) { + throw new ToolkitError("No language server target found matching the system's architecture and platform") + } + + const targetContents = lspTarget.contents + if (!targetContents) { + throw new ToolkitError('No matching target contents found') + } + return targetContents + } + + /** + * Get the latest language server version matching the toolkit compatible version range, + * not de-listed and contains the required target contents: + * architecture, platform and files + */ + private latestCompatibleLspVersion() { + if (this.manifest === null) { + throw new ToolkitError('No valid manifest') + } + + const latestCompatibleVersion = + this.manifest.versions + .filter((ver) => this.isCompatibleVersion(ver) && this.hasRequiredTargetContent(ver)) + .sort((a, b) => { + return a.serverVersion.localeCompare(b.serverVersion) + })[0] ?? undefined + + if (latestCompatibleVersion === undefined) { + // TODO fix these error range names + throw new ToolkitError( + `Unable to find a language server that satifies one or more of these conditions: version in range [${this.versionRange.range}], matching system's architecture and platform` + ) + } + + return latestCompatibleVersion + } + + /** + * Determine if the given lsp version is toolkit compatible + * i.e. in version range and not de-listed + */ + private isCompatibleVersion(version: LspVersion) { + // invalid version + if (semver.parse(version.serverVersion) === null) { + return false + } + + return semver.satisfies(version.serverVersion, this.versionRange) && !version.isDelisted + } + + /** + * Validates the lsp version contains the required toolkit compatible contents: + * architecture, platform and file + */ + private hasRequiredTargetContent(version: LspVersion) { + const targetContents = this.getTargetContents(version) + return targetContents !== undefined && targetContents.length > 0 + } + + /** + * Returns the target contents of the lsp version that contains the required + * toolkit compatible contents: architecture, platform and file + */ + private getTargetContents(version: LspVersion) { + const target = this.getCompatibleLspTarget(version) + return target?.contents + } + + /** + * Retrives the lsp target matching the user's system architecture and platform + * from the language server version object + */ + private getCompatibleLspTarget(version: LspVersion) { + // TODO make this web friendly + // TODO make this fully support windows + const platform = process.platform + const arch = process.arch + return version.targets.find((x) => x.arch === arch && x.platform === platform) + } + + private defaultDownloadFolder() { + const applicationSupportFolder = getApplicationSupportFolder() + return path.join(applicationSupportFolder, `aws/toolkits/language-servers/${this.lsName}`) + } + + getDownloadDirectory(version: string) { + const directory = this._defaultDownloadFolder ?? this.defaultDownloadFolder() + return `${directory}/${version}` + } +} diff --git a/packages/core/src/shared/languageServer/manifestResolver.ts b/packages/core/src/shared/languageServer/manifestResolver.ts new file mode 100644 index 00000000000..36685890620 --- /dev/null +++ b/packages/core/src/shared/languageServer/manifestResolver.ts @@ -0,0 +1,111 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { getLogger } from '../logger/logger' +import { ToolkitError } from '../errors' +import { RetryableResourceFetcher } from '../resourcefetcher/httpResourceFetcher' +import { Timeout } from '../utilities/timeoutUtils' +import globals from '../extensionGlobals' +import { Manifest } from './types' + +const logger = getLogger('lsp') + +interface StorageManifest { + etag: string + content: string +} + +type ManifestStorage = Record + +const manifestStorageKey = 'aws.toolkit.lsp.manifest' +const manifestTimeoutMs = 15000 + +export class ManifestResolver { + constructor( + private readonly manifestURL: string, + private readonly lsName: string + ) {} + + /** + * Fetches the latest manifest, falling back to local cache on failure + */ + async resolve(): Promise { + try { + return await this.fetchRemoteManifest() + } catch (error) { + return await this.getLocalManifest() + } + } + + private async fetchRemoteManifest(): Promise { + const resourceFetcher = new RetryableResourceFetcher({ + resource: this.manifestURL, + params: { + timeout: new Timeout(manifestTimeoutMs), + }, + }) + + const resp = await resourceFetcher.getNewETagContent(this.getEtag()) + if (!resp.content) { + throw new ToolkitError('New content was not downloaded; fallback to the locally stored manifest') + } + + const manifest = this.parseManifest(resp.content) + await this.saveManifest(resp.eTag, resp.content) + this.checkDeprecation(manifest) + + return manifest + } + + private async getLocalManifest(): Promise { + logger.info('Failed to download latest LSP manifest. Falling back to local manifest.') + const storage = this.getStorage() + const manifestData = storage[this.lsName] + + if (!manifestData?.content) { + throw new ToolkitError('Failed to download LSP manifest and no local manifest found.') + } + + const manifest = this.parseManifest(manifestData.content) + this.checkDeprecation(manifest) + return manifest + } + + private parseManifest(content: string): Manifest { + try { + return JSON.parse(content) as Manifest + } catch (error) { + throw new ToolkitError( + `Failed to parse manifest: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } + } + + private checkDeprecation(manifest: Manifest): void { + if (manifest.isManifestDeprecated) { + logger.info('This LSP manifest is deprecated. No future updates will be available.') + } + } + + private async saveManifest(etag: string, content: string): Promise { + const storage = this.getStorage() + + globals.globalState.tryUpdate(manifestStorageKey, { + ...storage, + [this.lsName]: { + etag, + content, + }, + }) + } + + private getEtag(): string | undefined { + return this.getStorage()[this.lsName]?.etag + } + + private getStorage(): ManifestStorage { + return globals.globalState.tryGet(manifestStorageKey, Object, {}) + } +} diff --git a/packages/core/src/shared/languageServer/types.ts b/packages/core/src/shared/languageServer/types.ts new file mode 100644 index 00000000000..ec365424eaa --- /dev/null +++ b/packages/core/src/shared/languageServer/types.ts @@ -0,0 +1,53 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { getLogger } from '../logger/logger' + +export const logger = getLogger('lsp') + +type Location = 'remote' | 'cache' | 'override' | 'fallback' | 'unknown' + +export interface LspResult { + location: Location + version: string + assetDirectory: string +} + +export interface LspResolver { + resolve(): Promise +} + +export interface TargetContent { + filename: string + url: string + hashes: string[] + bytes: number + serverVersion?: string +} + +export interface Target { + platform: string + arch: string + contents: TargetContent[] +} + +export interface LspVersion { + serverVersion: string + isDelisted: boolean + targets: Target[] +} + +export interface Manifest { + manifestSchemaVersion: string + artifactId: string + artifactDescription: string + isManifestDeprecated: boolean + versions: LspVersion[] +} + +export interface VersionRange { + start: number + end: number +} diff --git a/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts b/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts index f2da4ba98aa..0f7a7003d41 100644 --- a/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts +++ b/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts @@ -16,6 +16,7 @@ import { ResourceFetcher } from './resourcefetcher' import { Timeout, CancellationError, CancelEvent } from '../utilities/timeoutUtils' import { isCloud9 } from '../extensionUtilities' import { Headers } from 'got/dist/source/core' +import { withRetries } from '../utilities/functionUtils' // XXX: patched Got module for compatability with older VS Code versions (e.g. Cloud9) // `got` has also deprecated `urlToOptions` @@ -201,6 +202,50 @@ export class HttpResourceFetcher implements ResourceFetcher { } } +export class RetryableResourceFetcher extends HttpResourceFetcher { + private readonly retryNumber: number + private readonly retryIntervalMs: number + private readonly resource: string + + constructor({ + resource, + params: { retryNumber = 5, retryIntervalMs = 3000, showUrl = true, timeout = new Timeout(5000) }, + }: { + resource: string + params: { + retryNumber?: number + retryIntervalMs?: number + showUrl?: boolean + timeout?: Timeout + } + }) { + super(resource, { + showUrl, + timeout, + }) + this.retryNumber = retryNumber + this.retryIntervalMs = retryIntervalMs + this.resource = resource + } + + fetch(versionTag?: string) { + return withRetries( + async () => { + try { + return await this.getNewETagContent(versionTag) + } catch (err) { + getLogger('lsp').error('Failed to fetch at endpoint: %s, err: %s', this.resource, err) + throw err + } + }, + { + maxRetries: this.retryNumber, + delay: this.retryIntervalMs, + } + ) + } +} + /** * Retrieves JSON property value from a remote resource * @param property property to retrieve diff --git a/packages/core/src/shared/vscode/env.ts b/packages/core/src/shared/vscode/env.ts index e9c8c1983b1..7d3d0f5d234 100644 --- a/packages/core/src/shared/vscode/env.ts +++ b/packages/core/src/shared/vscode/env.ts @@ -12,6 +12,7 @@ import { onceChanged } from '../utilities/functionUtils' import { ChildProcess } from '../utilities/processUtils' import globals, { isWeb } from '../extensionGlobals' import * as devConfig from '../../dev/config' +import path from 'path' /** * Returns true if the current build is running on CI (build server). @@ -270,3 +271,14 @@ export async function getMachineId(): Promise { // TODO: check exit code. return (await proc.run()).stdout.trim() ?? 'unknown-host' } + +export function getApplicationSupportFolder() { + switch (process.platform) { + case 'darwin': { + return path.join(os.homedir(), 'Library/Application Support') + } + default: { + throw new Error('Only mac is supported right now') + } + } +} diff --git a/packages/core/src/testInteg/perf/getFileSha384.test.ts b/packages/core/src/testInteg/perf/getFileSha384.test.ts deleted file mode 100644 index c7768a3cdd1..00000000000 --- a/packages/core/src/testInteg/perf/getFileSha384.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -import assert from 'assert' -import path from 'path' -import sinon from 'sinon' -import { getTestWorkspaceFolder } from '../integrationTestsUtilities' -import { fs, getRandomString } from '../../shared' -import { LspController } from '../../amazonq' -import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' -import { FileSystem } from '../../shared/fs/fs' -import { getFsCallsUpperBound } from './utilities' - -interface SetupResult { - testFile: string - fsSpy: sinon.SinonSpiedInstance -} - -function performanceTestWrapper(label: string, fileSize: number) { - return performanceTest( - getEqualOSTestOptions({ - userCpuUsage: 400, - systemCpuUsage: 35, - heapTotal: 4, - }), - label, - function () { - return { - setup: async () => { - const workspace = getTestWorkspaceFolder() - const fileContent = getRandomString(fileSize) - const testFile = path.join(workspace, 'test-file') - await fs.writeFile(testFile, fileContent) - const fsSpy = sinon.spy(fs) - return { testFile, fsSpy } - }, - execute: async (setup: SetupResult) => { - return await LspController.instance.getFileSha384(setup.testFile) - }, - verify: async (setup: SetupResult, result: string) => { - assert.strictEqual(result.length, 96) - assert.ok(getFsCallsUpperBound(setup.fsSpy) <= 1, 'makes a single call to fs') - }, - } - } - ) -} - -describe('getFileSha384', function () { - describe('performance tests', function () { - afterEach(function () { - sinon.restore() - }) - performanceTestWrapper('1MB', 1000) - performanceTestWrapper('2MB', 2000) - performanceTestWrapper('4MB', 4000) - performanceTestWrapper('8MB', 8000) - }) -}) diff --git a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts deleted file mode 100644 index 9fdd5ee8bdd..00000000000 --- a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -import assert from 'assert' -import sinon from 'sinon' -import { Content } from 'aws-sdk/clients/codecommit' -import AdmZip from 'adm-zip' -import path from 'path' -import { LspController } from '../../amazonq' -import { fs, getRandomString } from '../../shared' -import { createTestWorkspace } from '../../test/testUtil' -import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' -import { getFsCallsUpperBound } from './utilities' -import { FileSystem } from '../../shared/fs/fs' - -// fakeFileContent is matched to fakeQServerContent based on hash. -const fakeHash = '4eb2865c8f40a322aa04e17d8d83bdaa605d6f1cb363af615240a5442a010e0aef66e21bcf4c88f20fabff06efe8a214' - -const fakeQServerContent = { - filename: 'qserver-fake.zip', - url: 'https://aws-language-servers/fake.zip', - hashes: [`sha384:${fakeHash}`], - bytes: 93610849, - serverVersion: '1.1.1', -} - -const fakeNodeContent = { - filename: 'fake-file', - url: 'https://aws-language-servers.fake-file', - hashes: [`sha384:${fakeHash}`], - bytes: 94144448, - serverVersion: '1.1.1', -} - -function createStubs(numberOfFiles: number, fileSize: number): sinon.SinonSpiedInstance { - // Avoid making HTTP request or mocking giant manifest, stub what we need directly from request. - sinon.stub(LspController.prototype, 'fetchManifest') - // Directly feed the runtime specifications. - sinon - .stub(LspController.prototype, 'getDependency') - .withArgs(sinon.match.any, 'qserver') - .returns(fakeQServerContent) - sinon.stub(LspController.prototype, 'getDependency').withArgs(sinon.match.any, 'node').returns(fakeNodeContent) - // avoid fetch call. - sinon.stub(LspController.prototype, '_download').callsFake(getFakeDownload(numberOfFiles, fileSize)) - // Hard code the hash since we are creating files on the spot, whose hashes can't be predicted. - sinon.stub(LspController.prototype, 'getFileSha384').resolves(fakeHash) - const fsSpy = sinon.spy(fs) - fsSpy.rename.restore() - // Don't allow tryInstallLsp to move runtimes out of temporary folder - sinon.stub(fsSpy, 'rename') - return fsSpy -} - -/** - * Creates a fake zip with some files in it. - * @param filepath where to write the zip to. - * @param _content unused parameter, for compatability with real function. - */ -const getFakeDownload = function (numberOfFiles: number, fileSize: number) { - return async function (filepath: string, _content: Content) { - const dummyFilesPath = ( - await createTestWorkspace(numberOfFiles, { - fileNamePrefix: 'fakeFile', - fileContent: getRandomString(fileSize), - workspaceName: 'workspace', - }) - ).uri.fsPath - await fs.writeFile(path.join(dummyFilesPath, 'qserver'), 'this value shouldnt matter') - const zip = new AdmZip() - zip.addLocalFolder(dummyFilesPath) - zip.writeZip(filepath) - } -} - -function performanceTestWrapper(numFiles: number, fileSize: number, message: string) { - return performanceTest( - getEqualOSTestOptions({ - userCpuUsage: 150, - systemCpuUsage: 35, - heapTotal: 6, - duration: 15, - }), - message, - function () { - return { - setup: async () => { - return createStubs(numFiles, fileSize) - }, - execute: async () => { - return await LspController.instance.tryInstallLsp() - }, - verify: async (fsSpy: sinon.SinonSpiedInstance, result: boolean) => { - assert.ok(result) - assert.ok(getFsCallsUpperBound(fsSpy) <= 6 * numFiles) - }, - } - } - ) -} - -describe('tryInstallLsp', function () { - afterEach(function () { - sinon.restore() - }) - describe('performance tests', function () { - performanceTestWrapper(250, 10, '250x10') - performanceTestWrapper(10, 1000, '10x1000') - }) -}) From 35b679c45d3a8a75f528b1dcad7b054b006a5b37 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:49:29 -0500 Subject: [PATCH 05/22] revert(amazonq): Remove loading mynah ui from language server (#6331) ## Problem we want to make feature/amazonqLSP mergable at all times, but it currently has some demo mynah ui code ## Solution remove the mynah ui code for now --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/chat/activation.ts | 94 --------- packages/amazonq/src/chat/handler.ts | 260 ------------------------ packages/amazonq/src/lsp/client.ts | 2 - 3 files changed, 356 deletions(-) delete mode 100644 packages/amazonq/src/chat/activation.ts delete mode 100644 packages/amazonq/src/chat/handler.ts diff --git a/packages/amazonq/src/chat/activation.ts b/packages/amazonq/src/chat/activation.ts deleted file mode 100644 index 8a7bb8c9736..00000000000 --- a/packages/amazonq/src/chat/activation.ts +++ /dev/null @@ -1,94 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - CancellationToken, - Uri, - Webview, - WebviewView, - WebviewViewProvider, - WebviewViewResolveContext, - window, -} from 'vscode' -import { LanguageClient } from 'vscode-languageclient' -import { globals } from 'aws-core-vscode/shared' -import { handle } from './handler' - -export class AmazonQChatViewProvider implements WebviewViewProvider { - public static readonly viewType = 'aws.AmazonQChatView' - - constructor(private readonly client: LanguageClient) {} - - public async resolveWebviewView( - webviewView: WebviewView, - context: WebviewViewResolveContext, - _token: CancellationToken - ) { - webviewView.webview.options = { - enableScripts: true, - localResourceRoots: [Uri.joinPath(globals.context.extensionUri, 'resources', 'qdeveloperclient')], - } - - webviewView.webview.html = this.getWebviewContent(webviewView.webview, globals.context.extensionUri) - handle(this.client, webviewView.webview) - } - - private getWebviewContent(webView: Webview, extensionUri: Uri) { - return ` - - - - - - Chat UI - ${this.generateCss()} - - - ${this.generateJS(webView, extensionUri)} - - ` - } - - private generateCss() { - return ` - ` - } - - private generateJS(webView: Webview, extensionUri: Uri): string { - const assetsPath = Uri.joinPath(extensionUri) - const chatUri = Uri.joinPath(assetsPath, 'resources', 'qdeveloperclient', 'amazonq-ui.js') - - const entrypoint = webView.asWebviewUri(chatUri) - - return ` - - - ` - } -} - -export function registerChat(client: LanguageClient) { - const panel = new AmazonQChatViewProvider(client) - window.registerWebviewViewProvider(AmazonQChatViewProvider.viewType, panel, { - webviewOptions: { - retainContextWhenHidden: true, - }, - }) -} diff --git a/packages/amazonq/src/chat/handler.ts b/packages/amazonq/src/chat/handler.ts deleted file mode 100644 index eedf976016d..00000000000 --- a/packages/amazonq/src/chat/handler.ts +++ /dev/null @@ -1,260 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -import { - isValidAuthFollowUpType, - INSERT_TO_CURSOR_POSITION, - AUTH_FOLLOW_UP_CLICKED, - CHAT_OPTIONS, - COPY_TO_CLIPBOARD, -} from '@aws/chat-client-ui-types' -import { - ChatResult, - chatRequestType, - ChatParams, - followUpClickNotificationType, - quickActionRequestType, - QuickActionResult, - QuickActionParams, - insertToCursorPositionNotificationType, -} from '@aws/language-server-runtimes/protocol' -import { v4 as uuidv4 } from 'uuid' -import { Webview, window } from 'vscode' -import { Disposable, LanguageClient, Position, State, TextDocumentIdentifier } from 'vscode-languageclient' -import * as jose from 'jose' -import { encryptionKey } from '../lsp/auth' -import { Commands } from 'aws-core-vscode/shared' - -export function handle(client: LanguageClient, webview: Webview) { - // Listen for Initialize handshake from LSP server to register quick actions dynamically - client.onDidChangeState(({ oldState, newState }) => { - if (oldState === State.Starting && newState === State.Running) { - client.info( - 'Language client received initializeResult from server:', - JSON.stringify(client.initializeResult) - ) - - const chatOptions = client.initializeResult?.awsServerCapabilities?.chatOptions - - void webview.postMessage({ - command: CHAT_OPTIONS, - params: chatOptions, - }) - } - }) - - client.onTelemetry((e) => { - client.info(`[VSCode Client] Received telemetry event from server ${JSON.stringify(e)}`) - }) - - webview.onDidReceiveMessage(async (message) => { - client.info(`[VSCode Client] Received ${JSON.stringify(message)} from chat`) - - switch (message.command) { - case COPY_TO_CLIPBOARD: - client.info('[VSCode Client] Copy to clipboard event received') - break - case INSERT_TO_CURSOR_POSITION: { - const editor = window.activeTextEditor - let textDocument: TextDocumentIdentifier | undefined = undefined - let cursorPosition: Position | undefined = undefined - if (editor) { - cursorPosition = editor.selection.active - textDocument = { uri: editor.document.uri.toString() } - } - - client.sendNotification(insertToCursorPositionNotificationType.method, { - ...message.params, - cursorPosition, - textDocument, - }) - break - } - case AUTH_FOLLOW_UP_CLICKED: - client.info('[VSCode Client] AuthFollowUp clicked') - break - case chatRequestType.method: { - const partialResultToken = uuidv4() - const chatDisposable = client.onProgress(chatRequestType, partialResultToken, (partialResult) => - handlePartialResult(partialResult, encryptionKey, message.params.tabId, webview) - ) - - const editor = - window.activeTextEditor || - window.visibleTextEditors.find((editor) => editor.document.languageId !== 'Log') - if (editor) { - message.params.cursorPosition = [editor.selection.active] - message.params.textDocument = { uri: editor.document.uri.toString() } - } - - const chatRequest = await encryptRequest(message.params, encryptionKey) - const chatResult = (await client.sendRequest(chatRequestType.method, { - ...chatRequest, - partialResultToken, - })) as string | ChatResult - void handleCompleteResult( - chatResult, - encryptionKey, - message.params.tabId, - chatDisposable, - webview - ) - break - } - case quickActionRequestType.method: { - const quickActionPartialResultToken = uuidv4() - const quickActionDisposable = client.onProgress( - quickActionRequestType, - quickActionPartialResultToken, - (partialResult) => - handlePartialResult( - partialResult, - encryptionKey, - message.params.tabId, - webview - ) - ) - - const quickActionRequest = await encryptRequest(message.params, encryptionKey) - const quickActionResult = (await client.sendRequest(quickActionRequestType.method, { - ...quickActionRequest, - partialResultToken: quickActionPartialResultToken, - })) as string | ChatResult - void handleCompleteResult( - quickActionResult, - encryptionKey, - message.params.tabId, - quickActionDisposable, - webview - ) - break - } - case followUpClickNotificationType.method: - if (!isValidAuthFollowUpType(message.params.followUp.type)) { - client.sendNotification(followUpClickNotificationType.method, message.params) - } - break - default: - if (isServerEvent(message.command)) { - client.sendNotification(message.command, message.params) - } - break - } - }, undefined) - - registerGenericCommand('aws.amazonq.explainCode', 'Explain', webview) - registerGenericCommand('aws.amazonq.refactorCode', 'Refactor', webview) - registerGenericCommand('aws.amazonq.fixCode', 'Fix', webview) - registerGenericCommand('aws.amazonq.optimizeCode', 'Optimize', webview) - - Commands.register('aws.amazonq.sendToPrompt', (data) => { - const triggerType = getCommandTriggerType(data) - const selection = getSelectedText() - - void webview.postMessage({ - command: 'sendToPrompt', - params: { selection: selection, triggerType }, - }) - }) -} - -function getSelectedText(): string { - const editor = window.activeTextEditor - if (editor) { - const selection = editor.selection - const selectedText = editor.document.getText(selection) - return selectedText - } - - return ' ' -} - -function getCommandTriggerType(data: any): string { - // data is undefined when commands triggered from keybinding or command palette. Currently no - // way to differentiate keybinding and command palette, so both interactions are recorded as keybinding - return data === undefined ? 'hotkeys' : 'contextMenu' -} - -function registerGenericCommand(commandName: string, genericCommand: string, webview?: Webview) { - Commands.register(commandName, (data) => { - const triggerType = getCommandTriggerType(data) - const selection = getSelectedText() - - void webview?.postMessage({ - command: 'genericCommand', - params: { genericCommand, selection, triggerType }, - }) - }) -} - -function isServerEvent(command: string) { - return command.startsWith('aws/chat/') || command === 'telemetry/event' -} - -// Encrypt the provided request if encryption key exists otherwise do nothing -async function encryptRequest(params: T, encryptionKey: Buffer | undefined): Promise<{ message: string } | T> { - if (!encryptionKey) { - return params - } - - const payload = new TextEncoder().encode(JSON.stringify(params)) - - const encryptedMessage = await new jose.CompactEncrypt(payload) - .setProtectedHeader({ alg: 'dir', enc: 'A256GCM' }) - .encrypt(encryptionKey) - - return { message: encryptedMessage } -} - -async function decodeRequest(request: string, key: Buffer): Promise { - const result = await jose.jwtDecrypt(request, key, { - clockTolerance: 60, // Allow up to 60 seconds to account for clock differences - contentEncryptionAlgorithms: ['A256GCM'], - keyManagementAlgorithms: ['dir'], - }) - - if (!result.payload) { - throw new Error('JWT payload not found') - } - return result.payload as T -} - -async function handlePartialResult( - partialResult: string | T, - encryptionKey: Buffer | undefined, - tabId: string, - webview: Webview -) { - const decryptedMessage = - typeof partialResult === 'string' && encryptionKey - ? await decodeRequest(partialResult, encryptionKey) - : (partialResult as T) - - if (decryptedMessage.body) { - void webview?.postMessage({ - command: chatRequestType.method, - params: decryptedMessage, - isPartialResult: true, - tabId: tabId, - }) - } -} - -async function handleCompleteResult( - result: string | T, - encryptionKey: Buffer | undefined, - tabId: string, - disposable: Disposable, - webview: Webview -) { - const decryptedMessage = - typeof result === 'string' && encryptionKey ? await decodeRequest(result, encryptionKey) : result - - void webview?.postMessage({ - command: chatRequestType.method, - params: decryptedMessage, - tabId: tabId, - }) - disposable.dispose() -} diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index bf29bc6dddd..cf07c20c131 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -12,7 +12,6 @@ import { registerInlineCompletion } from '../inline/completion' import { AmazonQLspAuth, notificationTypes, writeEncryptionInit } from './auth' import { AuthUtil } from 'aws-core-vscode/codewhisperer' import { ConnectionMetadata } from '@aws/language-server-runtimes/protocol' -import { registerChat } from '../chat/activation' const localize = nls.loadMessageBundle() @@ -87,7 +86,6 @@ export function startLanguageServer(extensionContext: vscode.ExtensionContext, s return client.onReady().then(async () => { await auth.init() registerInlineCompletion(client) - registerChat(client) // Request handler for when the server wants to know about the clients auth connnection client.onRequest(notificationTypes.getConnectionMetadata.method, () => { From 750145d034475589cb38129b09f9f72871210d30 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:49:50 -0500 Subject: [PATCH 06/22] ci(amazonq): fix linting issue on language server child process (#6332) ## Problem In a recent PR we disabled using child_process in favour of our internal one. This started flagging lsp/client.ts, since it wasn't skipped. ## Solution Skip eslint for child_process in this case, since the language server options expect the actual child process. The same is done for the workspace context language server: https://github.com/aws/aws-toolkit-vscode/blob/master/packages/core/src/amazonq/lsp/lspClient.ts#L13 --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index cf07c20c131..cb12e68d900 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -5,7 +5,7 @@ import vscode, { env, version } from 'vscode' import * as nls from 'vscode-nls' -import * as cp from 'child_process' +import * as cp from 'child_process' // eslint-disable-line no-restricted-imports -- language server options expect actual child process import * as crypto from 'crypto' import { LanguageClient, LanguageClientOptions, ServerOptions, TransportKind } from 'vscode-languageclient' import { registerInlineCompletion } from '../inline/completion' From f4f022fda6465ead2a87c16238b9416fc4429311 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:50:33 -0500 Subject: [PATCH 07/22] fix(amazonq): Language server overrides should reference assets directly (#6333) ## Problem The language server initialization logic incorrectly handles custom server paths. When users specify a custom language server location through an environment variable, the code still appends `servers/aws-lsp-codewhisperer.js` to the path, leading to incorrect server resolution. ## Solution When a custom language server path is provided via environment variable (indicated by `location === 'override'`), use the specified asset directory path directly instead of appending the default server path. This allows users to have full control over the language server location. --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/activation.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts index ceed3cef624..2a68a1f2511 100644 --- a/packages/amazonq/src/lsp/activation.ts +++ b/packages/amazonq/src/lsp/activation.ts @@ -12,7 +12,11 @@ import path from 'path' export async function activate(ctx: vscode.ExtensionContext): Promise { try { const installResult = await new AmazonQLSPResolver().resolve() - await startLanguageServer(ctx, path.join(installResult.assetDirectory, 'servers/aws-lsp-codewhisperer.js')) + const serverLocation = + installResult.location === 'override' + ? installResult.assetDirectory + : path.join(installResult.assetDirectory, 'servers/aws-lsp-codewhisperer.js') + await startLanguageServer(ctx, serverLocation) } catch (err) { const e = err as ToolkitError void vscode.window.showInformationMessage(`Unable to launch amazonq language server: ${e.message}`) From 7ba424d724056aac47c766715b3b4c0f4f372262 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:35:40 -0500 Subject: [PATCH 08/22] test(amazonq): Add e2e tests for lsp auto updating (#6326) ## Problem We don't have any e2e test for auto updating the language server ## Solution Add a test that verifies downloading, caching, and fallbacks work as expected --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/lspInstaller.ts | 2 +- .../amazonq/test/e2e/lsp/lspInstaller.test.ts | 122 ++++++++++++++++++ packages/core/src/shared/index.ts | 1 + .../src/shared/languageServer/lspResolver.ts | 19 ++- 4 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 packages/amazonq/test/e2e/lsp/lspInstaller.test.ts diff --git a/packages/amazonq/src/lsp/lspInstaller.ts b/packages/amazonq/src/lsp/lspInstaller.ts index 120cdebe39c..5148d895466 100644 --- a/packages/amazonq/src/lsp/lspInstaller.ts +++ b/packages/amazonq/src/lsp/lspInstaller.ts @@ -8,7 +8,7 @@ import { Range } from 'semver' import { ManifestResolver, LanguageServerResolver, LspResolver, LspResult } from 'aws-core-vscode/shared' const manifestURL = 'https://aws-toolkit-language-servers.amazonaws.com/codewhisperer/0/manifest.json' -const supportedLspServerVersions = '^2.3.1' +export const supportedLspServerVersions = '^2.3.0' export class AmazonQLSPResolver implements LspResolver { async resolve(): Promise { diff --git a/packages/amazonq/test/e2e/lsp/lspInstaller.test.ts b/packages/amazonq/test/e2e/lsp/lspInstaller.test.ts new file mode 100644 index 00000000000..0bf2edafa3f --- /dev/null +++ b/packages/amazonq/test/e2e/lsp/lspInstaller.test.ts @@ -0,0 +1,122 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import sinon from 'sinon' +import { AmazonQLSPResolver, supportedLspServerVersions } from '../../../src/lsp/lspInstaller' +import { + fs, + LanguageServerResolver, + makeTemporaryToolkitFolder, + ManifestResolver, + request, +} from 'aws-core-vscode/shared' +import * as semver from 'semver' + +function createVersion(version: string) { + return { + isDelisted: false, + serverVersion: version, + targets: [ + { + arch: process.arch, + platform: process.platform, + contents: [ + { + bytes: 0, + filename: 'servers.zip', + hashes: [], + url: 'http://fakeurl', + }, + ], + }, + ], + } +} + +describe('AmazonQLSPInstaller', () => { + let resolver: AmazonQLSPResolver + let sandbox: sinon.SinonSandbox + let tempDir: string + + beforeEach(async () => { + sandbox = sinon.createSandbox() + resolver = new AmazonQLSPResolver() + tempDir = await makeTemporaryToolkitFolder() + sandbox.stub(LanguageServerResolver.prototype, 'defaultDownloadFolder').returns(tempDir) + }) + + afterEach(async () => { + delete process.env.AWS_LANGUAGE_SERVER_OVERRIDE + sandbox.restore() + await fs.delete(tempDir, { + recursive: true, + }) + }) + + describe('resolve()', () => { + it('uses AWS_LANGUAGE_SERVER_OVERRIDE', async () => { + const overridePath = '/custom/path/to/lsp' + process.env.AWS_LANGUAGE_SERVER_OVERRIDE = overridePath + + const result = await resolver.resolve() + + assert.strictEqual(result.assetDirectory, overridePath) + assert.strictEqual(result.location, 'override') + assert.strictEqual(result.version, '0.0.0') + }) + + it('resolves', async () => { + // First try - should download the file + const download = await resolver.resolve() + + assert.ok(download.assetDirectory.startsWith(tempDir)) + assert.deepStrictEqual(download.location, 'remote') + assert.ok(semver.satisfies(download.version, supportedLspServerVersions)) + + // Second try - Should see the contents in the cache + const cache = await resolver.resolve() + + assert.ok(cache.assetDirectory.startsWith(tempDir)) + assert.deepStrictEqual(cache.location, 'cache') + assert.ok(semver.satisfies(cache.version, supportedLspServerVersions)) + + /** + * Always make sure the latest version is one patch higher. This stops a problem + * where the fallback can't be used because the latest compatible version + * is equal to the min version, so if the cache isn't valid, then there + * would be no fallback location + * + * Instead, increasing the latest compatible lsp version means we can just + * use the one we downloaded earlier in the test as the fallback + */ + const nextVer = semver.inc(cache.version, 'patch', true) + if (!nextVer) { + throw new Error('Could not increment version') + } + sandbox.stub(ManifestResolver.prototype, 'resolve').resolves({ + manifestSchemaVersion: '0.0.0', + artifactId: 'foo', + artifactDescription: 'foo', + isManifestDeprecated: false, + versions: [createVersion(nextVer), createVersion(cache.version)], + }) + + // fail the next http request for the language server + sandbox.stub(request, 'fetch').returns({ + response: Promise.resolve({ + ok: false, + }), + } as any) + + // Third try - Cache doesn't exist and we couldn't download from the internet, fallback to a local version + const fallback = await resolver.resolve() + + assert.ok(fallback.assetDirectory.startsWith(tempDir)) + assert.deepStrictEqual(fallback.location, 'fallback') + assert.ok(semver.satisfies(fallback.version, supportedLspServerVersions)) + }) + }) +}) diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index a77f45d9a91..d081bfc2bde 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -62,3 +62,4 @@ export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants' export * from './languageServer/manifestResolver' export * from './languageServer/lspResolver' export * from './languageServer/types' +export { default as request } from './request' diff --git a/packages/core/src/shared/languageServer/lspResolver.ts b/packages/core/src/shared/languageServer/lspResolver.ts index 636163327ab..7a6fc4102b0 100644 --- a/packages/core/src/shared/languageServer/lspResolver.ts +++ b/packages/core/src/shared/languageServer/lspResolver.ts @@ -77,9 +77,9 @@ export class LanguageServerResolver { throw new ToolkitError('Unable to find a compatible version of the Language Server') } - const version = path.basename(cacheDirectory) + const version = path.basename(fallbackDirectory) logger.info( - `Unable to install language server ${latestVersion.serverVersion}. Launching a previous version from ${fallbackDirectory}` + `Unable to install ${this.lsName} language server v${latestVersion.serverVersion}. Launching a previous version from ${fallbackDirectory}` ) result.location = 'fallback' @@ -107,6 +107,7 @@ export class LanguageServerResolver { .filter(([_, filetype]) => filetype === FileType.Directory) .map(([pathName, _]) => semver.parse(pathName)) .filter((ver): ver is semver.SemVer => ver !== null) + .map((x) => x.version) const expectedVersion = semver.parse(version) if (!expectedVersion) { @@ -115,7 +116,7 @@ export class LanguageServerResolver { const sortedCachedLspVersions = compatibleLspVersions .filter((v) => this.isValidCachedVersion(v, cachedVersions, expectedVersion)) - .sort((a, b) => semver.compare(a.serverVersion, b.serverVersion)) + .sort((a, b) => semver.compare(b.serverVersion, a.serverVersion)) const fallbackDir = ( await Promise.all(sortedCachedLspVersions.map((ver) => this.getValidLocalCacheDirectory(ver))) @@ -144,9 +145,9 @@ export class LanguageServerResolver { * A version is considered valid if it exists in the cache and is less than * or equal to the expected version. */ - private isValidCachedVersion(version: LspVersion, cachedVersions: semver.SemVer[], expectedVersion: semver.SemVer) { + private isValidCachedVersion(version: LspVersion, cachedVersions: string[], expectedVersion: semver.SemVer) { const serverVersion = semver.parse(version.serverVersion) as semver.SemVer - return cachedVersions.find((x) => x === serverVersion) && serverVersion <= expectedVersion + return cachedVersions.includes(serverVersion.version) && semver.lte(serverVersion, expectedVersion) } /** @@ -283,9 +284,7 @@ export class LanguageServerResolver { const latestCompatibleVersion = this.manifest.versions .filter((ver) => this.isCompatibleVersion(ver) && this.hasRequiredTargetContent(ver)) - .sort((a, b) => { - return a.serverVersion.localeCompare(b.serverVersion) - })[0] ?? undefined + .sort((a, b) => semver.compare(b.serverVersion, a.serverVersion))[0] ?? undefined if (latestCompatibleVersion === undefined) { // TODO fix these error range names @@ -340,12 +339,12 @@ export class LanguageServerResolver { return version.targets.find((x) => x.arch === arch && x.platform === platform) } - private defaultDownloadFolder() { + defaultDownloadFolder() { const applicationSupportFolder = getApplicationSupportFolder() return path.join(applicationSupportFolder, `aws/toolkits/language-servers/${this.lsName}`) } - getDownloadDirectory(version: string) { + private getDownloadDirectory(version: string) { const directory = this._defaultDownloadFolder ?? this.defaultDownloadFolder() return `${directory}/${version}` } From 206b8a5f29b135016df9bdb15f66f845385b43fe Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Wed, 15 Jan 2025 09:05:22 -0500 Subject: [PATCH 09/22] refactor(amazonq): Improve robustness of lsp installation process (#6364) ## Problem - resolvers should produce runable binaries with the correct permissions - nodepaths should correctly target the correct os - workspace context lsp should reference the correct location ## Solution - resolvers now produce runable binaries - nodepaths are now correct for the workspace context lsp on windows/mac/linux - lsp location is now correct for the workspace context lsp --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/activation.ts | 7 +---- packages/amazonq/src/lsp/client.ts | 7 +++-- packages/amazonq/src/lsp/lspInstaller.ts | 27 ++++++++++++++++--- packages/core/src/amazonq/lsp/lspClient.ts | 11 +++----- .../core/src/amazonq/lsp/lspController.ts | 2 +- .../src/amazonq/lsp/workspaceInstaller.ts | 23 +++++++++++++--- packages/core/src/shared/index.ts | 1 + .../core/src/shared/languageServer/types.ts | 10 ++++++- .../shared/languageServer/utils/platform.ts | 8 ++++++ 9 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 packages/core/src/shared/languageServer/utils/platform.ts diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts index 2a68a1f2511..bdf4a59ab1a 100644 --- a/packages/amazonq/src/lsp/activation.ts +++ b/packages/amazonq/src/lsp/activation.ts @@ -7,16 +7,11 @@ import vscode from 'vscode' import { startLanguageServer } from './client' import { AmazonQLSPResolver } from './lspInstaller' import { ToolkitError } from 'aws-core-vscode/shared' -import path from 'path' export async function activate(ctx: vscode.ExtensionContext): Promise { try { const installResult = await new AmazonQLSPResolver().resolve() - const serverLocation = - installResult.location === 'override' - ? installResult.assetDirectory - : path.join(installResult.assetDirectory, 'servers/aws-lsp-codewhisperer.js') - await startLanguageServer(ctx, serverLocation) + await startLanguageServer(ctx, installResult.resourcePaths) } catch (err) { const e = err as ToolkitError void vscode.window.showInformationMessage(`Unable to launch amazonq language server: ${e.message}`) diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index cb12e68d900..da09f54bb06 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -12,10 +12,11 @@ import { registerInlineCompletion } from '../inline/completion' import { AmazonQLspAuth, notificationTypes, writeEncryptionInit } from './auth' import { AuthUtil } from 'aws-core-vscode/codewhisperer' import { ConnectionMetadata } from '@aws/language-server-runtimes/protocol' +import { ResourcePaths } from 'aws-core-vscode/shared' const localize = nls.loadMessageBundle() -export function startLanguageServer(extensionContext: vscode.ExtensionContext, serverPath: string) { +export function startLanguageServer(extensionContext: vscode.ExtensionContext, resourcePaths: ResourcePaths) { const toDispose = extensionContext.subscriptions // The debug options for the server @@ -30,6 +31,8 @@ export function startLanguageServer(extensionContext: vscode.ExtensionContext, s ], } + const serverPath = resourcePaths.lsp + // If the extension is launch in debug mode the debug server options are use // Otherwise the run options are used let serverOptions: ServerOptions = { @@ -37,7 +40,7 @@ export function startLanguageServer(extensionContext: vscode.ExtensionContext, s debug: { module: serverPath, transport: TransportKind.ipc, options: debugOptions }, } - const child = cp.spawn('node', [serverPath, ...debugOptions.execArgv]) + const child = cp.spawn(resourcePaths.node, [serverPath, ...debugOptions.execArgv]) writeEncryptionInit(child.stdin) serverOptions = () => Promise.resolve(child) diff --git a/packages/amazonq/src/lsp/lspInstaller.ts b/packages/amazonq/src/lsp/lspInstaller.ts index 5148d895466..a087abf395c 100644 --- a/packages/amazonq/src/lsp/lspInstaller.ts +++ b/packages/amazonq/src/lsp/lspInstaller.ts @@ -5,13 +5,21 @@ import * as vscode from 'vscode' import { Range } from 'semver' -import { ManifestResolver, LanguageServerResolver, LspResolver, LspResult } from 'aws-core-vscode/shared' +import { + ManifestResolver, + LanguageServerResolver, + LspResolver, + fs, + LspResolution, + getNodeExecutableName, +} from 'aws-core-vscode/shared' +import path from 'path' const manifestURL = 'https://aws-toolkit-language-servers.amazonaws.com/codewhisperer/0/manifest.json' export const supportedLspServerVersions = '^2.3.0' export class AmazonQLSPResolver implements LspResolver { - async resolve(): Promise { + async resolve(): Promise { const overrideLocation = process.env.AWS_LANGUAGE_SERVER_OVERRIDE if (overrideLocation) { void vscode.window.showInformationMessage(`Using language server override location: ${overrideLocation}`) @@ -19,6 +27,10 @@ export class AmazonQLSPResolver implements LspResolver { assetDirectory: overrideLocation, location: 'override', version: '0.0.0', + resourcePaths: { + lsp: overrideLocation, + node: getNodeExecutableName(), + }, } } @@ -31,7 +43,16 @@ export class AmazonQLSPResolver implements LspResolver { new Range(supportedLspServerVersions) ).resolve() + const nodePath = path.join(installationResult.assetDirectory, `servers/${getNodeExecutableName()}`) + await fs.chmod(nodePath, 0o755) + // TODO Cleanup old versions of language servers - return installationResult + return { + ...installationResult, + resourcePaths: { + lsp: path.join(installationResult.assetDirectory, 'servers/aws-lsp-codewhisperer.js'), + node: nodePath, + }, + } } } diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts index b901dd20d67..f30aa5b681d 100644 --- a/packages/core/src/amazonq/lsp/lspClient.ts +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -32,7 +32,7 @@ import { } from './types' import { Writable } from 'stream' import { CodeWhispererSettings } from '../../codewhisperer/util/codewhispererSettings' -import { fs, getLogger, globals } from '../../shared' +import { ResourcePaths, fs, getLogger, globals } from '../../shared' const localize = nls.loadMessageBundle() @@ -172,7 +172,7 @@ export class LspClient { * It will create a output channel named Amazon Q Language Server. * This function assumes the LSP server has already been downloaded. */ -export async function activate(extensionContext: ExtensionContext, serverModule: string) { +export async function activate(extensionContext: ExtensionContext, resourcePaths: ResourcePaths) { LspClient.instance const toDispose = extensionContext.subscriptions @@ -195,12 +195,9 @@ export async function activate(extensionContext: ExtensionContext, serverModule: delete process.env.Q_WORKER_THREADS } - const nodename = process.platform === 'win32' ? 'node.exe' : 'node' + const serverModule = resourcePaths.lsp - const child = spawn(extensionContext.asAbsolutePath(path.join('resources', nodename)), [ - serverModule, - ...debugOptions.execArgv, - ]) + const child = spawn(resourcePaths.node, [serverModule, ...debugOptions.execArgv]) // share an encryption key using stdin // follow same practice of DEXP LSP server writeEncryptionInit(child.stdin) diff --git a/packages/core/src/amazonq/lsp/lspController.ts b/packages/core/src/amazonq/lsp/lspController.ts index bae7cfbd443..3b38d9a17a2 100644 --- a/packages/core/src/amazonq/lsp/lspController.ts +++ b/packages/core/src/amazonq/lsp/lspController.ts @@ -161,7 +161,7 @@ export class LspController { setImmediate(async () => { try { const installResult = await new WorkspaceLSPResolver().resolve() - await activateLsp(context, path.join(installResult.assetDirectory, 'resources/qserver/lspServer.js')) + await activateLsp(context, installResult.resourcePaths) getLogger().info('LspController: LSP activated') void LspController.instance.buildIndex(buildIndexConfig) // log the LSP server CPU and Memory usage per 30 minutes. diff --git a/packages/core/src/amazonq/lsp/workspaceInstaller.ts b/packages/core/src/amazonq/lsp/workspaceInstaller.ts index fda843fa063..f1aec614a4d 100644 --- a/packages/core/src/amazonq/lsp/workspaceInstaller.ts +++ b/packages/core/src/amazonq/lsp/workspaceInstaller.ts @@ -3,17 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { LspResolver, LspResult } from '../../shared/languageServer/types' +import path from 'path' +import { LspResolution, LspResolver } from '../../shared/languageServer/types' import { ManifestResolver } from '../../shared/languageServer/manifestResolver' import { LanguageServerResolver } from '../../shared/languageServer/lspResolver' import { Range } from 'semver' +import { getNodeExecutableName } from '../../shared/languageServer/utils/platform' +import { fs } from '../../shared/fs/fs' const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json' // this LSP client in Q extension is only going to work with these LSP server versions const supportedLspServerVersions = '0.1.32' export class WorkspaceLSPResolver implements LspResolver { - async resolve(): Promise { + async resolve(): Promise { const name = 'AmazonQ-Workspace' const manifest = await new ManifestResolver(manifestUrl, name).resolve() const installationResult = await new LanguageServerResolver( @@ -22,7 +25,21 @@ export class WorkspaceLSPResolver implements LspResolver { new Range(supportedLspServerVersions) ).resolve() + const nodeName = + process.platform === 'win32' ? getNodeExecutableName() : `node-${process.platform}-${process.arch}` + const nodePath = path.join(installationResult.assetDirectory, nodeName) + await fs.chmod(nodePath, 0o755) + // TODO Cleanup old versions of language servers - return installationResult + return { + ...installationResult, + resourcePaths: { + lsp: path.join( + installationResult.assetDirectory, + `qserver-${process.platform}-${process.arch}/qserver/lspServer.js` + ), + node: nodePath, + }, + } } } diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index d081bfc2bde..01f398e058c 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -63,3 +63,4 @@ export * from './languageServer/manifestResolver' export * from './languageServer/lspResolver' export * from './languageServer/types' export { default as request } from './request' +export * from './languageServer/utils/platform' diff --git a/packages/core/src/shared/languageServer/types.ts b/packages/core/src/shared/languageServer/types.ts index ec365424eaa..4f5a3cf1c87 100644 --- a/packages/core/src/shared/languageServer/types.ts +++ b/packages/core/src/shared/languageServer/types.ts @@ -15,8 +15,16 @@ export interface LspResult { assetDirectory: string } +export interface ResourcePaths { + lsp: string + node: string +} +export interface LspResolution extends LspResult { + resourcePaths: ResourcePaths +} + export interface LspResolver { - resolve(): Promise + resolve(): Promise } export interface TargetContent { diff --git a/packages/core/src/shared/languageServer/utils/platform.ts b/packages/core/src/shared/languageServer/utils/platform.ts new file mode 100644 index 00000000000..f313b66186f --- /dev/null +++ b/packages/core/src/shared/languageServer/utils/platform.ts @@ -0,0 +1,8 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +export function getNodeExecutableName(): string { + return process.platform === 'win32' ? 'node.exe' : 'node' +} From c47782a112b618b41b00d66cfe83d7a470ca747b Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Wed, 15 Jan 2025 09:47:05 -0500 Subject: [PATCH 10/22] refactor(core): Move shared/languageServer -> shared/lsp --- packages/core/src/amazonq/lsp/workspaceInstaller.ts | 8 ++++---- packages/core/src/shared/index.ts | 8 ++++---- .../shared/{languageServer => lsp}/languageModelCache.ts | 0 .../src/shared/{languageServer => lsp}/lspResolver.ts | 0 .../shared/{languageServer => lsp}/manifestResolver.ts | 0 packages/core/src/shared/{languageServer => lsp}/types.ts | 0 .../src/shared/{languageServer => lsp}/utils/platform.ts | 0 .../src/shared/{languageServer => lsp}/utils/runner.ts | 0 packages/core/src/stepFunctions/asl/aslServer.ts | 4 ++-- 9 files changed, 10 insertions(+), 10 deletions(-) rename packages/core/src/shared/{languageServer => lsp}/languageModelCache.ts (100%) rename packages/core/src/shared/{languageServer => lsp}/lspResolver.ts (100%) rename packages/core/src/shared/{languageServer => lsp}/manifestResolver.ts (100%) rename packages/core/src/shared/{languageServer => lsp}/types.ts (100%) rename packages/core/src/shared/{languageServer => lsp}/utils/platform.ts (100%) rename packages/core/src/shared/{languageServer => lsp}/utils/runner.ts (100%) diff --git a/packages/core/src/amazonq/lsp/workspaceInstaller.ts b/packages/core/src/amazonq/lsp/workspaceInstaller.ts index f1aec614a4d..6c5d869a70a 100644 --- a/packages/core/src/amazonq/lsp/workspaceInstaller.ts +++ b/packages/core/src/amazonq/lsp/workspaceInstaller.ts @@ -4,11 +4,11 @@ */ import path from 'path' -import { LspResolution, LspResolver } from '../../shared/languageServer/types' -import { ManifestResolver } from '../../shared/languageServer/manifestResolver' -import { LanguageServerResolver } from '../../shared/languageServer/lspResolver' +import { LspResolution, LspResolver } from '../../shared/lsp/types' +import { ManifestResolver } from '../../shared/lsp/manifestResolver' +import { LanguageServerResolver } from '../../shared/lsp/lspResolver' import { Range } from 'semver' -import { getNodeExecutableName } from '../../shared/languageServer/utils/platform' +import { getNodeExecutableName } from '../../shared/lsp/utils/platform' import { fs } from '../../shared/fs/fs' const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json' diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index 01f398e058c..206836ed45f 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -59,8 +59,8 @@ export { i18n } from './i18n-helper' export * from './icons' export * as textDocumentUtil from './utilities/textDocumentUtilities' export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants' -export * from './languageServer/manifestResolver' -export * from './languageServer/lspResolver' -export * from './languageServer/types' +export * from './lsp/manifestResolver' +export * from './lsp/lspResolver' +export * from './lsp/types' export { default as request } from './request' -export * from './languageServer/utils/platform' +export * from './lsp/utils/platform' diff --git a/packages/core/src/shared/languageServer/languageModelCache.ts b/packages/core/src/shared/lsp/languageModelCache.ts similarity index 100% rename from packages/core/src/shared/languageServer/languageModelCache.ts rename to packages/core/src/shared/lsp/languageModelCache.ts diff --git a/packages/core/src/shared/languageServer/lspResolver.ts b/packages/core/src/shared/lsp/lspResolver.ts similarity index 100% rename from packages/core/src/shared/languageServer/lspResolver.ts rename to packages/core/src/shared/lsp/lspResolver.ts diff --git a/packages/core/src/shared/languageServer/manifestResolver.ts b/packages/core/src/shared/lsp/manifestResolver.ts similarity index 100% rename from packages/core/src/shared/languageServer/manifestResolver.ts rename to packages/core/src/shared/lsp/manifestResolver.ts diff --git a/packages/core/src/shared/languageServer/types.ts b/packages/core/src/shared/lsp/types.ts similarity index 100% rename from packages/core/src/shared/languageServer/types.ts rename to packages/core/src/shared/lsp/types.ts diff --git a/packages/core/src/shared/languageServer/utils/platform.ts b/packages/core/src/shared/lsp/utils/platform.ts similarity index 100% rename from packages/core/src/shared/languageServer/utils/platform.ts rename to packages/core/src/shared/lsp/utils/platform.ts diff --git a/packages/core/src/shared/languageServer/utils/runner.ts b/packages/core/src/shared/lsp/utils/runner.ts similarity index 100% rename from packages/core/src/shared/languageServer/utils/runner.ts rename to packages/core/src/shared/lsp/utils/runner.ts diff --git a/packages/core/src/stepFunctions/asl/aslServer.ts b/packages/core/src/stepFunctions/asl/aslServer.ts index 8da1363969c..2d4c4fadde3 100644 --- a/packages/core/src/stepFunctions/asl/aslServer.ts +++ b/packages/core/src/stepFunctions/asl/aslServer.ts @@ -37,8 +37,8 @@ import { import { posix } from 'path' import * as URL from 'url' -import { getLanguageModelCache } from '../../shared/languageServer/languageModelCache' -import { formatError, runSafe, runSafeAsync } from '../../shared/languageServer/utils/runner' +import { getLanguageModelCache } from '../../shared/lsp/languageModelCache' +import { formatError, runSafe, runSafeAsync } from '../../shared/lsp/utils/runner' import { YAML_ASL, JSON_ASL } from '../constants/aslFormats' export const ResultLimitReached: NotificationType = new NotificationType('asl/resultLimitReached') From 45a1ab9cf194724547fe382ffccd862acd188e4e Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Fri, 17 Jan 2025 13:39:50 -0500 Subject: [PATCH 11/22] refactor(lsp): Use internal ChildProcess module for launching language servers (#6374) ## Problem - we want to use internal ChildProcess module over node's child process module - some code for spawning the language server is duplicated between the workspace server and the amazon q server ## Solution - use internal ChildProcess module and de-dup the common language server spawning code --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/client.ts | 23 +++++----- packages/core/src/amazonq/lsp/lspClient.ts | 29 +++--------- packages/core/src/shared/index.ts | 1 + .../core/src/shared/lsp/utils/platform.ts | 45 +++++++++++++++++++ .../core/src/shared/utilities/processUtils.ts | 4 ++ 5 files changed, 69 insertions(+), 33 deletions(-) diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index da09f54bb06..4595b137636 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -5,18 +5,17 @@ import vscode, { env, version } from 'vscode' import * as nls from 'vscode-nls' -import * as cp from 'child_process' // eslint-disable-line no-restricted-imports -- language server options expect actual child process import * as crypto from 'crypto' import { LanguageClient, LanguageClientOptions, ServerOptions, TransportKind } from 'vscode-languageclient' import { registerInlineCompletion } from '../inline/completion' -import { AmazonQLspAuth, notificationTypes, writeEncryptionInit } from './auth' +import { AmazonQLspAuth, encryptionKey, notificationTypes } from './auth' import { AuthUtil } from 'aws-core-vscode/codewhisperer' import { ConnectionMetadata } from '@aws/language-server-runtimes/protocol' -import { ResourcePaths } from 'aws-core-vscode/shared' +import { ResourcePaths, createServerOptions } from 'aws-core-vscode/shared' const localize = nls.loadMessageBundle() -export function startLanguageServer(extensionContext: vscode.ExtensionContext, resourcePaths: ResourcePaths) { +export async function startLanguageServer(extensionContext: vscode.ExtensionContext, resourcePaths: ResourcePaths) { const toDispose = extensionContext.subscriptions // The debug options for the server @@ -31,19 +30,21 @@ export function startLanguageServer(extensionContext: vscode.ExtensionContext, r ], } - const serverPath = resourcePaths.lsp + const serverModule = resourcePaths.lsp // If the extension is launch in debug mode the debug server options are use // Otherwise the run options are used let serverOptions: ServerOptions = { - run: { module: serverPath, transport: TransportKind.ipc }, - debug: { module: serverPath, transport: TransportKind.ipc, options: debugOptions }, + run: { module: serverModule, transport: TransportKind.ipc }, + debug: { module: serverModule, transport: TransportKind.ipc, options: debugOptions }, } - const child = cp.spawn(resourcePaths.node, [serverPath, ...debugOptions.execArgv]) - writeEncryptionInit(child.stdin) - - serverOptions = () => Promise.resolve(child) + serverOptions = createServerOptions({ + encryptionKey, + executable: resourcePaths.node, + serverModule, + execArgv: debugOptions.execArgv, + }) const documentSelector = [{ scheme: 'file', language: '*' }] diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts index f30aa5b681d..659df3ae078 100644 --- a/packages/core/src/amazonq/lsp/lspClient.ts +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -10,7 +10,6 @@ import * as vscode from 'vscode' import * as path from 'path' import * as nls from 'vscode-nls' -import { spawn } from 'child_process' // eslint-disable-line no-restricted-imports import * as crypto from 'crypto' import * as jose from 'jose' @@ -30,27 +29,13 @@ import { GetRepomapIndexJSONRequestType, Usage, } from './types' -import { Writable } from 'stream' import { CodeWhispererSettings } from '../../codewhisperer/util/codewhispererSettings' -import { ResourcePaths, fs, getLogger, globals } from '../../shared' +import { ResourcePaths, createServerOptions, fs, getLogger, globals } from '../../shared' const localize = nls.loadMessageBundle() const key = crypto.randomBytes(32) -/** - * Sends a json payload to the language server, who is waiting to know what the encryption key is. - * Code reference: https://github.com/aws/language-servers/blob/7da212185a5da75a72ce49a1a7982983f438651a/client/vscode/src/credentialsActivation.ts#L77 - */ -export function writeEncryptionInit(stream: Writable): void { - const request = { - version: '1.0', - mode: 'JWT', - key: key.toString('base64'), - } - stream.write(JSON.stringify(request)) - stream.write('\n') -} /** * LspClient manages the API call between VS Code extension and LSP server * It encryptes the payload of API call. @@ -197,11 +182,6 @@ export async function activate(extensionContext: ExtensionContext, resourcePaths const serverModule = resourcePaths.lsp - const child = spawn(resourcePaths.node, [serverModule, ...debugOptions.execArgv]) - // share an encryption key using stdin - // follow same practice of DEXP LSP server - writeEncryptionInit(child.stdin) - // If the extension is launch in debug mode the debug server options are use // Otherwise the run options are used let serverOptions: ServerOptions = { @@ -209,7 +189,12 @@ export async function activate(extensionContext: ExtensionContext, resourcePaths debug: { module: serverModule, transport: TransportKind.ipc, options: debugOptions }, } - serverOptions = () => Promise.resolve(child!) + serverOptions = createServerOptions({ + encryptionKey: key, + executable: resourcePaths.node, + serverModule, + execArgv: debugOptions.execArgv, + }) const documentSelector = [{ scheme: 'file', language: '*' }] diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index 206836ed45f..ed938d20b17 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -64,3 +64,4 @@ export * from './lsp/lspResolver' export * from './lsp/types' export { default as request } from './request' export * from './lsp/utils/platform' +export * as processUtils from './utilities/processUtils' diff --git a/packages/core/src/shared/lsp/utils/platform.ts b/packages/core/src/shared/lsp/utils/platform.ts index f313b66186f..5c2d7a59123 100644 --- a/packages/core/src/shared/lsp/utils/platform.ts +++ b/packages/core/src/shared/lsp/utils/platform.ts @@ -3,6 +3,51 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { ToolkitError } from '../../errors' +import { ChildProcess } from '../../utilities/processUtils' + export function getNodeExecutableName(): string { return process.platform === 'win32' ? 'node.exe' : 'node' } + +/** + * Get a json payload that will be sent to the language server, who is waiting to know what the encryption key is. + * Code reference: https://github.com/aws/language-servers/blob/7da212185a5da75a72ce49a1a7982983f438651a/client/vscode/src/credentialsActivation.ts#L77 + */ +function getEncryptionInit(key: Buffer): string { + const request = { + version: '1.0', + mode: 'JWT', + key: key.toString('base64'), + } + return JSON.stringify(request) + '\n' +} + +export function createServerOptions({ + encryptionKey, + executable, + serverModule, + execArgv, +}: { + encryptionKey: Buffer + executable: string + serverModule: string + execArgv: string[] +}) { + return async () => { + const lspProcess = new ChildProcess(executable, [serverModule, ...execArgv]) + + // this is a long running process, awaiting it will never resolve + void lspProcess.run() + + // share an encryption key using stdin + // follow same practice of DEXP LSP server + await lspProcess.send(getEncryptionInit(encryptionKey)) + + const proc = lspProcess.proc() + if (!proc) { + throw new ToolkitError('Language Server process was not started') + } + return proc + } +} diff --git a/packages/core/src/shared/utilities/processUtils.ts b/packages/core/src/shared/utilities/processUtils.ts index 3fcd7864438..9c514363033 100644 --- a/packages/core/src/shared/utilities/processUtils.ts +++ b/packages/core/src/shared/utilities/processUtils.ts @@ -389,6 +389,10 @@ export class ChildProcess { return this.#processResult } + public proc(): proc.ChildProcess | undefined { + return this.#childProcess + } + public pid(): number { return this.#childProcess?.pid ?? -1 } From ecc5c6ae8ab99d0985aa4eb0c4da98f342561f46 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Wed, 22 Jan 2025 07:34:17 -0500 Subject: [PATCH 12/22] refactor(lint): migrate use of .forEach to for...of #6406 ## Problem The lint task is failing on the feature branch due to a leftover usage of `forEach`. ## Solution - switch it to `for...of`. --- packages/amazonq/src/inline/completion.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/amazonq/src/inline/completion.ts b/packages/amazonq/src/inline/completion.ts index c4ff07b9648..b20690afbd4 100644 --- a/packages/amazonq/src/inline/completion.ts +++ b/packages/amazonq/src/inline/completion.ts @@ -16,7 +16,6 @@ import { } from 'vscode' import { LanguageClient } from 'vscode-languageclient' import { - InlineCompletionItemWithReferences, InlineCompletionListWithReferences, InlineCompletionWithReferencesParams, inlineCompletionWithReferencesRequestType, @@ -104,13 +103,13 @@ export class AmazonQInlineCompletionItemProvider implements InlineCompletionItem const firstCompletionDisplayLatency = Date.now() - requestStartTime // Add completion session tracking and attach onAcceptance command to each item to record used decision - list.items.forEach((item: InlineCompletionItemWithReferences) => { + for (const item of list.items) { item.command = { command: 'aws.sample-vscode-ext-amazonq.accept', title: 'On acceptance', arguments: [list.sessionId, item.itemId, requestStartTime, firstCompletionDisplayLatency], } - }) + } return list as InlineCompletionList } From ef7cd4e1290fc18fcaf778bba91c233a44622976 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:49:33 -0500 Subject: [PATCH 13/22] refactor(fetcher): migrate withRetries to waitUntil (#6429) ## Problem Recently a [PR](https://github.com/aws/aws-toolkit-vscode/pull/6387/files#diff-c0100187095b7c009b33063c21a517248290b37d02ef10e9370d28cd3ed5d1ea) on master combined `waitUntil` with `withRetries`. Once master merged into feature branch, it caused this case to start erroring. Leading to the following: ``` npm error src/shared/resourcefetcher/httpResourceFetcher.ts(164,16): error TS2304: Cannot find name 'withRetries'. ``` ## Solution - migrate to `waitUntil` in the same fashion as the PR. --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- .../src/shared/resourcefetcher/httpResourceFetcher.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts b/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts index ae1cb76fb77..4a61ae47854 100644 --- a/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts +++ b/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts @@ -135,7 +135,7 @@ export class HttpResourceFetcher implements ResourceFetcher { } export class RetryableResourceFetcher extends HttpResourceFetcher { - private readonly retryNumber: number + private readonly requestTimeoutMs: number private readonly retryIntervalMs: number private readonly resource: string @@ -155,13 +155,13 @@ export class RetryableResourceFetcher extends HttpResourceFetcher { showUrl, timeout, }) - this.retryNumber = retryNumber + this.requestTimeoutMs = retryNumber * retryIntervalMs this.retryIntervalMs = retryIntervalMs this.resource = resource } fetch(versionTag?: string) { - return withRetries( + return waitUntil( async () => { try { return await this.getNewETagContent(versionTag) @@ -171,8 +171,9 @@ export class RetryableResourceFetcher extends HttpResourceFetcher { } }, { - maxRetries: this.retryNumber, - delay: this.retryIntervalMs, + timeout: this.requestTimeoutMs, + interval: this.retryIntervalMs, + retryOnFail: true, } ) } From 41e35ff7332750d5ce8d0fa6c3227be3f32327a1 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:40:55 -0500 Subject: [PATCH 14/22] refactor(core): Manifest/LSP fetching should use httpResourceFetcher (#6411) ## Problem Now that we've refactored the httpResourceFetcher a little bit we can use that for fetching both the manifest and the language server ## Solution --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/core/src/shared/lsp/lspResolver.ts | 7 ++- .../core/src/shared/lsp/manifestResolver.ts | 13 +++--- .../resourcefetcher/httpResourceFetcher.ts | 45 ------------------- packages/webpack.web.config.js | 2 +- 4 files changed, 9 insertions(+), 58 deletions(-) diff --git a/packages/core/src/shared/lsp/lspResolver.ts b/packages/core/src/shared/lsp/lspResolver.ts index 7a6fc4102b0..4959f675986 100644 --- a/packages/core/src/shared/lsp/lspResolver.ts +++ b/packages/core/src/shared/lsp/lspResolver.ts @@ -12,7 +12,7 @@ import AdmZip from 'adm-zip' import { TargetContent, logger, LspResult, LspVersion, Manifest } from './types' import { getApplicationSupportFolder } from '../vscode/env' import { createHash } from '../crypto' -import request from '../request' +import { HttpResourceFetcher } from '../resourcefetcher/httpResourceFetcher' export class LanguageServerResolver { constructor( @@ -165,9 +165,8 @@ export class LanguageServerResolver { } const downloadTasks = contents.map(async (content) => { - // TODO This should be using the retryable http library but it doesn't seem to support zips right now - const res = await request.fetch('GET', content.url).response - if (!res.ok || !res.body) { + const res = await new HttpResourceFetcher(content.url, { showUrl: true }).get() + if (!res || !res.ok || !res.body) { return false } diff --git a/packages/core/src/shared/lsp/manifestResolver.ts b/packages/core/src/shared/lsp/manifestResolver.ts index 36685890620..0cf27b1293b 100644 --- a/packages/core/src/shared/lsp/manifestResolver.ts +++ b/packages/core/src/shared/lsp/manifestResolver.ts @@ -5,10 +5,10 @@ import { getLogger } from '../logger/logger' import { ToolkitError } from '../errors' -import { RetryableResourceFetcher } from '../resourcefetcher/httpResourceFetcher' import { Timeout } from '../utilities/timeoutUtils' import globals from '../extensionGlobals' import { Manifest } from './types' +import { HttpResourceFetcher } from '../resourcefetcher/httpResourceFetcher' const logger = getLogger('lsp') @@ -40,14 +40,11 @@ export class ManifestResolver { } private async fetchRemoteManifest(): Promise { - const resourceFetcher = new RetryableResourceFetcher({ - resource: this.manifestURL, - params: { - timeout: new Timeout(manifestTimeoutMs), - }, - }) + const resp = await new HttpResourceFetcher(this.manifestURL, { + showUrl: true, + timeout: new Timeout(manifestTimeoutMs), + }).getNewETagContent(this.getEtag()) - const resp = await resourceFetcher.getNewETagContent(this.getEtag()) if (!resp.content) { throw new ToolkitError('New content was not downloaded; fallback to the locally stored manifest') } diff --git a/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts b/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts index 4a61ae47854..e85e1ded70b 100644 --- a/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts +++ b/packages/core/src/shared/resourcefetcher/httpResourceFetcher.ts @@ -134,51 +134,6 @@ export class HttpResourceFetcher implements ResourceFetcher { } } -export class RetryableResourceFetcher extends HttpResourceFetcher { - private readonly requestTimeoutMs: number - private readonly retryIntervalMs: number - private readonly resource: string - - constructor({ - resource, - params: { retryNumber = 5, retryIntervalMs = 3000, showUrl = true, timeout = new Timeout(5000) }, - }: { - resource: string - params: { - retryNumber?: number - retryIntervalMs?: number - showUrl?: boolean - timeout?: Timeout - } - }) { - super(resource, { - showUrl, - timeout, - }) - this.requestTimeoutMs = retryNumber * retryIntervalMs - this.retryIntervalMs = retryIntervalMs - this.resource = resource - } - - fetch(versionTag?: string) { - return waitUntil( - async () => { - try { - return await this.getNewETagContent(versionTag) - } catch (err) { - getLogger('lsp').error('Failed to fetch at endpoint: %s, err: %s', this.resource, err) - throw err - } - }, - { - timeout: this.requestTimeoutMs, - interval: this.retryIntervalMs, - retryOnFail: true, - } - ) - } -} - /** * Retrieves JSON property value from a remote resource * @param property property to retrieve diff --git a/packages/webpack.web.config.js b/packages/webpack.web.config.js index fc6ca86d1d9..0f82af67389 100644 --- a/packages/webpack.web.config.js +++ b/packages/webpack.web.config.js @@ -41,7 +41,7 @@ module.exports = (env, argv) => { * environments. The following allows compilation to pass in Web mode by never bundling the module in the final output for web mode. */ new webpack.IgnorePlugin({ - resourceRegExp: /httpResourceFetcher/, // matches the path in the require() statement + resourceRegExp: /node\/httpResourceFetcher/, // matches the path in the require() statement }), /** * HACK: the ps-list module breaks Web mode if imported, BUT we still dynamically import this module for non web mode From 7488e0ae602e74b03b7e83ccad27c0899d32f9d2 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Tue, 28 Jan 2025 08:23:45 -0500 Subject: [PATCH 15/22] feat(lsp): older and delisted versions of lsp are automatically removed (#6409) ## Problem - Installed LSP artifacts remain forever. There is no cleanup effort made. ## Solution Implement the following heuristic: - Delete all currently installed delisted versions. - Delete all versions that remain, except the most recent 2. Included in this change is a new utility function `partition`. `partition` is like `filter`, but it produces both the positive and negative result in two separate sublists. See the tests for a simple example. --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/lspInstaller.ts | 3 +- .../src/amazonq/lsp/workspaceInstaller.ts | 3 +- packages/core/src/shared/index.ts | 1 + packages/core/src/shared/lsp/lspResolver.ts | 8 +- packages/core/src/shared/lsp/utils/cleanup.ts | 41 ++++++++ packages/core/src/shared/utilities/tsUtils.ts | 15 +++ .../src/test/shared/lsp/utils/cleanup.test.ts | 93 +++++++++++++++++++ .../src/test/shared/utilities/tsUtils.test.ts | 16 ++++ 8 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 packages/core/src/shared/lsp/utils/cleanup.ts create mode 100644 packages/core/src/test/shared/lsp/utils/cleanup.test.ts create mode 100644 packages/core/src/test/shared/utilities/tsUtils.test.ts diff --git a/packages/amazonq/src/lsp/lspInstaller.ts b/packages/amazonq/src/lsp/lspInstaller.ts index a087abf395c..72d0746cdcf 100644 --- a/packages/amazonq/src/lsp/lspInstaller.ts +++ b/packages/amazonq/src/lsp/lspInstaller.ts @@ -12,6 +12,7 @@ import { fs, LspResolution, getNodeExecutableName, + cleanLspDownloads, } from 'aws-core-vscode/shared' import path from 'path' @@ -46,7 +47,7 @@ export class AmazonQLSPResolver implements LspResolver { const nodePath = path.join(installationResult.assetDirectory, `servers/${getNodeExecutableName()}`) await fs.chmod(nodePath, 0o755) - // TODO Cleanup old versions of language servers + await cleanLspDownloads(manifest.versions, path.dirname(installationResult.assetDirectory)) return { ...installationResult, resourcePaths: { diff --git a/packages/core/src/amazonq/lsp/workspaceInstaller.ts b/packages/core/src/amazonq/lsp/workspaceInstaller.ts index 6c5d869a70a..c4c688d7bc1 100644 --- a/packages/core/src/amazonq/lsp/workspaceInstaller.ts +++ b/packages/core/src/amazonq/lsp/workspaceInstaller.ts @@ -10,6 +10,7 @@ import { LanguageServerResolver } from '../../shared/lsp/lspResolver' import { Range } from 'semver' import { getNodeExecutableName } from '../../shared/lsp/utils/platform' import { fs } from '../../shared/fs/fs' +import { cleanLspDownloads } from '../../shared' const manifestUrl = 'https://aws-toolkit-language-servers.amazonaws.com/q-context/manifest.json' // this LSP client in Q extension is only going to work with these LSP server versions @@ -30,7 +31,7 @@ export class WorkspaceLSPResolver implements LspResolver { const nodePath = path.join(installationResult.assetDirectory, nodeName) await fs.chmod(nodePath, 0o755) - // TODO Cleanup old versions of language servers + await cleanLspDownloads(manifest.versions, path.basename(installationResult.assetDirectory)) return { ...installationResult, resourcePaths: { diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index 7cdf3ad12f0..236badac4be 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -63,6 +63,7 @@ export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants' export * from './lsp/manifestResolver' export * from './lsp/lspResolver' export * from './lsp/types' +export * from './lsp/utils/cleanup' export { default as request } from './request' export * from './lsp/utils/platform' export * as processUtils from './utilities/processUtils' diff --git a/packages/core/src/shared/lsp/lspResolver.ts b/packages/core/src/shared/lsp/lspResolver.ts index 4959f675986..5a907b96a02 100644 --- a/packages/core/src/shared/lsp/lspResolver.ts +++ b/packages/core/src/shared/lsp/lspResolver.ts @@ -338,9 +338,13 @@ export class LanguageServerResolver { return version.targets.find((x) => x.arch === arch && x.platform === platform) } + // lazy calls to `getApplicationSupportFolder()` to avoid failure on windows. + public static get defaultDir() { + return path.join(getApplicationSupportFolder(), `aws/toolkits/language-servers`) + } + defaultDownloadFolder() { - const applicationSupportFolder = getApplicationSupportFolder() - return path.join(applicationSupportFolder, `aws/toolkits/language-servers/${this.lsName}`) + return path.join(LanguageServerResolver.defaultDir, `${this.lsName}`) } private getDownloadDirectory(version: string) { diff --git a/packages/core/src/shared/lsp/utils/cleanup.ts b/packages/core/src/shared/lsp/utils/cleanup.ts new file mode 100644 index 00000000000..874f56e46ff --- /dev/null +++ b/packages/core/src/shared/lsp/utils/cleanup.ts @@ -0,0 +1,41 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'path' +import { LspVersion } from '../types' +import { fs } from '../../../shared/fs/fs' +import { partition } from '../../../shared/utilities/tsUtils' +import { sort } from 'semver' + +async function getDownloadedVersions(installLocation: string) { + return (await fs.readdir(installLocation)).map(([f, _], __) => f) +} + +function isDelisted(manifestVersions: LspVersion[], targetVersion: string): boolean { + return manifestVersions.find((v) => v.serverVersion === targetVersion)?.isDelisted ?? false +} + +/** + * Delete all delisted versions and keep the two newest versions that remain + * @param manifest + * @param downloadDirectory + */ +export async function cleanLspDownloads(manifestVersions: LspVersion[], downloadDirectory: string): Promise { + const downloadedVersions = await getDownloadedVersions(downloadDirectory) + const [delistedVersions, remainingVersions] = partition(downloadedVersions, (v: string) => + isDelisted(manifestVersions, v) + ) + for (const v of delistedVersions) { + await fs.delete(path.join(downloadDirectory, v), { force: true, recursive: true }) + } + + if (remainingVersions.length <= 2) { + return + } + + for (const v of sort(remainingVersions).slice(0, -2)) { + await fs.delete(path.join(downloadDirectory, v), { force: true, recursive: true }) + } +} diff --git a/packages/core/src/shared/utilities/tsUtils.ts b/packages/core/src/shared/utilities/tsUtils.ts index e4fbf5a2b3f..2e4838c5b7a 100644 --- a/packages/core/src/shared/utilities/tsUtils.ts +++ b/packages/core/src/shared/utilities/tsUtils.ts @@ -94,6 +94,21 @@ export function createFactoryFunction any>(cto return (...args) => new ctor(...args) } +/** + * Split a list into two sublists based on the result of a predicate. + * @param lst list to split + * @param pred predicate to apply to each element + * @returns two nested lists, where for all items x in the left sublist, pred(x) returns true. The remaining elements are in the right sublist. + */ +export function partition(lst: T[], pred: (arg: T) => boolean): [T[], T[]] { + return lst.reduce( + ([leftAcc, rightAcc], item) => { + return pred(item) ? [[...leftAcc, item], rightAcc] : [leftAcc, [...rightAcc, item]] + }, + [[], []] as [T[], T[]] + ) +} + type NoSymbols = { [Property in keyof T]: Property extends symbol ? never : Property }[keyof T] export type InterfaceNoSymbol = Pick> /** diff --git a/packages/core/src/test/shared/lsp/utils/cleanup.test.ts b/packages/core/src/test/shared/lsp/utils/cleanup.test.ts new file mode 100644 index 00000000000..377039566de --- /dev/null +++ b/packages/core/src/test/shared/lsp/utils/cleanup.test.ts @@ -0,0 +1,93 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Uri } from 'vscode' +import { cleanLspDownloads, fs } from '../../../../shared' +import { createTestWorkspaceFolder } from '../../../testUtil' +import path from 'path' +import assert from 'assert' + +async function fakeInstallVersion(version: string, installationDir: string): Promise { + const versionDir = path.join(installationDir, version) + await fs.mkdir(versionDir) + await fs.writeFile(path.join(versionDir, 'file.txt'), 'content') +} + +async function fakeInstallVersions(versions: string[], installationDir: string): Promise { + for (const v of versions) { + await fakeInstallVersion(v, installationDir) + } +} + +describe('cleanLSPDownloads', function () { + let installationDir: Uri + + before(async function () { + installationDir = (await createTestWorkspaceFolder()).uri + }) + + afterEach(async function () { + const files = await fs.readdir(installationDir.fsPath) + for (const [name, _type] of files) { + await fs.delete(path.join(installationDir.fsPath, name), { force: true, recursive: true }) + } + }) + + after(async function () { + await fs.delete(installationDir, { force: true, recursive: true }) + }) + + it('keeps two newest versions', async function () { + await fakeInstallVersions(['1.0.0', '1.0.1', '1.1.1', '2.1.1'], installationDir.fsPath) + await cleanLspDownloads([], installationDir.fsPath) + + const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename) + assert.strictEqual(result.length, 2) + assert.ok(result.includes('2.1.1')) + assert.ok(result.includes('1.1.1')) + }) + + it('deletes delisted versions', async function () { + await fakeInstallVersions(['1.0.0', '1.0.1', '1.1.1', '2.1.1'], installationDir.fsPath) + await cleanLspDownloads([{ serverVersion: '1.1.1', isDelisted: true, targets: [] }], installationDir.fsPath) + + const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename) + assert.strictEqual(result.length, 2) + assert.ok(result.includes('2.1.1')) + assert.ok(result.includes('1.0.1')) + }) + + it('handles case where less than 2 versions are not delisted', async function () { + await fakeInstallVersions(['1.0.0', '1.0.1', '1.1.1', '2.1.1'], installationDir.fsPath) + await cleanLspDownloads( + [ + { serverVersion: '1.1.1', isDelisted: true, targets: [] }, + { serverVersion: '2.1.1', isDelisted: true, targets: [] }, + { serverVersion: '1.0.0', isDelisted: true, targets: [] }, + ], + installationDir.fsPath + ) + + const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename) + assert.strictEqual(result.length, 1) + assert.ok(result.includes('1.0.1')) + }) + + it('handles case where less than 2 versions exist', async function () { + await fakeInstallVersions(['1.0.0'], installationDir.fsPath) + await cleanLspDownloads([], installationDir.fsPath) + + const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename) + assert.strictEqual(result.length, 1) + }) + + it('does not install delisted version when no other option exists', async function () { + await fakeInstallVersions(['1.0.0'], installationDir.fsPath) + await cleanLspDownloads([{ serverVersion: '1.0.0', isDelisted: true, targets: [] }], installationDir.fsPath) + + const result = (await fs.readdir(installationDir.fsPath)).map(([filename, _filetype], _index) => filename) + assert.strictEqual(result.length, 0) + }) +}) diff --git a/packages/core/src/test/shared/utilities/tsUtils.test.ts b/packages/core/src/test/shared/utilities/tsUtils.test.ts new file mode 100644 index 00000000000..eb04da035e5 --- /dev/null +++ b/packages/core/src/test/shared/utilities/tsUtils.test.ts @@ -0,0 +1,16 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { partition } from '../../../shared/utilities/tsUtils' +import assert from 'assert' + +describe('partition', function () { + it('should split the list according to predicate', function () { + const items = [1, 2, 3, 4, 5, 6, 7, 8] + const [even, odd] = partition(items, (i) => i % 2 === 0) + assert.deepStrictEqual(even, [2, 4, 6, 8]) + assert.deepStrictEqual(odd, [1, 3, 5, 7]) + }) +}) From bdc49ebb0fe9682964cd13e7daec2368146751df Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Tue, 28 Jan 2025 09:53:54 -0500 Subject: [PATCH 16/22] fix(amazonq): Seperate inline completion from regular codewhisperer activation --- packages/amazonq/src/extension.ts | 14 ++- packages/amazonq/src/extensionNode.ts | 19 +-- packages/core/src/codewhisperer/activation.ts | 111 +++++++++--------- packages/core/src/codewhisperer/index.ts | 2 +- 4 files changed, 75 insertions(+), 71 deletions(-) diff --git a/packages/amazonq/src/extension.ts b/packages/amazonq/src/extension.ts index a4b53dbf66d..554c16acb94 100644 --- a/packages/amazonq/src/extension.ts +++ b/packages/amazonq/src/extension.ts @@ -4,7 +4,11 @@ */ import { AuthUtils, CredentialsStore, LoginManager, initializeAuth } from 'aws-core-vscode/auth' -import { activate as activateCodeWhisperer, shutdown as shutdownCodeWhisperer } from 'aws-core-vscode/codewhisperer' +import { + activate as activateCodeWhisperer, + shutdown as shutdownCodeWhisperer, + activateInlineCompletion, +} from 'aws-core-vscode/codewhisperer' import { makeEndpointsProvider, registerGenericCommands } from 'aws-core-vscode' import { CommonAuthWebview } from 'aws-core-vscode/login' import { @@ -31,6 +35,7 @@ import { setContext, setupUninstallHandler, maybeShowMinVscodeWarning, + Experiments, } from 'aws-core-vscode/shared' import { ExtStartUpSources } from 'aws-core-vscode/telemetry' import { VSCODE_EXTENSION_ID } from 'aws-core-vscode/utils' @@ -39,6 +44,7 @@ import * as semver from 'semver' import * as vscode from 'vscode' import { registerCommands } from './commands' import { focusAmazonQPanel } from 'aws-core-vscode/codewhispererChat' +import { activate as activateAmazonqLsp } from './lsp/activation' export const amazonQContextPrefix = 'amazonq' @@ -113,7 +119,13 @@ export async function activateAmazonQCommon(context: vscode.ExtensionContext, is const extContext = { extensionContext: context, } + // This contains every lsp agnostic things (auth, security scan, code scan) await activateCodeWhisperer(extContext as ExtContext) + if (Experiments.instance.get('amazonqLSP', false)) { + await activateAmazonqLsp(context) + } else { + await activateInlineCompletion() + } // Generic extension commands registerGenericCommands(context, amazonQContextPrefix) diff --git a/packages/amazonq/src/extensionNode.ts b/packages/amazonq/src/extensionNode.ts index 39e0a80643f..1a8506c2577 100644 --- a/packages/amazonq/src/extensionNode.ts +++ b/packages/amazonq/src/extensionNode.ts @@ -7,15 +7,7 @@ import * as vscode from 'vscode' import { activateAmazonQCommon, amazonQContextPrefix, deactivateCommon } from './extension' import { DefaultAmazonQAppInitContext } from 'aws-core-vscode/amazonq' import { activate as activateQGumby } from 'aws-core-vscode/amazonqGumby' -import { - ExtContext, - globals, - CrashMonitoring, - getLogger, - isNetworkError, - isSageMaker, - Experiments, -} from 'aws-core-vscode/shared' +import { ExtContext, globals, CrashMonitoring, getLogger, isNetworkError, isSageMaker } from 'aws-core-vscode/shared' import { filetypes, SchemaService } from 'aws-core-vscode/sharedNode' import { updateDevMode } from 'aws-core-vscode/dev' import { CommonAuthViewProvider } from 'aws-core-vscode/login' @@ -29,7 +21,6 @@ import { beta } from 'aws-core-vscode/dev' import { activate as activateNotifications, NotificationsController } from 'aws-core-vscode/notifications' import { AuthState, AuthUtil } from 'aws-core-vscode/codewhisperer' import { telemetry, AuthUserState } from 'aws-core-vscode/telemetry' -import { activate as activateAmazonqLsp } from './lsp/activation' export async function activate(context: vscode.ExtensionContext) { // IMPORTANT: No other code should be added to this function. Place it in one of the following 2 functions where appropriate. @@ -52,12 +43,8 @@ async function activateAmazonQNode(context: vscode.ExtensionContext) { extensionContext: context, } - if (Experiments.instance.get('amazonqLSP', false)) { - await activateAmazonqLsp(context) - } else { - await activateCWChat(context) - await activateQGumby(extContext as ExtContext) - } + await activateCWChat(context) + await activateQGumby(extContext as ExtContext) const authProvider = new CommonAuthViewProvider( context, diff --git a/packages/core/src/codewhisperer/activation.ts b/packages/core/src/codewhisperer/activation.ts index 17934f2fe38..05e11a7d52d 100644 --- a/packages/core/src/codewhisperer/activation.ts +++ b/packages/core/src/codewhisperer/activation.ts @@ -98,12 +98,12 @@ import { setContext } from '../shared/vscode/setContext' import { syncSecurityIssueWebview } from './views/securityIssue/securityIssueWebview' import { detectCommentAboveLine } from '../shared/utilities/commentUtils' import { UserWrittenCodeTracker } from './tracker/userWrittenCodeTracker' +import globals from '../shared/extensionGlobals' let localize: nls.LocalizeFunc export async function activate(context: ExtContext): Promise { localize = nls.loadMessageBundle() - const codewhispererSettings = CodeWhispererSettings.instance // initialize AuthUtil earlier to make sure it can listen to connection change events. const auth = AuthUtil.instance @@ -303,16 +303,6 @@ export async function activate(context: ExtContext): Promise { SecurityIssueProvider.instance.issues.some((group) => group.issues.some((issue) => issue.visible)) void setContext('aws.amazonq.security.noMatches', noMatches) }), - // manual trigger - Commands.register({ id: 'aws.amazonq.invokeInlineCompletion', autoconnect: true }, async () => { - invokeRecommendation( - vscode.window.activeTextEditor as vscode.TextEditor, - client, - await getConfigEntry() - ).catch((e) => { - getLogger().error('invokeRecommendation failed: %s', (e as Error).message) - }) - }), // select customization selectCustomizationPrompt.register(), // notify new customizations @@ -480,6 +470,52 @@ export async function activate(context: ExtContext): Promise { }) } + void FeatureConfigProvider.instance.fetchFeatureConfigs().catch((error) => { + getLogger().error('Failed to fetch feature configs - %s', error) + }) + + await Commands.tryExecute('aws.amazonq.refreshConnectionCallback') + container.ready() + + function setSubscriptionsForCodeIssues() { + context.extensionContext.subscriptions.push( + vscode.workspace.onDidChangeTextDocument(async (e) => { + if (e.document.uri.scheme !== 'file') { + return + } + const diagnostics = securityScanRender.securityDiagnosticCollection?.get(e.document.uri) + if (!diagnostics || diagnostics.length === 0) { + return + } + disposeSecurityDiagnostic(e) + + SecurityIssueProvider.instance.handleDocumentChange(e) + SecurityIssueTreeViewProvider.instance.refresh() + await syncSecurityIssueWebview(context) + + toggleIssuesVisibility((issue, filePath) => + filePath !== e.document.uri.fsPath + ? issue.visible + : !detectCommentAboveLine( + e.document, + issue.startLine, + CodeWhispererConstants.amazonqIgnoreNextLine + ) + ) + }) + ) + } +} + +export async function activateInlineCompletion() { + const codewhispererSettings = CodeWhispererSettings.instance + const client = new codewhispererClient.DefaultCodeWhispererClient() + + if (isInlineCompletionEnabled()) { + await setSubscriptionsforInlineCompletion() + await AuthUtil.instance.setVscodeContextProps() + } + function getAutoTriggerStatus(): boolean { return CodeSuggestionsState.instance.isSuggestionsEnabled() } @@ -500,17 +536,12 @@ export async function activate(context: ExtContext): Promise { } } - if (isInlineCompletionEnabled()) { - await setSubscriptionsforInlineCompletion() - await AuthUtil.instance.setVscodeContextProps() - } - async function setSubscriptionsforInlineCompletion() { RecommendationHandler.instance.subscribeSuggestionCommands() /** * Automated trigger */ - context.extensionContext.subscriptions.push( + globals.context.subscriptions.push( vscode.window.onDidChangeActiveTextEditor(async (editor) => { await RecommendationHandler.instance.onEditorChange() }), @@ -560,42 +591,16 @@ export async function activate(context: ExtContext): Promise { if (!RecommendationHandler.instance.isSuggestionVisible()) { await KeyStrokeHandler.instance.processKeyStroke(e, editor, client, await getConfigEntry()) } - }) - ) - } - - void FeatureConfigProvider.instance.fetchFeatureConfigs().catch((error) => { - getLogger().error('Failed to fetch feature configs - %s', error) - }) - - await Commands.tryExecute('aws.amazonq.refreshConnectionCallback') - container.ready() - - function setSubscriptionsForCodeIssues() { - context.extensionContext.subscriptions.push( - vscode.workspace.onDidChangeTextDocument(async (e) => { - if (e.document.uri.scheme !== 'file') { - return - } - const diagnostics = securityScanRender.securityDiagnosticCollection?.get(e.document.uri) - if (!diagnostics || diagnostics.length === 0) { - return - } - disposeSecurityDiagnostic(e) - - SecurityIssueProvider.instance.handleDocumentChange(e) - SecurityIssueTreeViewProvider.instance.refresh() - await syncSecurityIssueWebview(context) - - toggleIssuesVisibility((issue, filePath) => - filePath !== e.document.uri.fsPath - ? issue.visible - : !detectCommentAboveLine( - e.document, - issue.startLine, - CodeWhispererConstants.amazonqIgnoreNextLine - ) - ) + }), + // manual trigger + Commands.register({ id: 'aws.amazonq.invokeInlineCompletion', autoconnect: true }, async () => { + invokeRecommendation( + vscode.window.activeTextEditor as vscode.TextEditor, + client, + await getConfigEntry() + ).catch((e) => { + getLogger().error('invokeRecommendation failed: %s', (e as Error).message) + }) }) ) } diff --git a/packages/core/src/codewhisperer/index.ts b/packages/core/src/codewhisperer/index.ts index 3aea72fb4ca..7c516985d5c 100644 --- a/packages/core/src/codewhisperer/index.ts +++ b/packages/core/src/codewhisperer/index.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { activate, shutdown } from './activation' +export { activate, shutdown, activateInlineCompletion } from './activation' export * from './util/authUtil' export * from './models/model' export * from './models/constants' From f93a6bf3b8a1db7a79d0e9472b700d024268ce4c Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Tue, 28 Jan 2025 11:03:31 -0500 Subject: [PATCH 17/22] refactor(amazonq): Move codewhisperer activation from core to amazonq --- packages/amazonq/src/extension.ts | 7 +- packages/amazonq/src/inline/activation.ts | 124 ++++++++++++++++++ packages/core/src/codewhisperer/activation.ts | 108 --------------- packages/core/src/codewhisperer/index.ts | 2 +- 4 files changed, 127 insertions(+), 114 deletions(-) create mode 100644 packages/amazonq/src/inline/activation.ts diff --git a/packages/amazonq/src/extension.ts b/packages/amazonq/src/extension.ts index 554c16acb94..56a3e2f006c 100644 --- a/packages/amazonq/src/extension.ts +++ b/packages/amazonq/src/extension.ts @@ -4,11 +4,7 @@ */ import { AuthUtils, CredentialsStore, LoginManager, initializeAuth } from 'aws-core-vscode/auth' -import { - activate as activateCodeWhisperer, - shutdown as shutdownCodeWhisperer, - activateInlineCompletion, -} from 'aws-core-vscode/codewhisperer' +import { activate as activateCodeWhisperer, shutdown as shutdownCodeWhisperer } from 'aws-core-vscode/codewhisperer' import { makeEndpointsProvider, registerGenericCommands } from 'aws-core-vscode' import { CommonAuthWebview } from 'aws-core-vscode/login' import { @@ -45,6 +41,7 @@ import * as vscode from 'vscode' import { registerCommands } from './commands' import { focusAmazonQPanel } from 'aws-core-vscode/codewhispererChat' import { activate as activateAmazonqLsp } from './lsp/activation' +import { activate as activateInlineCompletion } from './inline/activation' export const amazonQContextPrefix = 'amazonq' diff --git a/packages/amazonq/src/inline/activation.ts b/packages/amazonq/src/inline/activation.ts new file mode 100644 index 00000000000..d786047b2aa --- /dev/null +++ b/packages/amazonq/src/inline/activation.ts @@ -0,0 +1,124 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import vscode from 'vscode' +import { + AuthUtil, + CodeSuggestionsState, + CodeWhispererCodeCoverageTracker, + CodeWhispererConstants, + CodeWhispererSettings, + ConfigurationEntry, + DefaultCodeWhispererClient, + invokeRecommendation, + isInlineCompletionEnabled, + KeyStrokeHandler, + RecommendationHandler, + runtimeLanguageContext, + TelemetryHelper, + UserWrittenCodeTracker, + vsCodeState, +} from 'aws-core-vscode/codewhisperer' +import { Commands, getLogger, globals, sleep } from 'aws-core-vscode/shared' + +export async function activate() { + const codewhispererSettings = CodeWhispererSettings.instance + const client = new DefaultCodeWhispererClient() + + if (isInlineCompletionEnabled()) { + await setSubscriptionsforInlineCompletion() + await AuthUtil.instance.setVscodeContextProps() + } + + function getAutoTriggerStatus(): boolean { + return CodeSuggestionsState.instance.isSuggestionsEnabled() + } + + async function getConfigEntry(): Promise { + const isShowMethodsEnabled: boolean = + vscode.workspace.getConfiguration('editor').get('suggest.showMethods') || false + const isAutomatedTriggerEnabled: boolean = getAutoTriggerStatus() + const isManualTriggerEnabled: boolean = true + const isSuggestionsWithCodeReferencesEnabled = codewhispererSettings.isSuggestionsWithCodeReferencesEnabled() + + // TODO:remove isManualTriggerEnabled + return { + isShowMethodsEnabled, + isManualTriggerEnabled, + isAutomatedTriggerEnabled, + isSuggestionsWithCodeReferencesEnabled, + } + } + + async function setSubscriptionsforInlineCompletion() { + RecommendationHandler.instance.subscribeSuggestionCommands() + + /** + * Automated trigger + */ + globals.context.subscriptions.push( + vscode.window.onDidChangeActiveTextEditor(async (editor) => { + await RecommendationHandler.instance.onEditorChange() + }), + vscode.window.onDidChangeWindowState(async (e) => { + await RecommendationHandler.instance.onFocusChange() + }), + vscode.window.onDidChangeTextEditorSelection(async (e) => { + await RecommendationHandler.instance.onCursorChange(e) + }), + vscode.workspace.onDidChangeTextDocument(async (e) => { + const editor = vscode.window.activeTextEditor + if (!editor) { + return + } + if (e.document !== editor.document) { + return + } + if (!runtimeLanguageContext.isLanguageSupported(e.document)) { + return + } + + CodeWhispererCodeCoverageTracker.getTracker(e.document.languageId)?.countTotalTokens(e) + UserWrittenCodeTracker.instance.onTextDocumentChange(e) + /** + * Handle this keystroke event only when + * 1. It is not a backspace + * 2. It is not caused by CodeWhisperer editing + * 3. It is not from undo/redo. + */ + if (e.contentChanges.length === 0 || vsCodeState.isCodeWhispererEditing) { + return + } + + if (vsCodeState.lastUserModificationTime) { + TelemetryHelper.instance.setTimeSinceLastModification( + performance.now() - vsCodeState.lastUserModificationTime + ) + } + vsCodeState.lastUserModificationTime = performance.now() + /** + * Important: Doing this sleep(10) is to make sure + * 1. this event is processed by vs code first + * 2. editor.selection.active has been successfully updated by VS Code + * Then this event can be processed by our code. + */ + await sleep(CodeWhispererConstants.vsCodeCursorUpdateDelay) + if (!RecommendationHandler.instance.isSuggestionVisible()) { + await KeyStrokeHandler.instance.processKeyStroke(e, editor, client, await getConfigEntry()) + } + }), + // manual trigger + Commands.register({ id: 'aws.amazonq.invokeInlineCompletion', autoconnect: true }, async () => { + invokeRecommendation( + vscode.window.activeTextEditor as vscode.TextEditor, + client, + await getConfigEntry() + ).catch((e) => { + getLogger().error('invokeRecommendation failed: %s', (e as Error).message) + }) + }) + ) + } +} diff --git a/packages/core/src/codewhisperer/activation.ts b/packages/core/src/codewhisperer/activation.ts index 05e11a7d52d..64c6217b383 100644 --- a/packages/core/src/codewhisperer/activation.ts +++ b/packages/core/src/codewhisperer/activation.ts @@ -6,12 +6,9 @@ import * as vscode from 'vscode' import * as nls from 'vscode-nls' import { getTabSizeSetting } from '../shared/utilities/editorUtilities' -import { KeyStrokeHandler } from './service/keyStrokeHandler' import * as EditorContext from './util/editorContext' import * as CodeWhispererConstants from './models/constants' import { - vsCodeState, - ConfigurationEntry, CodeSuggestionsState, CodeScansState, SecurityTreeViewFilterState, @@ -19,13 +16,11 @@ import { CodeScanIssue, CodeIssueGroupingStrategyState, } from './models/model' -import { invokeRecommendation } from './commands/invokeRecommendation' import { acceptSuggestion } from './commands/onInlineAcceptance' import { CodeWhispererSettings } from './util/codewhispererSettings' import { ExtContext } from '../shared/extensions' import { CodeWhispererTracker } from './tracker/codewhispererTracker' import * as codewhispererClient from './client/codewhisperer' -import { runtimeLanguageContext } from './util/runtimeLanguageContext' import { getLogger } from '../shared/logger' import { enableCodeSuggestions, @@ -59,7 +54,6 @@ import { showExploreAgentsView, showCodeIssueGroupingQuickPick, } from './commands/basicCommands' -import { sleep } from '../shared/utilities/timeoutUtils' import { ReferenceLogViewProvider } from './service/referenceLogViewProvider' import { ReferenceHoverProvider } from './service/referenceHoverProvider' import { ReferenceInlineProvider } from './service/referenceInlineProvider' @@ -73,7 +67,6 @@ import { RecommendationHandler } from './service/recommendationHandler' import { Commands, registerCommandErrorHandler, registerDeclaredCommands } from '../shared/vscode/commands2' import { InlineCompletionService, refreshStatusBar } from './service/inlineCompletionService' import { isInlineCompletionEnabled } from './util/commonUtil' -import { CodeWhispererCodeCoverageTracker } from './tracker/codewhispererCodeCoverageTracker' import { AuthUtil } from './util/authUtil' import { ImportAdderProvider } from './service/importAdderProvider' import { TelemetryHelper } from './util/telemetryHelper' @@ -97,8 +90,6 @@ import { SecurityIssueTreeViewProvider } from './service/securityIssueTreeViewPr import { setContext } from '../shared/vscode/setContext' import { syncSecurityIssueWebview } from './views/securityIssue/securityIssueWebview' import { detectCommentAboveLine } from '../shared/utilities/commentUtils' -import { UserWrittenCodeTracker } from './tracker/userWrittenCodeTracker' -import globals from '../shared/extensionGlobals' let localize: nls.LocalizeFunc @@ -507,105 +498,6 @@ export async function activate(context: ExtContext): Promise { } } -export async function activateInlineCompletion() { - const codewhispererSettings = CodeWhispererSettings.instance - const client = new codewhispererClient.DefaultCodeWhispererClient() - - if (isInlineCompletionEnabled()) { - await setSubscriptionsforInlineCompletion() - await AuthUtil.instance.setVscodeContextProps() - } - - function getAutoTriggerStatus(): boolean { - return CodeSuggestionsState.instance.isSuggestionsEnabled() - } - - async function getConfigEntry(): Promise { - const isShowMethodsEnabled: boolean = - vscode.workspace.getConfiguration('editor').get('suggest.showMethods') || false - const isAutomatedTriggerEnabled: boolean = getAutoTriggerStatus() - const isManualTriggerEnabled: boolean = true - const isSuggestionsWithCodeReferencesEnabled = codewhispererSettings.isSuggestionsWithCodeReferencesEnabled() - - // TODO:remove isManualTriggerEnabled - return { - isShowMethodsEnabled, - isManualTriggerEnabled, - isAutomatedTriggerEnabled, - isSuggestionsWithCodeReferencesEnabled, - } - } - - async function setSubscriptionsforInlineCompletion() { - RecommendationHandler.instance.subscribeSuggestionCommands() - /** - * Automated trigger - */ - globals.context.subscriptions.push( - vscode.window.onDidChangeActiveTextEditor(async (editor) => { - await RecommendationHandler.instance.onEditorChange() - }), - vscode.window.onDidChangeWindowState(async (e) => { - await RecommendationHandler.instance.onFocusChange() - }), - vscode.window.onDidChangeTextEditorSelection(async (e) => { - await RecommendationHandler.instance.onCursorChange(e) - }), - vscode.workspace.onDidChangeTextDocument(async (e) => { - const editor = vscode.window.activeTextEditor - if (!editor) { - return - } - if (e.document !== editor.document) { - return - } - if (!runtimeLanguageContext.isLanguageSupported(e.document)) { - return - } - - CodeWhispererCodeCoverageTracker.getTracker(e.document.languageId)?.countTotalTokens(e) - UserWrittenCodeTracker.instance.onTextDocumentChange(e) - /** - * Handle this keystroke event only when - * 1. It is not a backspace - * 2. It is not caused by CodeWhisperer editing - * 3. It is not from undo/redo. - */ - if (e.contentChanges.length === 0 || vsCodeState.isCodeWhispererEditing) { - return - } - - if (vsCodeState.lastUserModificationTime) { - TelemetryHelper.instance.setTimeSinceLastModification( - performance.now() - vsCodeState.lastUserModificationTime - ) - } - vsCodeState.lastUserModificationTime = performance.now() - /** - * Important: Doing this sleep(10) is to make sure - * 1. this event is processed by vs code first - * 2. editor.selection.active has been successfully updated by VS Code - * Then this event can be processed by our code. - */ - await sleep(CodeWhispererConstants.vsCodeCursorUpdateDelay) - if (!RecommendationHandler.instance.isSuggestionVisible()) { - await KeyStrokeHandler.instance.processKeyStroke(e, editor, client, await getConfigEntry()) - } - }), - // manual trigger - Commands.register({ id: 'aws.amazonq.invokeInlineCompletion', autoconnect: true }, async () => { - invokeRecommendation( - vscode.window.activeTextEditor as vscode.TextEditor, - client, - await getConfigEntry() - ).catch((e) => { - getLogger().error('invokeRecommendation failed: %s', (e as Error).message) - }) - }) - ) - } -} - export async function shutdown() { RecommendationHandler.instance.reportUserDecisions(-1) await CodeWhispererTracker.getTracker().shutdown() diff --git a/packages/core/src/codewhisperer/index.ts b/packages/core/src/codewhisperer/index.ts index 7c516985d5c..3aea72fb4ca 100644 --- a/packages/core/src/codewhisperer/index.ts +++ b/packages/core/src/codewhisperer/index.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { activate, shutdown, activateInlineCompletion } from './activation' +export { activate, shutdown } from './activation' export * from './util/authUtil' export * from './models/model' export * from './models/constants' From b4b7d1716fb307ba4311fb060448c4588332759d Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Tue, 28 Jan 2025 12:23:09 -0500 Subject: [PATCH 18/22] fix(amazonq): Register invokeInlineCompletion and rejectCodeSuggestion with lsp implementation --- packages/amazonq/src/lsp/activation.ts | 10 +++- .../service/recommendationHandler.ts | 46 +++++++++++++------ 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts index bdf4a59ab1a..10e0c93eec5 100644 --- a/packages/amazonq/src/lsp/activation.ts +++ b/packages/amazonq/src/lsp/activation.ts @@ -6,12 +6,20 @@ import vscode from 'vscode' import { startLanguageServer } from './client' import { AmazonQLSPResolver } from './lspInstaller' -import { ToolkitError } from 'aws-core-vscode/shared' +import { Commands, ToolkitError } from 'aws-core-vscode/shared' export async function activate(ctx: vscode.ExtensionContext): Promise { try { const installResult = await new AmazonQLSPResolver().resolve() await startLanguageServer(ctx, installResult.resourcePaths) + ctx.subscriptions.push( + Commands.register({ id: 'aws.amazonq.invokeInlineCompletion', autoconnect: true }, async () => { + await vscode.commands.executeCommand('editor.action.inlineSuggest.trigger') + }), + Commands.declare('aws.amazonq.rejectCodeSuggestion', () => async () => { + await vscode.commands.executeCommand('editor.action.inlineSuggest.hide') + }).register() + ) } catch (err) { const e = err as ToolkitError void vscode.window.showInformationMessage(`Unable to launch amazonq language server: ${e.message}`) diff --git a/packages/core/src/codewhisperer/service/recommendationHandler.ts b/packages/core/src/codewhisperer/service/recommendationHandler.ts index 1f5096ad1cc..66542ee6681 100644 --- a/packages/core/src/codewhisperer/service/recommendationHandler.ts +++ b/packages/core/src/codewhisperer/service/recommendationHandler.ts @@ -50,23 +50,38 @@ import { UserWrittenCodeTracker } from '../tracker/userWrittenCodeTracker' * It does not contain UI/UX related logic */ -// below commands override VS Code inline completion commands -const prevCommand = Commands.declare('editor.action.inlineSuggest.showPrevious', () => async () => { - await RecommendationHandler.instance.showRecommendation(-1) -}) -const nextCommand = Commands.declare('editor.action.inlineSuggest.showNext', () => async () => { - await RecommendationHandler.instance.showRecommendation(1) -}) - -const rejectCommand = Commands.declare('aws.amazonq.rejectCodeSuggestion', () => async () => { - telemetry.record({ - traceId: TelemetryHelper.instance.traceId, +/** + * Commands as a level of indirection so that declare doesn't intercept any registrations for the + * language server implementation. + * + * Otherwise you'll get: + * "Unable to launch amazonq language server: Command "aws.amazonq.rejectCodeSuggestion" has already been declared by the Toolkit" + */ +function createCommands() { + // below commands override VS Code inline completion commands + const prevCommand = Commands.declare('editor.action.inlineSuggest.showPrevious', () => async () => { + await RecommendationHandler.instance.showRecommendation(-1) + }) + const nextCommand = Commands.declare('editor.action.inlineSuggest.showNext', () => async () => { + await RecommendationHandler.instance.showRecommendation(1) }) - await vscode.commands.executeCommand('editor.action.inlineSuggest.hide') - RecommendationHandler.instance.reportUserDecisions(-1) - await Commands.tryExecute('aws.amazonq.refreshAnnotation') -}) + const rejectCommand = Commands.declare('aws.amazonq.rejectCodeSuggestion', () => async () => { + telemetry.record({ + traceId: TelemetryHelper.instance.traceId, + }) + + await vscode.commands.executeCommand('editor.action.inlineSuggest.hide') + RecommendationHandler.instance.reportUserDecisions(-1) + await Commands.tryExecute('aws.amazonq.refreshAnnotation') + }) + + return { + prevCommand, + nextCommand, + rejectCommand, + } +} const lock = new AsyncLock({ maxPending: 1 }) @@ -579,6 +594,7 @@ export class RecommendationHandler { // They are subscribed when suggestion starts and disposed when suggestion is accepted/rejected // to avoid impacting other plugins or user who uses this API private registerCommandOverrides() { + const { prevCommand, nextCommand, rejectCommand } = createCommands() this.prev = prevCommand.register() this.next = nextCommand.register() this.reject = rejectCommand.register() From df5a240105721a315eb77e8ba7604805f3c68e22 Mon Sep 17 00:00:00 2001 From: Josh Pinkney Date: Wed, 29 Jan 2025 10:01:54 -0500 Subject: [PATCH 19/22] refactor(amazonq): move src/inline -> src/app/inline --- packages/amazonq/src/{ => app}/inline/activation.ts | 0 packages/amazonq/src/{ => app}/inline/completion.ts | 0 packages/amazonq/src/extension.ts | 2 +- packages/amazonq/src/lsp/client.ts | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename packages/amazonq/src/{ => app}/inline/activation.ts (100%) rename packages/amazonq/src/{ => app}/inline/completion.ts (100%) diff --git a/packages/amazonq/src/inline/activation.ts b/packages/amazonq/src/app/inline/activation.ts similarity index 100% rename from packages/amazonq/src/inline/activation.ts rename to packages/amazonq/src/app/inline/activation.ts diff --git a/packages/amazonq/src/inline/completion.ts b/packages/amazonq/src/app/inline/completion.ts similarity index 100% rename from packages/amazonq/src/inline/completion.ts rename to packages/amazonq/src/app/inline/completion.ts diff --git a/packages/amazonq/src/extension.ts b/packages/amazonq/src/extension.ts index 56a3e2f006c..7e8216387d6 100644 --- a/packages/amazonq/src/extension.ts +++ b/packages/amazonq/src/extension.ts @@ -41,7 +41,7 @@ import * as vscode from 'vscode' import { registerCommands } from './commands' import { focusAmazonQPanel } from 'aws-core-vscode/codewhispererChat' import { activate as activateAmazonqLsp } from './lsp/activation' -import { activate as activateInlineCompletion } from './inline/activation' +import { activate as activateInlineCompletion } from './app/inline/activation' export const amazonQContextPrefix = 'amazonq' diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index 4595b137636..c7e20bbd480 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -7,7 +7,7 @@ import vscode, { env, version } from 'vscode' import * as nls from 'vscode-nls' import * as crypto from 'crypto' import { LanguageClient, LanguageClientOptions, ServerOptions, TransportKind } from 'vscode-languageclient' -import { registerInlineCompletion } from '../inline/completion' +import { registerInlineCompletion } from '../app/inline/completion' import { AmazonQLspAuth, encryptionKey, notificationTypes } from './auth' import { AuthUtil } from 'aws-core-vscode/codewhisperer' import { ConnectionMetadata } from '@aws/language-server-runtimes/protocol' From 9d4f971914d3e2a005d0d3b2b622fb00ab401836 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Wed, 29 Jan 2025 12:38:56 -0500 Subject: [PATCH 20/22] docs(amazonq): Add instructions for debugging flare lsps (#6455) ## Problem It's painful to setup debugging with flare ## Solution Add instructions to make it easy --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- docs/lsp-debugging.md | 23 ++++++++++++++++ packages/amazonq/.vscode/launch.json | 26 ++++++++++++++++++ packages/amazonq/src/lsp/client.ts | 27 +++++-------------- .../core/src/shared/lsp/utils/platform.ts | 4 ++- 4 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 docs/lsp-debugging.md diff --git a/docs/lsp-debugging.md b/docs/lsp-debugging.md new file mode 100644 index 00000000000..be65c2882a7 --- /dev/null +++ b/docs/lsp-debugging.md @@ -0,0 +1,23 @@ +## Language Server Debugging + +1. Clone https://github.com/aws/language-servers.git and set it up in the same workspace as this project + + e.g. + + ``` + /aws-toolkit-vscode + /toolkit + /core + /amazonq + /language-servers + ``` + +2. Inside of the language-servers project run: + ``` + npm install + npm run compile + npm run package + ``` + to get the project setup +3. Uncomment the `AWS_LANGUAGE_SERVER_OVERRIDE` variable in `amazonq/.vscode/launch.json` Extension configuration +4. Use the `Launch LSP with Debugging` configuration and set breakpoints in VSCode or the language server diff --git a/packages/amazonq/.vscode/launch.json b/packages/amazonq/.vscode/launch.json index 120eacdb44d..be456c89ac1 100644 --- a/packages/amazonq/.vscode/launch.json +++ b/packages/amazonq/.vscode/launch.json @@ -19,6 +19,7 @@ "env": { "SSMDOCUMENT_LANGUAGESERVER_PORT": "6010", "WEBPACK_DEVELOPER_SERVER": "http://localhost:8080" + // "AWS_LANGUAGE_SERVER_OVERRIDE": "${workspaceFolder}/../../../language-servers/app/aws-lsp-codewhisperer-runtimes/out/token-standalone.js", }, "envFile": "${workspaceFolder}/.local.env", "outFiles": ["${workspaceFolder}/dist/**/*.js", "${workspaceFolder}/../core/dist/**/*.js"], @@ -140,6 +141,31 @@ "group": "4_E2ETestCurrentFile", "order": 2 } + }, + { + "name": "Attach to Language Server", + "type": "node", + "request": "attach", + "port": 6080, // Hard defined in core/src/shared/lsp/platform.ts + "outFiles": ["${workspaceFolder}/../../../language-servers/**/out/**/*.js"], + "skipFiles": [ + "/**", + "${workspaceFolder}/../../../language-servers/**/node_modules/**/*.js" + ], + "restart": { + "maxAttempts": 10, + "delay": 1000 + } + } + ], + "compounds": [ + { + "name": "Launch LSP with Debugging", + "configurations": ["Extension", "Attach to Language Server"], + "presentation": { + "group": "1_Extension", + "order": 5 + } } ] } diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index c7e20bbd480..40965af43ed 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -6,7 +6,7 @@ import vscode, { env, version } from 'vscode' import * as nls from 'vscode-nls' import * as crypto from 'crypto' -import { LanguageClient, LanguageClientOptions, ServerOptions, TransportKind } from 'vscode-languageclient' +import { LanguageClient, LanguageClientOptions } from 'vscode-languageclient' import { registerInlineCompletion } from '../app/inline/completion' import { AmazonQLspAuth, encryptionKey, notificationTypes } from './auth' import { AuthUtil } from 'aws-core-vscode/codewhisperer' @@ -18,9 +18,12 @@ const localize = nls.loadMessageBundle() export async function startLanguageServer(extensionContext: vscode.ExtensionContext, resourcePaths: ResourcePaths) { const toDispose = extensionContext.subscriptions - // The debug options for the server - // --inspect=6009: runs the server in Node's Inspector mode so VS Code can attach to the server for debugging - const debugOptions = { + const serverModule = resourcePaths.lsp + + const serverOptions = createServerOptions({ + encryptionKey, + executable: resourcePaths.node, + serverModule, execArgv: [ '--nolazy', '--preserve-symlinks', @@ -28,22 +31,6 @@ export async function startLanguageServer(extensionContext: vscode.ExtensionCont '--pre-init-encryption', '--set-credentials-encryption-key', ], - } - - const serverModule = resourcePaths.lsp - - // If the extension is launch in debug mode the debug server options are use - // Otherwise the run options are used - let serverOptions: ServerOptions = { - run: { module: serverModule, transport: TransportKind.ipc }, - debug: { module: serverModule, transport: TransportKind.ipc, options: debugOptions }, - } - - serverOptions = createServerOptions({ - encryptionKey, - executable: resourcePaths.node, - serverModule, - execArgv: debugOptions.execArgv, }) const documentSelector = [{ scheme: 'file', language: '*' }] diff --git a/packages/core/src/shared/lsp/utils/platform.ts b/packages/core/src/shared/lsp/utils/platform.ts index 5c2d7a59123..44e68c423d2 100644 --- a/packages/core/src/shared/lsp/utils/platform.ts +++ b/packages/core/src/shared/lsp/utils/platform.ts @@ -5,6 +5,7 @@ import { ToolkitError } from '../../errors' import { ChildProcess } from '../../utilities/processUtils' +import { isDebugInstance } from '../../vscode/env' export function getNodeExecutableName(): string { return process.platform === 'win32' ? 'node.exe' : 'node' @@ -35,7 +36,8 @@ export function createServerOptions({ execArgv: string[] }) { return async () => { - const lspProcess = new ChildProcess(executable, [serverModule, ...execArgv]) + const debugArgs = isDebugInstance() ? '--inspect=6080' : '' + const lspProcess = new ChildProcess(executable, [debugArgs, serverModule, ...execArgv]) // this is a long running process, awaiting it will never resolve void lspProcess.run() From 633b05b9e0d2a2b0902d6452ae3d2f242d303008 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Thu, 6 Feb 2025 12:48:58 -0500 Subject: [PATCH 21/22] telemetry(lsp): Integrate language server/manifest resolver telemetry (#6385) ## Problem LSP downloading processes does not emit any telemetry. ## Solution ### Refactoring - separate verification from core downloading steps so that we can capture it as its own telemetry event. - Note: this refactor means that if all the downloaded content cannot be verified, no files will be written to disk. - Introduce abstraction of `StageResolver` to make telemetry instrumentation more natural. ### Metric Behavior - metric for each stage of the LSP setup process. - can be emitted multiple times per stage to capture specific error codes. See tests for examples. - Commons repo PR: https://github.com/aws/aws-toolkit-common/pull/961 - bumped commons version to [1.0.296](https://github.com/aws/aws-toolkit-common/commit/8df7a87557f40e5ccd499a8f3ce38976893c8b10) to include this change. ### Testing - adds basic tests for `ManifestResolver` in `packages/core/src/test/shared/lsp/manifestResolver.test.ts`. - adds basic tests for `LanguageServerResolver` in `packages/core/src/test/shared/lsp/lspResolver.test.ts`. - Currently `LanguageServerResolver` only supports mac, so its tests skip on non-mac platforms. There is a techdebt test to address this. ## Future Work - Update logging messages to be LSP specific. --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- package-lock.json | 8 +- package.json | 2 +- packages/amazonq/src/lsp/activation.ts | 8 +- .../amazonq/test/e2e/lsp/lspInstaller.test.ts | 107 ++++++++++++ packages/core/package.nls.json | 2 +- .../core/src/amazonq/lsp/lspController.ts | 13 +- packages/core/src/shared/index.ts | 1 + packages/core/src/shared/lsp/lspResolver.ts | 156 +++++++++++------- .../core/src/shared/lsp/manifestResolver.ts | 27 ++- packages/core/src/shared/lsp/types.ts | 6 +- .../core/src/shared/lsp/utils/setupStage.ts | 66 ++++++++ packages/core/src/shared/utilities/tsUtils.ts | 17 ++ .../test/shared/lsp/manifestResolver.test.ts | 101 ++++++++++++ .../src/test/shared/utilities/tsUtils.test.ts | 25 ++- 14 files changed, 459 insertions(+), 80 deletions(-) create mode 100644 packages/core/src/shared/lsp/utils/setupStage.ts create mode 100644 packages/core/src/test/shared/lsp/manifestResolver.test.ts diff --git a/package-lock.json b/package-lock.json index db334e69287..f081d12331f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "vscode-nls-dev": "^4.0.4" }, "devDependencies": { - "@aws-toolkits/telemetry": "^1.0.295", + "@aws-toolkits/telemetry": "^1.0.296", "@playwright/browser-chromium": "^1.43.1", "@stylistic/eslint-plugin": "^2.11.0", "@types/he": "^1.2.3", @@ -6047,9 +6047,9 @@ } }, "node_modules/@aws-toolkits/telemetry": { - "version": "1.0.295", - "resolved": "https://registry.npmjs.org/@aws-toolkits/telemetry/-/telemetry-1.0.295.tgz", - "integrity": "sha512-NGBM5vhNNHwEhok3asXpUW7oZv/z8mjZaf34LGflqEh/5+VraTd76T+QBz18sC+nE2sPvhTO+zjptR9zg5bBUA==", + "version": "1.0.296", + "resolved": "https://registry.npmjs.org/@aws-toolkits/telemetry/-/telemetry-1.0.296.tgz", + "integrity": "sha512-laPLYzImOCyAyZWWrmEXWWXZv2xeS5PKatVksGuCxjSohI+20fuCAUjhwMtyq7bRUPkOTkG9pzVYB7Rg49QHFg==", "dev": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index f3b9b4a308c..8e03e4326c8 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "mergeReports": "ts-node ./scripts/mergeReports.ts" }, "devDependencies": { - "@aws-toolkits/telemetry": "^1.0.295", + "@aws-toolkits/telemetry": "^1.0.296", "@playwright/browser-chromium": "^1.43.1", "@stylistic/eslint-plugin": "^2.11.0", "@types/he": "^1.2.3", diff --git a/packages/amazonq/src/lsp/activation.ts b/packages/amazonq/src/lsp/activation.ts index 10e0c93eec5..783c35ddc94 100644 --- a/packages/amazonq/src/lsp/activation.ts +++ b/packages/amazonq/src/lsp/activation.ts @@ -6,12 +6,14 @@ import vscode from 'vscode' import { startLanguageServer } from './client' import { AmazonQLSPResolver } from './lspInstaller' -import { Commands, ToolkitError } from 'aws-core-vscode/shared' +import { Commands, lspSetupStage, ToolkitError } from 'aws-core-vscode/shared' export async function activate(ctx: vscode.ExtensionContext): Promise { try { - const installResult = await new AmazonQLSPResolver().resolve() - await startLanguageServer(ctx, installResult.resourcePaths) + await lspSetupStage('all', async () => { + const installResult = await new AmazonQLSPResolver().resolve() + await lspSetupStage('launch', async () => await startLanguageServer(ctx, installResult.resourcePaths)) + }) ctx.subscriptions.push( Commands.register({ id: 'aws.amazonq.invokeInlineCompletion', autoconnect: true }, async () => { await vscode.commands.executeCommand('editor.action.inlineSuggest.trigger') diff --git a/packages/amazonq/test/e2e/lsp/lspInstaller.test.ts b/packages/amazonq/test/e2e/lsp/lspInstaller.test.ts index 0bf2edafa3f..5e1d3f68d47 100644 --- a/packages/amazonq/test/e2e/lsp/lspInstaller.test.ts +++ b/packages/amazonq/test/e2e/lsp/lspInstaller.test.ts @@ -8,12 +8,17 @@ import sinon from 'sinon' import { AmazonQLSPResolver, supportedLspServerVersions } from '../../../src/lsp/lspInstaller' import { fs, + globals, LanguageServerResolver, makeTemporaryToolkitFolder, ManifestResolver, + manifestStorageKey, request, } from 'aws-core-vscode/shared' import * as semver from 'semver' +import { assertTelemetry } from 'aws-core-vscode/test' +import { LspController } from 'aws-core-vscode/amazonq' +import { LanguageServerSetup } from 'aws-core-vscode/telemetry' function createVersion(version: string) { return { @@ -40,12 +45,22 @@ describe('AmazonQLSPInstaller', () => { let resolver: AmazonQLSPResolver let sandbox: sinon.SinonSandbox let tempDir: string + // If globalState contains an ETag that is up to date with remote, we won't fetch it resulting in inconsistent behavior. + // Therefore, we clear it temporarily for these tests to ensure consistent behavior. + let manifestStorage: { [key: string]: any } + + before(async () => { + manifestStorage = globals.globalState.get(manifestStorageKey) || {} + }) beforeEach(async () => { sandbox = sinon.createSandbox() resolver = new AmazonQLSPResolver() tempDir = await makeTemporaryToolkitFolder() sandbox.stub(LanguageServerResolver.prototype, 'defaultDownloadFolder').returns(tempDir) + // Called on extension activation and can contaminate telemetry. + sandbox.stub(LspController.prototype, 'trySetupLsp') + await globals.globalState.update(manifestStorageKey, {}) }) afterEach(async () => { @@ -56,6 +71,10 @@ describe('AmazonQLSPInstaller', () => { }) }) + after(async () => { + await globals.globalState.update(manifestStorageKey, manifestStorage) + }) + describe('resolve()', () => { it('uses AWS_LANGUAGE_SERVER_OVERRIDE', async () => { const overridePath = '/custom/path/to/lsp' @@ -117,6 +136,94 @@ describe('AmazonQLSPInstaller', () => { assert.ok(fallback.assetDirectory.startsWith(tempDir)) assert.deepStrictEqual(fallback.location, 'fallback') assert.ok(semver.satisfies(fallback.version, supportedLspServerVersions)) + + /* First Try Telemetry + getManifest: remote succeeds + getServer: cache fails then remote succeeds. + validate: succeeds. + */ + const firstTryTelemetry: Partial[] = [ + { + id: 'AmazonQ', + manifestLocation: 'remote', + languageServerSetupStage: 'getManifest', + result: 'Succeeded', + }, + { + id: 'AmazonQ', + languageServerLocation: 'cache', + languageServerSetupStage: 'getServer', + result: 'Failed', + }, + { + id: 'AmazonQ', + languageServerLocation: 'remote', + languageServerSetupStage: 'validate', + result: 'Succeeded', + }, + { + id: 'AmazonQ', + languageServerLocation: 'remote', + languageServerSetupStage: 'getServer', + result: 'Succeeded', + }, + ] + + /* Second Try Telemetry + getManifest: remote fails, then cache succeeds. + getServer: cache succeeds + validate: doesn't run since its cached. + */ + const secondTryTelemetry: Partial[] = [ + { + id: 'AmazonQ', + manifestLocation: 'remote', + languageServerSetupStage: 'getManifest', + result: 'Failed', + }, + { + id: 'AmazonQ', + manifestLocation: 'cache', + languageServerSetupStage: 'getManifest', + result: 'Succeeded', + }, + { + id: 'AmazonQ', + languageServerLocation: 'cache', + languageServerSetupStage: 'getServer', + result: 'Succeeded', + }, + ] + + /* Third Try Telemetry + getManifest: (stubbed to fail, no telemetry) + getServer: remote and cache fail + validate: no validation since not remote. + */ + const thirdTryTelemetry: Partial[] = [ + { + id: 'AmazonQ', + languageServerLocation: 'cache', + languageServerSetupStage: 'getServer', + result: 'Failed', + }, + { + id: 'AmazonQ', + languageServerLocation: 'remote', + languageServerSetupStage: 'getServer', + result: 'Failed', + }, + { + id: 'AmazonQ', + languageServerLocation: 'fallback', + languageServerSetupStage: 'getServer', + result: 'Succeeded', + }, + ] + + const expectedTelemetry = firstTryTelemetry.concat(secondTryTelemetry, thirdTryTelemetry) + + assertTelemetry('languageServer_setup', expectedTelemetry) }) }) }) diff --git a/packages/core/package.nls.json b/packages/core/package.nls.json index 5d96e40e71f..ef158f01c66 100644 --- a/packages/core/package.nls.json +++ b/packages/core/package.nls.json @@ -20,7 +20,7 @@ "AWS.configuration.description.suppressPrompts": "Prompts which ask for confirmation. Checking an item suppresses the prompt.", "AWS.configuration.enableCodeLenses": "Enable SAM hints in source code and template.yaml files", "AWS.configuration.description.resources.enabledResources": "AWS resources to display in the 'Resources' portion of the explorer.", - "AWS.configuration.description.featureDevelopment.allowRunningCodeAndTests": "Allow /dev to run code and test commands", + "AWS.configuration.description.featureDevelopment.allowRunningCodeAndTests": "Allow /dev to run code and test commands", "AWS.configuration.description.experiments": "Try experimental features and give feedback. Note that experimental features may be removed at any time.\n * `jsonResourceModification` - Enables basic create, update, and delete support for cloud resources via the JSON Resources explorer component.", "AWS.stepFunctions.asl.format.enable.desc": "Enables the default formatter used with Amazon States Language files", "AWS.stepFunctions.asl.maxItemsComputed.desc": "The maximum number of outline symbols and folding regions computed (limited for performance reasons).", diff --git a/packages/core/src/amazonq/lsp/lspController.ts b/packages/core/src/amazonq/lsp/lspController.ts index 3b38d9a17a2..f556a070775 100644 --- a/packages/core/src/amazonq/lsp/lspController.ts +++ b/packages/core/src/amazonq/lsp/lspController.ts @@ -15,6 +15,7 @@ import { isCloud9 } from '../../shared/extensionUtilities' import globals, { isWeb } from '../../shared/extensionGlobals' import { isAmazonInternalOs } from '../../shared/vscode/env' import { WorkspaceLSPResolver } from './workspaceInstaller' +import { lspSetupStage } from '../../shared' export interface Chunk { readonly filePath: string @@ -160,9 +161,7 @@ export class LspController { } setImmediate(async () => { try { - const installResult = await new WorkspaceLSPResolver().resolve() - await activateLsp(context, installResult.resourcePaths) - getLogger().info('LspController: LSP activated') + await this.setupLsp(context) void LspController.instance.buildIndex(buildIndexConfig) // log the LSP server CPU and Memory usage per 30 minutes. globals.clock.setInterval( @@ -183,4 +182,12 @@ export class LspController { } }) } + + private async setupLsp(context: vscode.ExtensionContext) { + await lspSetupStage('all', async () => { + const installResult = await new WorkspaceLSPResolver().resolve() + await lspSetupStage('launch', async () => activateLsp(context, installResult.resourcePaths)) + getLogger().info('LspController: LSP activated') + }) + } } diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index f82990bc83e..b45948308bf 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -65,6 +65,7 @@ export { TabTypeDataMap } from '../amazonq/webview/ui/tabs/constants' export * from './lsp/manifestResolver' export * from './lsp/lspResolver' export * from './lsp/types' +export * from './lsp/utils/setupStage' export * from './lsp/utils/cleanup' export { default as request } from './request' export * from './lsp/utils/platform' diff --git a/packages/core/src/shared/lsp/lspResolver.ts b/packages/core/src/shared/lsp/lspResolver.ts index 5a907b96a02..265de137ed9 100644 --- a/packages/core/src/shared/lsp/lspResolver.ts +++ b/packages/core/src/shared/lsp/lspResolver.ts @@ -12,6 +12,7 @@ import AdmZip from 'adm-zip' import { TargetContent, logger, LspResult, LspVersion, Manifest } from './types' import { getApplicationSupportFolder } from '../vscode/env' import { createHash } from '../crypto' +import { lspSetupStage, StageResolver, tryStageResolvers } from './utils/setupStage' import { HttpResourceFetcher } from '../resourcefetcher/httpResourceFetcher' export class LanguageServerResolver { @@ -30,51 +31,40 @@ export class LanguageServerResolver { * @throws ToolkitError if no compatible version can be found */ async resolve() { - const result: LspResult = { - location: 'unknown', - version: '', - assetDirectory: '', - } - const latestVersion = this.latestCompatibleLspVersion() const targetContents = this.getLSPTargetContents(latestVersion) const cacheDirectory = this.getDownloadDirectory(latestVersion.serverVersion) - if (await this.hasValidLocalCache(cacheDirectory, targetContents)) { - result.location = 'cache' - result.version = latestVersion.serverVersion - result.assetDirectory = cacheDirectory - return result - } else { - // Delete the cached directory since it's invalid - if (await fs.existsDir(cacheDirectory)) { - await fs.delete(cacheDirectory, { - recursive: true, - }) - } - } - - if (await this.downloadRemoteTargetContent(targetContents, latestVersion.serverVersion)) { - result.location = 'remote' - result.version = latestVersion.serverVersion - result.assetDirectory = cacheDirectory - return result - } else { - // clean up any leftover content that may have been downloaded - if (await fs.existsDir(cacheDirectory)) { - await fs.delete(cacheDirectory, { - recursive: true, - }) + const serverResolvers: StageResolver[] = [ + { + resolve: async () => await this.getLocalServer(cacheDirectory, latestVersion, targetContents), + telemetryMetadata: { id: this.lsName, languageServerLocation: 'cache' }, + }, + { + resolve: async () => await this.fetchRemoteServer(cacheDirectory, latestVersion, targetContents), + telemetryMetadata: { id: this.lsName, languageServerLocation: 'remote' }, + }, + { + resolve: async () => await this.getFallbackServer(latestVersion), + telemetryMetadata: { id: this.lsName, languageServerLocation: 'fallback' }, + }, + ] + + return await tryStageResolvers('getServer', serverResolvers, getServerVersion) + + function getServerVersion(result: LspResult) { + return { + languageServerVersion: result.version, } } + } - logger.info( - `Unable to download language server version ${latestVersion.serverVersion}. Attempting to fetch from fallback location` - ) - + private async getFallbackServer(latestVersion: LspVersion): Promise { const fallbackDirectory = await this.getFallbackDir(latestVersion.serverVersion) if (!fallbackDirectory) { - throw new ToolkitError('Unable to find a compatible version of the Language Server') + throw new ToolkitError('Unable to find a compatible version of the Language Server', { + code: 'IncompatibleVersion', + }) } const version = path.basename(fallbackDirectory) @@ -82,11 +72,49 @@ export class LanguageServerResolver { `Unable to install ${this.lsName} language server v${latestVersion.serverVersion}. Launching a previous version from ${fallbackDirectory}` ) - result.location = 'fallback' - result.version = version - result.assetDirectory = fallbackDirectory + return { + location: 'fallback', + version: version, + assetDirectory: fallbackDirectory, + } + } + + private async fetchRemoteServer( + cacheDirectory: string, + latestVersion: LspVersion, + targetContents: TargetContent[] + ): Promise { + if (await this.downloadRemoteTargetContent(targetContents, latestVersion.serverVersion)) { + return { + location: 'remote', + version: latestVersion.serverVersion, + assetDirectory: cacheDirectory, + } + } else { + throw new ToolkitError('Failed to download server from remote', { code: 'RemoteDownloadFailed' }) + } + } - return result + private async getLocalServer( + cacheDirectory: string, + latestVersion: LspVersion, + targetContents: TargetContent[] + ): Promise { + if (await this.hasValidLocalCache(cacheDirectory, targetContents)) { + return { + location: 'cache', + version: latestVersion.serverVersion, + assetDirectory: cacheDirectory, + } + } else { + // Delete the cached directory since it's invalid + if (await fs.existsDir(cacheDirectory)) { + await fs.delete(cacheDirectory, { + recursive: true, + }) + } + throw new ToolkitError('Failed to retrieve server from cache', { code: 'InvalidCache' }) + } } /** @@ -164,25 +192,37 @@ export class LanguageServerResolver { await fs.mkdir(downloadDirectory) } - const downloadTasks = contents.map(async (content) => { - const res = await new HttpResourceFetcher(content.url, { showUrl: true }).get() - if (!res || !res.ok || !res.body) { - return false + const fetchTasks = contents.map(async (content) => { + return { + res: await new HttpResourceFetcher(content.url, { showUrl: true }).get(), + hash: content.hashes[0], + filename: content.filename, } + }) + const fetchResults = await Promise.all(fetchTasks) + const verifyTasks = fetchResults + .filter((fetchResult) => fetchResult.res && fetchResult.res.ok && fetchResult.res.body) + .flatMap(async (fetchResult) => { + const arrBuffer = await fetchResult.res!.arrayBuffer() + const data = Buffer.from(arrBuffer) + + const hash = createHash('sha384', data) + if (hash === fetchResult.hash) { + return [{ filename: fetchResult.filename, data }] + } + return [] + }) + if (verifyTasks.length !== contents.length) { + return false + } - const arrBuffer = await res.arrayBuffer() - const data = Buffer.from(arrBuffer) + const filesToDownload = await lspSetupStage('validate', async () => (await Promise.all(verifyTasks)).flat()) - const hash = createHash('sha384', data) - if (hash === content.hashes[0]) { - await fs.writeFile(`${downloadDirectory}/${content.filename}`, data) - return true - } - return false - }) - const downloadResults = await Promise.all(downloadTasks) - const downloadResult = downloadResults.every(Boolean) - return downloadResult && this.extractZipFilesFromRemote(downloadDirectory) + for (const file of filesToDownload) { + await fs.writeFile(`${downloadDirectory}/${file.filename}`, file.data) + } + + return this.extractZipFilesFromRemote(downloadDirectory) } private async extractZipFilesFromRemote(downloadDirectory: string) { @@ -333,7 +373,9 @@ export class LanguageServerResolver { private getCompatibleLspTarget(version: LspVersion) { // TODO make this web friendly // TODO make this fully support windows - const platform = process.platform + + // Workaround: Manifest platform field is `windows`, whereas node returns win32 + const platform = process.platform === 'win32' ? 'windows' : process.platform const arch = process.arch return version.targets.find((x) => x.arch === arch && x.platform === platform) } diff --git a/packages/core/src/shared/lsp/manifestResolver.ts b/packages/core/src/shared/lsp/manifestResolver.ts index 0cf27b1293b..e19dcb0ced1 100644 --- a/packages/core/src/shared/lsp/manifestResolver.ts +++ b/packages/core/src/shared/lsp/manifestResolver.ts @@ -8,6 +8,7 @@ import { ToolkitError } from '../errors' import { Timeout } from '../utilities/timeoutUtils' import globals from '../extensionGlobals' import { Manifest } from './types' +import { StageResolver, tryStageResolvers } from './utils/setupStage' import { HttpResourceFetcher } from '../resourcefetcher/httpResourceFetcher' const logger = getLogger('lsp') @@ -19,7 +20,7 @@ interface StorageManifest { type ManifestStorage = Record -const manifestStorageKey = 'aws.toolkit.lsp.manifest' +export const manifestStorageKey = 'aws.toolkit.lsp.manifest' const manifestTimeoutMs = 15000 export class ManifestResolver { @@ -32,10 +33,23 @@ export class ManifestResolver { * Fetches the latest manifest, falling back to local cache on failure */ async resolve(): Promise { - try { - return await this.fetchRemoteManifest() - } catch (error) { - return await this.getLocalManifest() + const resolvers: StageResolver[] = [ + { + resolve: async () => await this.fetchRemoteManifest(), + telemetryMetadata: { id: this.lsName, manifestLocation: 'remote' }, + }, + { + resolve: async () => await this.getLocalManifest(), + telemetryMetadata: { id: this.lsName, manifestLocation: 'cache' }, + }, + ] + + return await tryStageResolvers('getManifest', resolvers, extractMetadata) + + function extractMetadata(r: Manifest) { + return { + manifestSchemaVersion: r.manifestSchemaVersion, + } } } @@ -52,7 +66,7 @@ export class ManifestResolver { const manifest = this.parseManifest(resp.content) await this.saveManifest(resp.eTag, resp.content) this.checkDeprecation(manifest) - + manifest.location = 'remote' return manifest } @@ -67,6 +81,7 @@ export class ManifestResolver { const manifest = this.parseManifest(manifestData.content) this.checkDeprecation(manifest) + manifest.location = 'cache' return manifest } diff --git a/packages/core/src/shared/lsp/types.ts b/packages/core/src/shared/lsp/types.ts index 4f5a3cf1c87..ef7518db69a 100644 --- a/packages/core/src/shared/lsp/types.ts +++ b/packages/core/src/shared/lsp/types.ts @@ -4,13 +4,12 @@ */ import { getLogger } from '../logger/logger' +import { LanguageServerLocation, ManifestLocation } from '../telemetry' export const logger = getLogger('lsp') -type Location = 'remote' | 'cache' | 'override' | 'fallback' | 'unknown' - export interface LspResult { - location: Location + location: LanguageServerLocation version: string assetDirectory: string } @@ -53,6 +52,7 @@ export interface Manifest { artifactDescription: string isManifestDeprecated: boolean versions: LspVersion[] + location?: ManifestLocation } export interface VersionRange { diff --git a/packages/core/src/shared/lsp/utils/setupStage.ts b/packages/core/src/shared/lsp/utils/setupStage.ts new file mode 100644 index 00000000000..f8932f0b2ef --- /dev/null +++ b/packages/core/src/shared/lsp/utils/setupStage.ts @@ -0,0 +1,66 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import { LanguageServerSetup, LanguageServerSetupStage, telemetry } from '../../telemetry' +import { tryFunctions } from '../../utilities/tsUtils' + +/** + * Runs the designated stage within a telemetry span and optionally uses the getMetadata extractor to record metadata from the result of the stage. + * @param stageName name of stage for telemetry. + * @param runStage stage to be run. + * @param getMetadata metadata extractor to be applied to result. + * @returns result of stage + */ +export async function lspSetupStage( + stageName: LanguageServerSetupStage, + runStage: () => Promise, + getMetadata?: MetadataExtractor +) { + return await telemetry.languageServer_setup.run(async (span) => { + span.record({ languageServerSetupStage: stageName }) + const result = await runStage() + if (getMetadata) { + span.record(getMetadata(result)) + } + return result + }) +} +/** + * Tries to resolve the result of a stage using the resolvers provided in order. The first one to succceed + * has its result returned, but all intermediate will emit telemetry. + * @param stageName name of stage to resolve. + * @param resolvers stage resolvers to try IN ORDER + * @param getMetadata function to be applied to result to extract necessary metadata for telemetry. + * @returns result of the first succesful resolver. + */ +export async function tryStageResolvers( + stageName: LanguageServerSetupStage, + resolvers: StageResolver[], + getMetadata: MetadataExtractor +) { + const fs = resolvers.map((resolver) => async () => { + return await lspSetupStage( + stageName, + async () => { + telemetry.record(resolver.telemetryMetadata) + const result = await resolver.resolve() + return result + }, + getMetadata + ) + }) + + return await tryFunctions(fs) +} + +/** + * A method that returns the result of a stage along with the default telemetry metadata to attach to the stage metric. + */ +export interface StageResolver { + resolve: () => Promise + telemetryMetadata: Partial +} + +type MetadataExtractor = (r: R) => Partial diff --git a/packages/core/src/shared/utilities/tsUtils.ts b/packages/core/src/shared/utilities/tsUtils.ts index 2e4838c5b7a..09a9b276280 100644 --- a/packages/core/src/shared/utilities/tsUtils.ts +++ b/packages/core/src/shared/utilities/tsUtils.ts @@ -94,6 +94,23 @@ export function createFactoryFunction any>(cto return (...args) => new ctor(...args) } +/** + * Try functions in the order presented and return the first returned result. If none return, throw the final error. + * @param functions non-empty list of functions to try. + * @returns + */ +export async function tryFunctions(functions: (() => Promise)[]): Promise { + let currentError: Error = new Error('No functions provided') + for (const func of functions) { + try { + return await func() + } catch (e) { + currentError = e as Error + } + } + throw currentError +} + /** * Split a list into two sublists based on the result of a predicate. * @param lst list to split diff --git a/packages/core/src/test/shared/lsp/manifestResolver.test.ts b/packages/core/src/test/shared/lsp/manifestResolver.test.ts new file mode 100644 index 00000000000..eb5f8e90893 --- /dev/null +++ b/packages/core/src/test/shared/lsp/manifestResolver.test.ts @@ -0,0 +1,101 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import sinon from 'sinon' +import { Manifest, ManifestResolver } from '../../../shared' +import { assertTelemetry } from '../../testUtil' +import { ManifestLocation } from '../../../shared/telemetry' + +const manifestSchemaVersion = '1.0.0' +const serverName = 'myLS' + +/** + * Helper function generating valid manifest results for tests. + * @param location + * @returns + */ +function manifestResult(location: ManifestLocation): Manifest { + return { + location, + manifestSchemaVersion, + artifactId: 'artifact-id', + artifactDescription: 'artifact-description', + isManifestDeprecated: false, + versions: [], + } +} + +describe('manifestResolver', function () { + let remoteStub: sinon.SinonStub + let localStub: sinon.SinonStub + + before(function () { + remoteStub = sinon.stub(ManifestResolver.prototype, 'fetchRemoteManifest' as any) + localStub = sinon.stub(ManifestResolver.prototype, 'getLocalManifest' as any) + }) + + after(function () { + sinon.restore() + }) + + it('attempts to fetch from remote first', async function () { + remoteStub.resolves(manifestResult('remote')) + + const r = await new ManifestResolver('remote-manifest.com', serverName).resolve() + assert.strictEqual(r.location, 'remote') + assertTelemetry('languageServer_setup', { + manifestLocation: 'remote', + manifestSchemaVersion, + languageServerSetupStage: 'getManifest', + id: serverName, + result: 'Succeeded', + }) + }) + + it('uses local cache when remote fails', async function () { + remoteStub.rejects(new Error('failed to fetch')) + localStub.resolves(manifestResult('cache')) + + const r = await new ManifestResolver('remote-manifest.com', serverName).resolve() + assert.strictEqual(r.location, 'cache') + assertTelemetry('languageServer_setup', [ + { + manifestLocation: 'remote', + languageServerSetupStage: 'getManifest', + id: serverName, + result: 'Failed', + }, + { + manifestLocation: 'cache', + manifestSchemaVersion, + languageServerSetupStage: 'getManifest', + id: serverName, + result: 'Succeeded', + }, + ]) + }) + + it('fails if both local and remote fail', async function () { + remoteStub.rejects(new Error('failed to fetch')) + localStub.rejects(new Error('failed to fetch')) + + await assert.rejects(new ManifestResolver('remote-manifest.com', serverName).resolve(), /failed to fetch/) + assertTelemetry('languageServer_setup', [ + { + manifestLocation: 'remote', + languageServerSetupStage: 'getManifest', + id: serverName, + result: 'Failed', + }, + { + manifestLocation: 'cache', + languageServerSetupStage: 'getManifest', + id: serverName, + result: 'Failed', + }, + ]) + }) +}) diff --git a/packages/core/src/test/shared/utilities/tsUtils.test.ts b/packages/core/src/test/shared/utilities/tsUtils.test.ts index eb04da035e5..384d47df6ec 100644 --- a/packages/core/src/test/shared/utilities/tsUtils.test.ts +++ b/packages/core/src/test/shared/utilities/tsUtils.test.ts @@ -2,9 +2,30 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ - -import { partition } from '../../../shared/utilities/tsUtils' import assert from 'assert' +import { tryFunctions } from '../../../shared/utilities/tsUtils' +import { partition } from '../../../shared/utilities/tsUtils' + +describe('tryFunctions', function () { + it('should return the result of the first function that returns', async function () { + const f1 = () => Promise.reject('f1') + const f2 = () => Promise.resolve('f2') + const f3 = () => Promise.reject('f3') + + assert.strictEqual(await tryFunctions([f1, f2, f3]), 'f2') + }) + + it('if all reject, then should throw final error', async function () { + const f1 = () => Promise.reject('f1') + const f2 = () => Promise.reject('f2') + const f3 = () => Promise.reject('f3') + + await assert.rejects( + async () => await tryFunctions([f1, f2, f3]), + (e) => e === 'f3' + ) + }) +}) describe('partition', function () { it('should split the list according to predicate', function () { From 91adc3fb42620c5eae8398d6798efc2609edaef1 Mon Sep 17 00:00:00 2001 From: Josh Pinkney <103940141+jpinkney-aws@users.noreply.github.com> Date: Fri, 7 Feb 2025 11:43:12 -0500 Subject: [PATCH 22/22] deps(amazonq): Update amazon q language server to 3.x.x (#6528) ## Problem Amazon q currently uses the 2.3.2 version of the language server which doesn't appear to be working ## Solution Update to the 3.x.x series for now Independently i'm going to follow up with teams to understand how this happened. Had we released this feature to prod amazon q inline would have been completely broken --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license. --- packages/amazonq/src/lsp/lspInstaller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/amazonq/src/lsp/lspInstaller.ts b/packages/amazonq/src/lsp/lspInstaller.ts index 72d0746cdcf..f2bc26d2a22 100644 --- a/packages/amazonq/src/lsp/lspInstaller.ts +++ b/packages/amazonq/src/lsp/lspInstaller.ts @@ -17,7 +17,7 @@ import { import path from 'path' const manifestURL = 'https://aws-toolkit-language-servers.amazonaws.com/codewhisperer/0/manifest.json' -export const supportedLspServerVersions = '^2.3.0' +export const supportedLspServerVersions = '^3.1.1' export class AmazonQLSPResolver implements LspResolver { async resolve(): Promise {