From f37e9558d35aa1cbb207d0e6f8fe4fb00c669338 Mon Sep 17 00:00:00 2001 From: florianbgt Date: Fri, 1 Nov 2024 13:47:52 +0100 Subject: [PATCH] better handling of file editing --- README.md | 3 +- packages/cli/src/helper/file.ts | 18 +- .../helper/languages/javascript/cleanup.ts | 334 ++++++++++-------- packages/cli/src/helper/tree.ts | 16 +- packages/cli/src/telemetry.ts | 19 +- 5 files changed, 213 insertions(+), 177 deletions(-) diff --git a/README.md b/README.md index a6e7ee1..665cc8c 100644 --- a/README.md +++ b/README.md @@ -164,9 +164,8 @@ We welcome contributions from the community. Please read our [contributing guide This project is in the early stages of development. We are actively working on the project and will be releasing new features and improvements regularly, which may include a rewrite into a more efficient and generic language like Rust or Go. Please check our issues and project board for more information, and don't for. -- [x] Limited support for NodeJS/Typescript (no require or dynamic imports, no decorators) +- [x] Limited support for NodeJS/Typescript - [x] Simple UI -- [x] Support NodeJS require and dynamic imports - [ ] Full support for NodeJS/Typescript - [ ] Python support with Flask - [ ] Django support diff --git a/packages/cli/src/helper/file.ts b/packages/cli/src/helper/file.ts index 4e76df7..fa7d575 100644 --- a/packages/cli/src/helper/file.ts +++ b/packages/cli/src/helper/file.ts @@ -129,7 +129,6 @@ function getFilePathsFromTree(tree: dependencyTree.Tree) { return uniqueFilePaths; } -// Resolve file paths from import/require statements export function resolveFilePath( importPath: string, currentFile: string, @@ -169,3 +168,20 @@ export function resolveFilePath( // Skip external dependencies (e.g., node_modules) return null; } + +export function removeIndexesFromSourceCode( + sourceCode: string, + indexesToRemove: { startIndex: number; endIndex: number }[], +) { + let newSourceCode = sourceCode; + + // sort to start removing from the of the file end + indexesToRemove.sort((a, b) => b.startIndex - a.startIndex); + + indexesToRemove.forEach(({ startIndex, endIndex }) => { + newSourceCode = + newSourceCode.slice(0, startIndex) + newSourceCode.slice(endIndex); + }); + + return newSourceCode; +} diff --git a/packages/cli/src/helper/languages/javascript/cleanup.ts b/packages/cli/src/helper/languages/javascript/cleanup.ts index 152a6fe..0ad1ad7 100644 --- a/packages/cli/src/helper/languages/javascript/cleanup.ts +++ b/packages/cli/src/helper/languages/javascript/cleanup.ts @@ -1,6 +1,9 @@ import Parser from "tree-sitter"; import { Group } from "../../types"; -import { getNanoApiAnnotationFromCommentValue } from "../../file"; +import { + getNanoApiAnnotationFromCommentValue, + removeIndexesFromSourceCode, +} from "../../file"; import { getImportStatements, extractFileImportsFromImportStatements, @@ -14,11 +17,13 @@ import { } from "./imports"; function removeAnnotations( - rootNode: Parser.SyntaxNode, + parser: Parser, sourceCode: string, groupToKeep: Group, ): string { - let updatedSourceCode = sourceCode; + const tree = parser.parse(sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; function traverse(node: Parser.SyntaxNode) { if (node.type === "comment") { @@ -34,21 +39,20 @@ function removeAnnotations( ); if (!endpointToKeep) { - const nextNode = node.nextNamedSibling; - // TODO test this piece of code with decorators on a nestjs project - // // We need to remove all decorators too - // while (nextNode && nextNode.type === "decorator") { - // nextNode = nextNode.nextNamedSibling; - // } + let nextNode = node.nextNamedSibling; + // We need to remove all decorators too + while (nextNode && nextNode.type === "decorator") { + nextNode = nextNode.nextNamedSibling; + } if (!nextNode) { throw new Error("Could not find next node"); } - // delete this node (comment) and the next node(s) (api endpoint) - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring(node.startIndex, nextNode.endIndex + 1), - "", - ); + // Remove this node (comment) and the next node(s) (api endpoint) + indexesToRemove.push({ + startIndex: node.startIndex, + endIndex: nextNode.endIndex, + }); } } @@ -57,20 +61,28 @@ function removeAnnotations( }); } - traverse(rootNode); + traverse(tree.rootNode); + + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); return updatedSourceCode; } function removeInvalidFileImports( - rootNode: Parser.SyntaxNode, + parser: Parser, sourceCode: string, invalidDependencies: string[], ) { - let updatedSourceCode = sourceCode; + const tree = parser.parse(sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + const removedIdentifiers: Parser.SyntaxNode[] = []; - const importStatements = getImportStatements(rootNode); + const importStatements = getImportStatements(tree.rootNode); importStatements.forEach((importStatement) => { const importName = extractFileImportsFromImportStatements(importStatement); @@ -80,28 +92,33 @@ function removeInvalidFileImports( removedIdentifiers.push(...identifiers); // Remove the import statement - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - importStatement.startIndex, - importStatement.endIndex, - ), - "", - ); + indexesToRemove.push({ + startIndex: importStatement.startIndex, + endIndex: importStatement.endIndex, + }); } }); + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + return { updatedSourceCode, removedIdentifiers }; } function removeInvalidFileRequires( - rootNode: Parser.SyntaxNode, + parser: Parser, sourceCode: string, invalidDependencies: string[], ) { - let updatedSourceCode = sourceCode; + const tree = parser.parse(sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + const removedIdentifiers: Parser.SyntaxNode[] = []; - const requireStatements = getRequireDeclarations(rootNode); + const requireStatements = getRequireDeclarations(tree.rootNode); requireStatements.forEach((requireStatement) => { const importName = extractFileImportsFromRequireDeclarations(requireStatement); @@ -111,28 +128,33 @@ function removeInvalidFileRequires( removedIdentifiers.push(...identifiers); // Remove the require statement - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - requireStatement.startIndex, - requireStatement.endIndex, - ), - "", - ); + indexesToRemove.push({ + startIndex: requireStatement.startIndex, + endIndex: requireStatement.endIndex, + }); } }); + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + return { updatedSourceCode, removedIdentifiers }; } function removeInvalidFileDynamicImports( - rootNode: Parser.SyntaxNode, + parser: Parser, sourceCode: string, invalidDependencies: string[], ) { - let updatedSourceCode = sourceCode; + const tree = parser.parse(sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + const removedIdentifiers: Parser.SyntaxNode[] = []; - const dynamicImportStatements = getDynamicImportDeclarations(rootNode); + const dynamicImportStatements = getDynamicImportDeclarations(tree.rootNode); dynamicImportStatements.forEach((dynamicImportStatement) => { const importName = extractFileImportsFromDynamicImportDeclarations( dynamicImportStatement, @@ -144,43 +166,61 @@ function removeInvalidFileDynamicImports( removedIdentifiers.push(...identifiers); // Remove the require statement - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - dynamicImportStatement.startIndex, - dynamicImportStatement.endIndex, - ), - "", - ); + indexesToRemove.push({ + startIndex: dynamicImportStatement.startIndex, + endIndex: dynamicImportStatement.endIndex, + }); } }); + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + return { updatedSourceCode, removedIdentifiers }; } function removeDeletedImportUsage( - rootNode: Parser.SyntaxNode, + parser: Parser, sourceCode: string, removedImportsNames: Parser.SyntaxNode[], ): string { - let updatedSourceCode = sourceCode; + const tree = parser.parse(sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; removedImportsNames.forEach((removedImportsName) => { function traverse(node: Parser.SyntaxNode) { if (node.type === "identifier" && removedImportsName.text === node.text) { // find the next parent expression statement (can be several levels up) let parent = node.parent; - while (parent && parent.type !== "expression_statement") { + + while (parent) { + if (parent && parent.type === "array") { + // remove iditenfier from the array + const endIndex = + sourceCode.substring(node.endIndex, node.endIndex) === "," + ? node.endIndex + : node.endIndex; + indexesToRemove.push({ + startIndex: node.startIndex, + endIndex, + }); + return; + } + + if (parent && parent.type === "expression_statement") { + // Remove the expression statement + indexesToRemove.push({ + startIndex: parent.startIndex, + endIndex: parent.endIndex, + }); + return; + } + parent = parent.parent; } - if (!parent) { - throw new Error("Could not find parent expression statement"); - } - - // Remove the expression statement - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring(parent.startIndex, parent.endIndex), - "", - ); } node.children.forEach((child) => { @@ -188,9 +228,14 @@ function removeDeletedImportUsage( }); } - traverse(rootNode); + traverse(tree.rootNode); }); + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + return updatedSourceCode; } @@ -219,13 +264,12 @@ function isIdentifierUsed( return isUsed; } -function cleanUnusedImportsStatements( - rootNode: Parser.SyntaxNode, - sourceCode: string, -) { - let updatedSourceCode = sourceCode; +function cleanUnusedImportsStatements(parser: Parser, sourceCode: string) { + const tree = parser.parse(sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; - const importStatements = getImportStatements(rootNode); + const importStatements = getImportStatements(tree.rootNode); importStatements.forEach((importStatement) => { const importIdentifiers = @@ -234,7 +278,7 @@ function cleanUnusedImportsStatements( const importIdentifiersToRemove: Parser.SyntaxNode[] = []; importIdentifiers.forEach((importIdentifier) => { - const isUsed = isIdentifierUsed(rootNode, importIdentifier); + const isUsed = isIdentifierUsed(tree.rootNode, importIdentifier); if (!isUsed) { importIdentifiersToRemove.push(importIdentifier); } @@ -244,36 +288,34 @@ function cleanUnusedImportsStatements( importIdentifiersToRemove.length === importIdentifiers.length; if (removeImportStatement) { - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - importStatement.startIndex, - importStatement.endIndex + 1, - ), - "", - ); + indexesToRemove.push({ + startIndex: importStatement.startIndex, + endIndex: importStatement.endIndex, + }); + } else { + importIdentifiersToRemove.forEach((importIdentifier) => { + indexesToRemove.push({ + startIndex: importIdentifier.startIndex, + endIndex: importIdentifier.endIndex, + }); + }); } - - importIdentifiersToRemove.forEach((importIdentifier) => { - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - importIdentifier.startIndex, - importIdentifier.endIndex + 1, - ), - "", - ); - }); }); + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + return updatedSourceCode; } -function cleanUnusedRequireDeclarations( - rootNode: Parser.SyntaxNode, - sourceCode: string, -) { - let updatedSourceCode = sourceCode; +function cleanUnusedRequireDeclarations(parser: Parser, sourceCode: string) { + const tree = parser.parse(sourceCode); - const requireDeclarations = getRequireDeclarations(rootNode); + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + + const requireDeclarations = getRequireDeclarations(tree.rootNode); requireDeclarations.forEach((requireDeclaration) => { const requireIdentifiers = @@ -282,7 +324,7 @@ function cleanUnusedRequireDeclarations( const requireIdentifiersToRemove: Parser.SyntaxNode[] = []; requireIdentifiers.forEach((requireIdentifier) => { - const isUsed = isIdentifierUsed(rootNode, requireIdentifier); + const isUsed = isIdentifierUsed(tree.rootNode, requireIdentifier); if (!isUsed) { requireIdentifiersToRemove.push(requireIdentifier); } @@ -292,36 +334,37 @@ function cleanUnusedRequireDeclarations( requireIdentifiersToRemove.length === requireIdentifiers.length; if (removeRequireDeclaration) { - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - requireDeclaration.startIndex, - requireDeclaration.endIndex + 1, - ), - "", - ); + indexesToRemove.push({ + startIndex: requireDeclaration.startIndex, + endIndex: requireDeclaration.endIndex, + }); + } else { + requireIdentifiersToRemove.forEach((requireIdentifier) => { + indexesToRemove.push({ + startIndex: requireIdentifier.startIndex, + endIndex: requireIdentifier.endIndex, + }); + }); } - - requireIdentifiersToRemove.forEach((requireIdentifier) => { - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - requireIdentifier.startIndex, - requireIdentifier.endIndex + 1, - ), - "", - ); - }); }); + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + return updatedSourceCode; } function cleanUnusedDynamicImportDeclarations( - rootNode: Parser.SyntaxNode, + parser: Parser, sourceCode: string, ) { - let updatedSourceCode = sourceCode; + const tree = parser.parse(sourceCode); - const dynamicImportDeclarations = getDynamicImportDeclarations(rootNode); + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + + const dynamicImportDeclarations = getDynamicImportDeclarations(tree.rootNode); dynamicImportDeclarations.forEach((dynamicImportDeclaration) => { const dynamicImportIdentifiers = @@ -330,7 +373,7 @@ function cleanUnusedDynamicImportDeclarations( const dynamicImportIdentifiersToRemove: Parser.SyntaxNode[] = []; dynamicImportIdentifiers.forEach((dynamicImportIdentifier) => { - const isUsed = isIdentifierUsed(rootNode, dynamicImportIdentifier); + const isUsed = isIdentifierUsed(tree.rootNode, dynamicImportIdentifier); if (!isUsed) { dynamicImportIdentifiersToRemove.push(dynamicImportIdentifier); } @@ -341,26 +384,25 @@ function cleanUnusedDynamicImportDeclarations( dynamicImportIdentifiers.length; if (removeDynamicImportDeclaration) { - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - dynamicImportDeclaration.startIndex, - dynamicImportDeclaration.endIndex + 1, - ), - "", - ); + indexesToRemove.push({ + startIndex: dynamicImportDeclaration.startIndex, + endIndex: dynamicImportDeclaration.endIndex, + }); + } else { + dynamicImportIdentifiersToRemove.forEach((dynamicImportIdentifier) => { + indexesToRemove.push({ + startIndex: dynamicImportIdentifier.startIndex, + endIndex: dynamicImportIdentifier.endIndex, + }); + }); } - - dynamicImportIdentifiersToRemove.forEach((dynamicImportIdentifier) => { - updatedSourceCode = updatedSourceCode.replace( - sourceCode.substring( - dynamicImportIdentifier.startIndex, - dynamicImportIdentifier.endIndex + 1, - ), - "", - ); - }); }); + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + return updatedSourceCode; } @@ -370,34 +412,26 @@ export function cleanupJavascriptFile( group: Group, invalidDependencies: string[], ): string { - let tree = parser.parse(sourceCode); - - let updatedSourceCode = removeAnnotations(tree.rootNode, sourceCode, group); - - tree = parser.parse(updatedSourceCode); + let updatedSourceCode = removeAnnotations(parser, sourceCode, group); const resultAfterImportCleaning = removeInvalidFileImports( - tree.rootNode, + parser, updatedSourceCode, invalidDependencies, ); updatedSourceCode = resultAfterImportCleaning.updatedSourceCode; const removedIdentifiers = resultAfterImportCleaning.removedIdentifiers; - tree = parser.parse(updatedSourceCode); - const resultAfterRequireCleaning = removeInvalidFileRequires( - tree.rootNode, + parser, updatedSourceCode, invalidDependencies, ); updatedSourceCode = resultAfterRequireCleaning.updatedSourceCode; removedIdentifiers.push(...resultAfterRequireCleaning.removedIdentifiers); - tree = parser.parse(updatedSourceCode); - const resultAfterDynamicImportCleaning = removeInvalidFileDynamicImports( - tree.rootNode, + parser, updatedSourceCode, invalidDependencies, ); @@ -406,32 +440,18 @@ export function cleanupJavascriptFile( ...resultAfterDynamicImportCleaning.removedIdentifiers, ); - tree = parser.parse(updatedSourceCode); - updatedSourceCode = removeDeletedImportUsage( - tree.rootNode, + parser, updatedSourceCode, removedIdentifiers, ); - tree = parser.parse(updatedSourceCode); - - updatedSourceCode = cleanUnusedImportsStatements( - tree.rootNode, - updatedSourceCode, - ); - - tree = parser.parse(updatedSourceCode); - - updatedSourceCode = cleanUnusedRequireDeclarations( - tree.rootNode, - updatedSourceCode, - ); + updatedSourceCode = cleanUnusedImportsStatements(parser, updatedSourceCode); - tree = parser.parse(updatedSourceCode); + updatedSourceCode = cleanUnusedRequireDeclarations(parser, updatedSourceCode); updatedSourceCode = cleanUnusedDynamicImportDeclarations( - tree.rootNode, + parser, updatedSourceCode, ); diff --git a/packages/cli/src/helper/tree.ts b/packages/cli/src/helper/tree.ts index 515b08e..21f5485 100644 --- a/packages/cli/src/helper/tree.ts +++ b/packages/cli/src/helper/tree.ts @@ -11,15 +11,15 @@ export function getEndpontsFromTree( endpoints: Endpoint[] = [], ) { for (const [filePath, value] of Object.entries(tree)) { - const annotations = getEndpointsFromFile(parentFiles, filePath, tree); - for (const annotation of annotations) { + const endpointsFromFile = getEndpointsFromFile(parentFiles, filePath, tree); + for (const endpointFromFile of endpointsFromFile) { const endpoint = { - method: annotation.method, - path: annotation.path, - group: annotation.group, - filePath: annotation.filePath, - parentFilePaths: annotation.parentFilePaths, - childrenFilePaths: annotation.childrenFilePaths, + method: endpointFromFile.method, + path: endpointFromFile.path, + group: endpointFromFile.group, + filePath: endpointFromFile.filePath, + parentFilePaths: endpointFromFile.parentFilePaths, + childrenFilePaths: endpointFromFile.childrenFilePaths, } as Endpoint; endpoints.push(endpoint); } diff --git a/packages/cli/src/telemetry.ts b/packages/cli/src/telemetry.ts index 3200d26..c5a137e 100644 --- a/packages/cli/src/telemetry.ts +++ b/packages/cli/src/telemetry.ts @@ -19,8 +19,7 @@ export enum TelemetryEvents { export interface TelemetryEvent { sessionId: string; eventId: TelemetryEvents; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - data: any; + data: Record; timestamp: string; } @@ -31,7 +30,7 @@ const TELEMETRY_ENDPOINT = const SESSION_FILE_PATH = join("/tmp", "napi_session_id"); // getSessionId generates a new session ID and cache it in SESSION_FILE_PATH -const getSessionId = (): string => { +function getSessionId() { if (existsSync(SESSION_FILE_PATH)) { const fileContent = readFileSync(SESSION_FILE_PATH, "utf-8"); const [storedDate, sessionId] = fileContent.split(":"); @@ -46,7 +45,7 @@ const getSessionId = (): string => { const today = new Date().toISOString().slice(0, 10); writeFileSync(SESSION_FILE_PATH, `${today}:${newSessionId}`); return newSessionId; -}; +} const SESSION_ID = getSessionId(); @@ -54,7 +53,7 @@ telemetry.on("event", (data) => { sendTelemetryData(data); }); -const sendTelemetryData = async (data: TelemetryEvent) => { +async function sendTelemetryData(data: TelemetryEvent) { try { await axios.post(TELEMETRY_ENDPOINT, data, { headers: { @@ -66,10 +65,12 @@ const sendTelemetryData = async (data: TelemetryEvent) => { } catch (error) { console.error(`Failed to send telemetry data: ${error}`); } -}; +} -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export const trackEvent = (eventId: TelemetryEvents, eventData: any) => { +export function trackEvent( + eventId: TelemetryEvents, + eventData: Record, +) { const telemetryPayload: TelemetryEvent = { sessionId: SESSION_ID, eventId, @@ -78,4 +79,4 @@ export const trackEvent = (eventId: TelemetryEvents, eventData: any) => { }; telemetry.emit("event", telemetryPayload); -}; +}