From c44e270daae9f6aab4bc5f1361cc7d8005a61285 Mon Sep 17 00:00:00 2001 From: florianbgt Date: Mon, 25 Nov 2024 13:58:59 +0100 Subject: [PATCH] wip --- packages/cli/src/annotationManager.ts | 93 ++++ packages/cli/src/api/scan.ts | 3 +- packages/cli/src/api/split.ts | 10 +- packages/cli/src/api/sync.ts | 68 ++- packages/cli/src/commands/annotate.ts | 4 +- packages/cli/src/commands/split.ts | 8 +- .../dependencyManager.ts} | 105 ++-- .../{helper => dependencyManager}/types.ts | 20 +- packages/cli/src/helper/annotations.ts | 75 --- packages/cli/src/helper/cleanup.ts | 276 ----------- packages/cli/src/helper/file.ts | 32 ++ .../languages/javascript/annotations.ts | 22 - .../helper/languages/javascript/cleanup.ts | 276 ----------- .../helper/languages/javascript/exports.ts | 89 ---- .../helper/languages/javascript/imports.ts | 319 ------------ .../src/helper/languages/python/imports.ts | 57 --- .../languages/typescript/annotations.ts | 9 - .../helper/languages/typescript/cleanup.ts | 46 -- .../helper/languages/typescript/exports.ts | 6 - .../helper/languages/typescript/imports.ts | 17 - packages/cli/src/helper/split.ts | 139 ------ packages/cli/src/helper/treeSitter.ts | 18 - packages/cli/src/languages/index.ts | 18 + packages/cli/src/languages/javascript.ts | 459 ++++++++++++++++++ .../languages/javascript/annotations.test.ts | 0 .../src/languages/javascript/annotations.ts | 3 + .../languages/javascript/exports.test.ts | 0 .../languages/javascript/imports.test.ts | 0 packages/cli/src/languages/types.ts | 44 ++ packages/cli/src/languages/typescript.ts | 45 ++ .../languages/typescript/annotations.test.ts | 0 .../src/languages/typescript/annotations.ts | 8 + .../languages/typescript/exports.test.ts | 0 .../languages/typescript/imports.test.ts | 0 packages/cli/src/splitRunner/splitRunner.ts | 190 ++++++++ packages/cli/src/splitRunner/types.ts | 17 + 36 files changed, 994 insertions(+), 1482 deletions(-) create mode 100644 packages/cli/src/annotationManager.ts rename packages/cli/src/{helper/dependencyTree.ts => dependencyManager/dependencyManager.ts} (53%) rename packages/cli/src/{helper => dependencyManager}/types.ts (70%) delete mode 100644 packages/cli/src/helper/annotations.ts delete mode 100644 packages/cli/src/helper/cleanup.ts delete mode 100644 packages/cli/src/helper/languages/javascript/annotations.ts delete mode 100644 packages/cli/src/helper/languages/javascript/cleanup.ts delete mode 100644 packages/cli/src/helper/languages/javascript/exports.ts delete mode 100644 packages/cli/src/helper/languages/javascript/imports.ts delete mode 100644 packages/cli/src/helper/languages/python/imports.ts delete mode 100644 packages/cli/src/helper/languages/typescript/annotations.ts delete mode 100644 packages/cli/src/helper/languages/typescript/cleanup.ts delete mode 100644 packages/cli/src/helper/languages/typescript/exports.ts delete mode 100644 packages/cli/src/helper/languages/typescript/imports.ts delete mode 100644 packages/cli/src/helper/split.ts delete mode 100644 packages/cli/src/helper/treeSitter.ts create mode 100644 packages/cli/src/languages/index.ts create mode 100644 packages/cli/src/languages/javascript.ts rename packages/cli/src/{helper => }/languages/javascript/annotations.test.ts (100%) create mode 100644 packages/cli/src/languages/javascript/annotations.ts rename packages/cli/src/{helper => }/languages/javascript/exports.test.ts (100%) rename packages/cli/src/{helper => }/languages/javascript/imports.test.ts (100%) create mode 100644 packages/cli/src/languages/types.ts create mode 100644 packages/cli/src/languages/typescript.ts rename packages/cli/src/{helper => }/languages/typescript/annotations.test.ts (100%) create mode 100644 packages/cli/src/languages/typescript/annotations.ts rename packages/cli/src/{helper => }/languages/typescript/exports.test.ts (100%) rename packages/cli/src/{helper => }/languages/typescript/imports.test.ts (100%) create mode 100644 packages/cli/src/splitRunner/splitRunner.ts create mode 100644 packages/cli/src/splitRunner/types.ts diff --git a/packages/cli/src/annotationManager.ts b/packages/cli/src/annotationManager.ts new file mode 100644 index 0000000..c0e7454 --- /dev/null +++ b/packages/cli/src/annotationManager.ts @@ -0,0 +1,93 @@ +import { Group } from "./dependencyManager/types"; +import { LanguagePlugin } from "./languages/types"; + +class AnnotationManager { + private nanoapiRegex: RegExp; + private commentPrefix: string; + path: string; + method?: string; + group?: string; + + constructor(comment: string, languagePlugin: LanguagePlugin) { + this.nanoapiRegex = languagePlugin.annotationRegex; + this.commentPrefix = languagePlugin.commentPrefix; + const { path, method, group } = this.#parsetext(comment); + this.path = path; + this.method = method; + this.group = group; + } + + #parsetext(text: string) { + const matches = text.match(this.nanoapiRegex); + + if (!matches) { + throw new Error("Could not parse annotation"); + } + + const methodRegex = /method:([^ ]+)/; + const pathRegex = /path:([^ ]+)/; + const groupRegex = /group:([^ ]+)/; + + const pathMatches = text.match(pathRegex); + const methodMatches = text.match(methodRegex); + const groupMatches = text.match(groupRegex); + + if (!pathMatches) { + throw new Error("Could not parse path from annotation"); + } + + const path = pathMatches[1]; + const method = methodMatches ? methodMatches[1] : undefined; + const group = groupMatches ? groupMatches[1] : undefined; + + return { path, method, group }; + } + + matchesEndpoint(path: string, method: string | undefined) { + return this.path === path && this.method === method; + } + + isInGroup(group: Group) { + // check if annotation has a method (actual endpoint) + if (this.method) { + const endpoint = group.endpoints.find( + (endpoint) => + endpoint.method === this.method && endpoint.path === this.path, + ); + + if (endpoint) { + return true; + } + + return false; + } + + // if annotation has no method, we treat it as a module that contains other endpoints + const endpoints = group.endpoints.filter((endpoint) => + endpoint.path.startsWith(this.path), + ); + + if (endpoints.length > 0) { + return true; + } + + return false; + } + + stringify() { + let annotation = `${this.commentPrefix} @nanoapi`; + if (this.method) { + annotation += ` method:${this.method}`; + } + if (this.path) { + annotation += ` path:${this.path}`; + } + if (this.group) { + annotation += ` group:${this.group}`; + } + + return annotation; + } +} + +export default AnnotationManager; diff --git a/packages/cli/src/api/scan.ts b/packages/cli/src/api/scan.ts index 88bb8dd..ba5069e 100644 --- a/packages/cli/src/api/scan.ts +++ b/packages/cli/src/api/scan.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import DependencyTreeManager from "../helper/dependencyTree"; +import DependencyTreeManager from "../dependencyManager/dependencyManager"; import { scanSchema } from "./helpers/validation"; export function scan(payload: z.infer) { @@ -7,5 +7,6 @@ export function scan(payload: z.infer) { payload.entrypointPath, ); const endpoints = dependencyTreeManager.getEndponts(); + return { endpoints }; } diff --git a/packages/cli/src/api/split.ts b/packages/cli/src/api/split.ts index 3f43e12..dc90c4e 100644 --- a/packages/cli/src/api/split.ts +++ b/packages/cli/src/api/split.ts @@ -1,15 +1,15 @@ import fs from "fs"; import path from "path"; -import DependencyTreeManager from "../helper/dependencyTree"; +import DependencyTreeManager from "../dependencyManager/dependencyManager"; import { cleanupOutputDir, createOutputDir } from "../helper/file"; -import SplitRunner from "../helper/split"; +import SplitRunner from "../splitRunner/splitRunner"; import { splitSchema } from "./helpers/validation"; import { z } from "zod"; -import { GroupMap } from "../helper/types"; +import { Group } from "../dependencyManager/types"; export function split(payload: z.infer) { console.time("split command"); - const groupMap: GroupMap = {}; + const groupMap: Record = {}; // Get the dependency tree const dependencyTreeManager = new DependencyTreeManager( @@ -27,8 +27,6 @@ export function split(payload: z.infer) { // Process each group for splitting groups.forEach((group, index) => { - console.log(JSON.stringify(dependencyTreeManager.dependencyTree, null, 2)); - const splitRunner = new SplitRunner(dependencyTreeManager, group); const files = splitRunner.run(); diff --git a/packages/cli/src/api/sync.ts b/packages/cli/src/api/sync.ts index a30686e..c8abd9b 100644 --- a/packages/cli/src/api/sync.ts +++ b/packages/cli/src/api/sync.ts @@ -1,16 +1,10 @@ import { z } from "zod"; import { syncSchema } from "./helpers/validation"; import fs from "fs"; -import { - parseNanoApiAnnotation, - updateCommentFromAnnotation, -} from "../helper/annotations"; -import DependencyTreeManager from "../helper/dependencyTree"; -import Parser from "tree-sitter"; -import { getParserLanguageFromFile } from "../helper/treeSitter"; -import { replaceIndexesFromSourceCode } from "../helper/cleanup"; -import { getJavascriptAnnotationNodes } from "../helper/languages/javascript/annotations"; -import { getTypescriptAnnotationNodes } from "../helper/languages/typescript/annotations"; +import DependencyTreeManager from "../dependencyManager/dependencyManager"; +import AnnotationManager from "../annotationManager"; +import { getLanguagePluginFromFilePath } from "../languages"; +import { replaceIndexesFromSourceCode } from "../helper/file"; export function sync(payload: z.infer) { const dependencyTreeManager = new DependencyTreeManager( @@ -35,13 +29,11 @@ export function sync(payload: z.infer) { }); updatedEndpoints.forEach((endpoint) => { - const language = getParserLanguageFromFile(endpoint.filePath); - const parser = new Parser(); - parser.setLanguage(language); + const languagePlugin = getLanguagePluginFromFilePath(endpoint.filePath); - let sourceCode = fs.readFileSync(endpoint.filePath, "utf-8"); + const sourceCode = fs.readFileSync(endpoint.filePath, "utf-8"); - const tree = parser.parse(sourceCode); + const tree = languagePlugin.parser.parse(sourceCode); const indexesToReplace: { startIndex: number; @@ -49,37 +41,35 @@ export function sync(payload: z.infer) { text: string; }[] = []; - let annotationNodes: Parser.SyntaxNode[]; - if (language.name === "javascript") { - annotationNodes = getJavascriptAnnotationNodes(parser, tree.rootNode); - } else if (language.name === "typescript") { - annotationNodes = getTypescriptAnnotationNodes(parser, tree.rootNode); - } else { - throw new Error("Language not supported"); - } + const commentNodes = languagePlugin.getCommentNodes(tree.rootNode); - annotationNodes.forEach((node) => { - const annotation = parseNanoApiAnnotation(node.text); - if ( - annotation.path === endpoint.path && - annotation.method === endpoint.method - ) { - annotation.group = endpoint.group; - const updatedComment = updateCommentFromAnnotation( + commentNodes.forEach((node) => { + try { + const annotationManager = new AnnotationManager( node.text, - annotation, + languagePlugin, ); + if (annotationManager.matchesEndpoint(endpoint.path, endpoint.method)) { + annotationManager.group = endpoint.group; + const updatedComment = annotationManager.stringify(); - indexesToReplace.push({ - startIndex: node.startIndex, - endIndex: node.endIndex, - text: updatedComment, - }); + indexesToReplace.push({ + startIndex: node.startIndex, + endIndex: node.endIndex, + text: updatedComment, + }); + } + } catch { + // Skip if annotation is not valid, we assume it is a regular comment + return; } }); - sourceCode = replaceIndexesFromSourceCode(sourceCode, indexesToReplace); + const updatedSourceCode = replaceIndexesFromSourceCode( + sourceCode, + indexesToReplace, + ); - fs.writeFileSync(endpoint.filePath, sourceCode, "utf-8"); + fs.writeFileSync(endpoint.filePath, updatedSourceCode, "utf-8"); }); } diff --git a/packages/cli/src/commands/annotate.ts b/packages/cli/src/commands/annotate.ts index 4bb781c..d83765a 100644 --- a/packages/cli/src/commands/annotate.ts +++ b/packages/cli/src/commands/annotate.ts @@ -1,7 +1,7 @@ import fs from "fs"; -import DependencyTreeManager from "../helper/dependencyTree"; +import DependencyTreeManager from "../dependencyManager/dependencyManager"; import OpenAI from "openai"; -import { DependencyTree } from "../helper/types"; +import { DependencyTree } from "../dependencyManager/types"; export default async function annotateOpenAICommandHandler( entrypoint: string, // Path to the entrypoint file diff --git a/packages/cli/src/commands/split.ts b/packages/cli/src/commands/split.ts index f159c2a..fd45e1a 100644 --- a/packages/cli/src/commands/split.ts +++ b/packages/cli/src/commands/split.ts @@ -1,15 +1,15 @@ import path from "path"; import fs from "fs"; -import DependencyTreeManager from "../helper/dependencyTree"; +import DependencyTreeManager from "../dependencyManager/dependencyManager"; import { cleanupOutputDir, createOutputDir } from "../helper/file"; -import SplitRunner from "../helper/split"; -import { GroupMap } from "../helper/types"; +import SplitRunner from "../splitRunner/splitRunner"; +import { Group } from "../dependencyManager/types"; export default function splitCommandHandler( entrypointPath: string, // Path to the entrypoint file outputDir: string, // Path to the output directory ) { - const groupMap: GroupMap = {}; + const groupMap: Record = {}; const dependencyTreeManager = new DependencyTreeManager(entrypointPath); diff --git a/packages/cli/src/helper/dependencyTree.ts b/packages/cli/src/dependencyManager/dependencyManager.ts similarity index 53% rename from packages/cli/src/helper/dependencyTree.ts rename to packages/cli/src/dependencyManager/dependencyManager.ts index 7149548..e2456bb 100644 --- a/packages/cli/src/helper/dependencyTree.ts +++ b/packages/cli/src/dependencyManager/dependencyManager.ts @@ -1,28 +1,19 @@ import fs from "fs"; -import Parser from "tree-sitter"; - -import { resolveFilePath } from "./file"; -import { DependencyTree, Group } from "./types"; -import { Endpoint } from "./types"; -import { getParserLanguageFromFile } from "./treeSitter"; -import { parseNanoApiAnnotation } from "./annotations"; -import { getJavascriptAnnotationNodes } from "./languages/javascript/annotations"; -import { getJavascriptImports } from "./languages/javascript/imports"; -import { getTypescriptImports } from "./languages/typescript/imports"; -import { getPythonImports } from "./languages/python/imports"; + +import { resolveFilePath } from "../helper/file"; +import { DependencyTree, Group, Endpoint } from "./types"; +import AnnotationManager from "../annotationManager"; +import { getLanguagePluginFromFilePath } from "../languages"; class DependencyTreeManager { - private parser: Parser; dependencyTree: DependencyTree; constructor(filePath: string) { - this.parser = new Parser(); - - const dependencyTree = this.#getDependencyTree(this.parser, filePath); + const dependencyTree = this.#getDependencyTree(filePath); this.dependencyTree = dependencyTree; } - #getDependencyTree(parser: Parser, filePath: string): DependencyTree { + #getDependencyTree(filePath: string): DependencyTree { const sourceCode = fs.readFileSync(filePath, "utf8"); const dependencyTree: DependencyTree = { @@ -31,39 +22,18 @@ class DependencyTreeManager { children: [], }; - const language = getParserLanguageFromFile(filePath); - - parser.setLanguage(language); - - const tree = parser.parse(sourceCode); - - let imports: { - node: Parser.SyntaxNode; - source: string; - importSpecifierIdentifiers: Parser.SyntaxNode[]; - importIdentifier?: Parser.SyntaxNode; - namespaceImport?: Parser.SyntaxNode; - }[]; - if (language.name === "javascript") { - imports = getJavascriptImports(parser, tree.rootNode); - imports = imports.filter((importPath) => - importPath.source.startsWith("."), - ); - } else if (language.name === "typescript") { - imports = getTypescriptImports(parser, tree.rootNode); - imports = imports.filter((importPath) => - importPath.source.startsWith("."), - ); - } else if (language.name === "python") { - imports = getPythonImports(parser, tree.rootNode); - } else { - throw new Error(`Unsupported language: ${language.name}`); - } + const languagePlugin = getLanguagePluginFromFilePath(filePath); + + const tree = languagePlugin.parser.parse(sourceCode); + + let imports = languagePlugin.getImports(tree.rootNode); + + imports = imports.filter((importPath) => importPath.source.startsWith(".")); imports.forEach((importPath) => { const resolvedPath = resolveFilePath(importPath.source, filePath); if (resolvedPath && fs.existsSync(resolvedPath)) { - const childTree = this.#getDependencyTree(parser, resolvedPath); + const childTree = this.#getDependencyTree(resolvedPath); dependencyTree.children.push(childTree); } }); @@ -88,32 +58,33 @@ class DependencyTreeManager { parentFilePaths: string[], dependencyTree: DependencyTree, ) { - const language = getParserLanguageFromFile(dependencyTree.path); - this.parser.setLanguage(language); + const languagePlugin = getLanguagePluginFromFilePath(dependencyTree.path); - const parsedTree = this.parser.parse(dependencyTree.sourceCode); + const tree = languagePlugin.parser.parse(dependencyTree.sourceCode); const endpoints: Endpoint[] = []; - const annotationNodes = getJavascriptAnnotationNodes( - this.parser, - parsedTree.rootNode, - ); - - annotationNodes.forEach((node) => { - const annotation = parseNanoApiAnnotation(node.text); - - // Only add endpoints, not annotations that are just grouping endpoints - if (annotation.method) { - const endpoint: Endpoint = { - path: annotation.path, - method: annotation.method, - group: annotation.group, - filePath: dependencyTree.path, - parentFilePaths, - childrenFilePaths: this.#getChildrenFilePaths(dependencyTree), - }; - endpoints.push(endpoint); + const commentNodes = languagePlugin.getCommentNodes(tree.rootNode); + + commentNodes.forEach((node) => { + try { + const annotationManager = new AnnotationManager( + node.text, + languagePlugin, + ); + if (annotationManager.method) { + const endpoint: Endpoint = { + path: annotationManager.path, + method: annotationManager.method, + group: annotationManager.group, + filePath: dependencyTree.path, + parentFilePaths, + childrenFilePaths: this.#getChildrenFilePaths(dependencyTree), + }; + endpoints.push(endpoint); + } + } catch { + return; } }); diff --git a/packages/cli/src/helper/types.ts b/packages/cli/src/dependencyManager/types.ts similarity index 70% rename from packages/cli/src/helper/types.ts rename to packages/cli/src/dependencyManager/types.ts index a8f2f2a..8a016b6 100644 --- a/packages/cli/src/helper/types.ts +++ b/packages/cli/src/dependencyManager/types.ts @@ -1,3 +1,9 @@ +export interface DependencyTree { + path: string; + sourceCode: string; + children: DependencyTree[]; +} + export interface Endpoint { path: string; method?: string; @@ -11,17 +17,3 @@ export interface Group { name: string; endpoints: Endpoint[]; } - -export type GroupMap = Record; - -export interface NanoAPIAnnotation { - path: string; - method?: string; - group?: string; -} - -export interface DependencyTree { - path: string; - sourceCode: string; - children: DependencyTree[]; -} diff --git a/packages/cli/src/helper/annotations.ts b/packages/cli/src/helper/annotations.ts deleted file mode 100644 index 32e5ac9..0000000 --- a/packages/cli/src/helper/annotations.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { Group, NanoAPIAnnotation } from "./types"; - -export function parseNanoApiAnnotation(value: string) { - const nanoapiRegex = /@nanoapi|((method|path|group):([^ ]+))/g; - const matches = value.match(nanoapiRegex); - // remove first match, which is the @nanoapi identifier - matches?.shift(); - - if (matches && matches.length > 0) { - return matches.reduce((acc, match) => { - // key, first element when split with ":" - const key = match.split(":")[0]; - // value, everything else - const value = match.split(":").slice(1).join(":"); - return { ...acc, [key]: value }; - }, {} as NanoAPIAnnotation); - } - - throw new Error("Could not parse annotation"); -} - -export function isAnnotationInGroup( - group: Group, - annotation: NanoAPIAnnotation, -) { - // check if annotation has a method (actual endpoint) - if (annotation.method) { - const endpoint = group.endpoints.find( - (endpoint) => - endpoint.method === annotation.method && - endpoint.path === annotation.path, - ); - - if (endpoint) { - return true; - } - - return false; - } - - // if annotation has no method, we treat it as a module that contains other endpoints - const endpoints = group.endpoints.filter((endpoint) => - endpoint.path.startsWith(annotation.path), - ); - - if (endpoints.length > 0) { - return true; - } - - return false; -} - -export function updateCommentFromAnnotation( - comment: string, - annotation: NanoAPIAnnotation, -) { - const commentRegex = /@nanoapi\s*(.*)/g; - - // Construct the new annotation string - let newAnnotation = "@nanoapi"; - if (annotation.method) { - newAnnotation += ` method:${annotation.method}`; - } - if (annotation.path) { - newAnnotation += ` path:${annotation.path}`; - } - if (annotation.group) { - newAnnotation += ` group:${annotation.group}`; - } - - // Replace the old annotation with the new annotation - const updatedComment = comment.replace(commentRegex, newAnnotation); - - return updatedComment; -} diff --git a/packages/cli/src/helper/cleanup.ts b/packages/cli/src/helper/cleanup.ts deleted file mode 100644 index af4a89d..0000000 --- a/packages/cli/src/helper/cleanup.ts +++ /dev/null @@ -1,276 +0,0 @@ -import Parser from "tree-sitter"; -import { - cleanupJavascriptAnnotations, - cleanupJavascriptInvalidImports, - cleanupUnusedJavascriptImports, -} from "./languages/javascript/cleanup"; -import { getJavascriptImports } from "./languages/javascript/imports"; - -import { resolveFilePath } from "./file"; -import { Group } from "./types"; -import { getParserLanguageFromFile } from "./treeSitter"; -import { getJavascriptExports } from "./languages/javascript/exports"; -import assert from "assert"; -import { - cleanupTypescriptAnnotations, - cleanupTypescriptInvalidImports, - cleanupUnusedTypescriptImports, -} from "./languages/typescript/cleanup"; -import { getTypescriptExports } from "./languages/typescript/exports"; -import { getTypescriptImports } from "./languages/typescript/imports"; - -export function cleanupAnnotations( - filePath: string, - sourceCode: string, - group: Group, -) { - const language = getParserLanguageFromFile(filePath); - const parser = new Parser(); - parser.setLanguage(language); - - let updatedSourceCode: string; - - if (language.name === "javascript") { - updatedSourceCode = cleanupJavascriptAnnotations(parser, sourceCode, group); - } else if (language.name === "typescript") { - updatedSourceCode = cleanupTypescriptAnnotations(parser, sourceCode, group); - } else { - throw new Error(`Unsupported language: ${language.language}`); - } - - return updatedSourceCode; -} - -export function getExportMap(files: { path: string; sourceCode: string }[]) { - const exportIdentifiersMap = new Map< - string, - { - namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[]; - defaultExport?: Parser.SyntaxNode; - } - >(); - files.forEach((file) => { - const language = getParserLanguageFromFile(file.path); - const parser = new Parser(); - parser.setLanguage(language); - - const tree = parser.parse(file.sourceCode); - - if (language.name === "javascript") { - const exports = getJavascriptExports(parser, tree.rootNode); - exportIdentifiersMap.set(file.path, exports); - } else if (language.name === "typescript") { - const exports = getTypescriptExports(parser, tree.rootNode); - exportIdentifiersMap.set(file.path, exports); - } else { - throw new Error(`Unsupported language: ${language.language}`); - } - }); - - return exportIdentifiersMap; -} - -export function cleanupInvalidImports( - filePath: string, - sourceCode: string, - exportIdentifiersMap: Map< - string, - { - namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[]; - defaultExport?: Parser.SyntaxNode; - } - >, -) { - const language = getParserLanguageFromFile(filePath); - const parser = new Parser(); - parser.setLanguage(language); - - let updatedSourceCode: string = sourceCode; - - if (language.name === "javascript") { - updatedSourceCode = cleanupJavascriptInvalidImports( - parser, - filePath, - sourceCode, - exportIdentifiersMap, - ); - } else if (language.name === "typescript") { - updatedSourceCode = cleanupTypescriptInvalidImports( - parser, - filePath, - sourceCode, - exportIdentifiersMap, - ); - } else { - throw new Error(`Unsupported language: ${language.language}`); - } - - return updatedSourceCode; -} - -export function cleanupUnusedImports(filePath: string, sourceCode: string) { - const language = getParserLanguageFromFile(filePath); - const parser = new Parser(); - parser.setLanguage(language); - - let updatedSourceCode: string; - if (language.name === "javascript") { - updatedSourceCode = cleanupUnusedJavascriptImports(parser, sourceCode); - } else if (language.name === "typescript") { - updatedSourceCode = cleanupUnusedTypescriptImports(parser, sourceCode); - } else { - throw new Error(`Unsupported language: ${language.language}`); - } - - return updatedSourceCode; -} - -export function cleanupUnusedFiles( - entrypointPath: string, - files: { path: string; sourceCode: string }[], -) { - let fileRemoved = true; - while (fileRemoved) { - fileRemoved = false; - - // We always want to keep the entrypoint file. - // It will never be imported anywhere, so we add it now. - const filesToKeep = new Set(); - filesToKeep.add(entrypointPath); - - files.forEach((file) => { - const language = getParserLanguageFromFile(file.path); - const parser = new Parser(); - parser.setLanguage(language); - - const tree = parser.parse(file.sourceCode); - - let dependencies: { - node: Parser.SyntaxNode; - source: string; - importSpecifierIdentifiers: Parser.SyntaxNode[]; - importIdentifier?: Parser.SyntaxNode; - namespaceImport?: Parser.SyntaxNode; - }[]; - if (language.name === "javascript") { - dependencies = getJavascriptImports(parser, tree.rootNode); - // Only keep files dependencies - dependencies = dependencies.filter((dep) => dep.source.startsWith(".")); - } else if (language.name === "typescript") { - dependencies = getTypescriptImports(parser, tree.rootNode); - // Only keep files dependencies - dependencies = dependencies.filter((dep) => dep.source.startsWith(".")); - } else { - throw new Error(`Unsupported language: ${language.language}`); - } - - dependencies.forEach((dep) => { - const resolvedPath = resolveFilePath(dep.source, file.path); - if (resolvedPath) { - filesToKeep.add(resolvedPath); - } - }); - }); - - const previousFilesLength = files.length; - - files = files.filter((file) => { - return filesToKeep.has(file.path); - }); - - if (files.length !== previousFilesLength) { - fileRemoved = true; - } - } - - return files; -} - -export function cleanupUnusedExports( - files: { path: string; sourceCode: string }[], - exportIdentifiersMap: Map< - string, - { - namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[]; - defaultExport?: Parser.SyntaxNode; - } - >, -) { - // TODO need to be implemented - - // Step 1, create variable to track which export is user - // Step 2, iterate over all file imports. If the import is used, mark the export as used - // Step 3, iterate over each file, and remove the unused exports - - // Repeat above step until no more unused exports are found - assert(exportIdentifiersMap); - - return files; -} - -export function cleanupErrors(filePath: string, sourceCode: string) { - const language = getParserLanguageFromFile(filePath); - const parser = new Parser(); - parser.setLanguage(language); - - const tree = parser.parse(sourceCode); - - const indexesToRemove: { startIndex: number; endIndex: number }[] = []; - - const query = new Parser.Query(parser.getLanguage(), "(ERROR) @error"); - const errorCaptures = query.captures(tree.rootNode); - errorCaptures.forEach((capture) => { - indexesToRemove.push({ - startIndex: capture.node.startIndex, - endIndex: capture.node.endIndex, - }); - }); - - const updatedSourceCode = removeIndexesFromSourceCode( - sourceCode, - indexesToRemove, - ); - - return updatedSourceCode; -} - -export function removeIndexesFromSourceCode( - sourceCode: string, - indexesToRemove: { startIndex: number; endIndex: number }[], -) { - let newSourceCode = sourceCode; - - // sort to start removing from the of the file end - indexesToRemove.sort((a, b) => b.startIndex - a.startIndex); - - indexesToRemove.forEach(({ startIndex, endIndex }) => { - newSourceCode = - newSourceCode.slice(0, startIndex) + newSourceCode.slice(endIndex); - }); - - return newSourceCode; -} - -export function replaceIndexesFromSourceCode( - sourceCode: string, - indexesToReplace: { startIndex: number; endIndex: number; text: string }[], -) { - // sort to start removing from the end of the file - indexesToReplace.sort((a, b) => b.startIndex - a.startIndex); - - indexesToReplace.forEach(({ startIndex, endIndex, text }) => { - sourceCode = - sourceCode.slice(0, startIndex) + text + sourceCode.slice(endIndex); - }); - - return sourceCode; -} diff --git a/packages/cli/src/helper/file.ts b/packages/cli/src/helper/file.ts index 971fad8..13264d0 100644 --- a/packages/cli/src/helper/file.ts +++ b/packages/cli/src/helper/file.ts @@ -48,3 +48,35 @@ export function resolveFilePath(importPath: string, currentFile: string) { // Skip external dependencies (e.g., node_modules) return null; } + +export function removeIndexesFromSourceCode( + sourceCode: string, + indexesToRemove: { startIndex: number; endIndex: number }[], +) { + let newSourceCode = sourceCode; + + // sort to start removing from the of the file end + indexesToRemove.sort((a, b) => b.startIndex - a.startIndex); + + indexesToRemove.forEach(({ startIndex, endIndex }) => { + newSourceCode = + newSourceCode.slice(0, startIndex) + newSourceCode.slice(endIndex); + }); + + return newSourceCode; +} + +export function replaceIndexesFromSourceCode( + sourceCode: string, + indexesToReplace: { startIndex: number; endIndex: number; text: string }[], +) { + // sort to start removing from the end of the file + indexesToReplace.sort((a, b) => b.startIndex - a.startIndex); + + indexesToReplace.forEach(({ startIndex, endIndex, text }) => { + sourceCode = + sourceCode.slice(0, startIndex) + text + sourceCode.slice(endIndex); + }); + + return sourceCode; +} diff --git a/packages/cli/src/helper/languages/javascript/annotations.ts b/packages/cli/src/helper/languages/javascript/annotations.ts deleted file mode 100644 index a6c0034..0000000 --- a/packages/cli/src/helper/languages/javascript/annotations.ts +++ /dev/null @@ -1,22 +0,0 @@ -import Parser from "tree-sitter"; - -export function getJavascriptAnnotationNodes( - parser: Parser, - node: Parser.SyntaxNode, -) { - const commentQuery = new Parser.Query( - parser.getLanguage(), - ` - ( - (comment) @comment - (#match? @comment "^//( *)@nanoapi") - ) - `, - ); - - const commentCaptures = commentQuery.captures(node); - - return commentCaptures.map((capture) => { - return capture.node; - }); -} diff --git a/packages/cli/src/helper/languages/javascript/cleanup.ts b/packages/cli/src/helper/languages/javascript/cleanup.ts deleted file mode 100644 index bd5b39b..0000000 --- a/packages/cli/src/helper/languages/javascript/cleanup.ts +++ /dev/null @@ -1,276 +0,0 @@ -import Parser from "tree-sitter"; -import { Group } from "../../types"; -import { parseNanoApiAnnotation, isAnnotationInGroup } from "../../annotations"; -import { - getJavascriptImports, - getJavascriptImportIdentifierUsage, -} from "./imports"; -import { removeIndexesFromSourceCode } from "../../cleanup"; -import { getJavascriptAnnotationNodes } from "./annotations"; -import { resolveFilePath } from "../../file"; -import { getTypescriptAnnotationNodes } from "../typescript/annotations"; -import { - getTypescriptImportIdentifierUsage, - getTypescriptImports, -} from "../typescript/imports"; - -export function cleanupJavascriptAnnotations( - parser: Parser, - sourceCode: string, - groupToKeep: Group, - isTypescript = false, -): string { - const tree = parser.parse(sourceCode); - - const indexesToRemove: { startIndex: number; endIndex: number }[] = []; - - const annotationNodes = isTypescript - ? getTypescriptAnnotationNodes(parser, tree.rootNode) - : getJavascriptAnnotationNodes(parser, tree.rootNode); - - annotationNodes.forEach((node) => { - const annotation = parseNanoApiAnnotation(node.text); - - const keepAnnotation = isAnnotationInGroup(groupToKeep, annotation); - - if (!keepAnnotation) { - let nextNode = node.nextNamedSibling; - // We need to remove all decorators too - while (nextNode && nextNode.type === "decorator") { - nextNode = nextNode.nextNamedSibling; - } - if (!nextNode) { - throw new Error("Could not find next node"); - } - - // Remove this node (comment) and the next node(s) (api endpoint) - indexesToRemove.push({ - startIndex: node.startIndex, - endIndex: nextNode.endIndex, - }); - } - }); - - const updatedSourceCode = removeIndexesFromSourceCode( - sourceCode, - indexesToRemove, - ); - - return updatedSourceCode; -} - -export function cleanupJavascriptInvalidImports( - parser: Parser, - filePath: string, - sourceCode: string, - exportIdentifiersMap: Map< - string, - { - namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[]; - defaultExport?: Parser.SyntaxNode; - } - >, - isTypescript = false, -) { - const indexesToRemove: { startIndex: number; endIndex: number }[] = []; - - const tree = parser.parse(sourceCode); - - const depImports = isTypescript - ? getTypescriptImports(parser, tree.rootNode) - : getJavascriptImports(parser, tree.rootNode); - // check if identifier exists in the imported file (as an export) - depImports.forEach((depImport) => { - // check if the import is a file, do not process external dependencies - if (depImport.source.startsWith(".")) { - const resolvedPath = resolveFilePath(depImport.source, filePath); - if (!resolvedPath) { - throw new Error("Could not resolve path"); - } - - const exportsForFile = exportIdentifiersMap.get(resolvedPath); - if (!exportsForFile) { - throw new Error("Could not find exports"); - } - - if (depImport.importIdentifier && !exportsForFile.defaultExport) { - let usages = isTypescript - ? getTypescriptImportIdentifierUsage( - parser, - tree.rootNode, - depImport.importIdentifier, - ) - : getJavascriptImportIdentifierUsage( - parser, - tree.rootNode, - depImport.importIdentifier, - ); - usages = usages.filter((usage) => { - return usage.id !== depImport.importIdentifier?.id; - }); - - indexesToRemove.push({ - startIndex: depImport.node.startIndex, - endIndex: depImport.node.endIndex, - }); - usages.forEach((usage) => { - indexesToRemove.push({ - startIndex: usage.startIndex, - endIndex: usage.endIndex, - }); - }); - } - - depImport.importSpecifierIdentifiers.forEach((importSpecifier) => { - if ( - !exportsForFile.namedExports.find( - (namedExport) => - namedExport.identifierNode.text === importSpecifier.text, - ) - ) { - let usages = isTypescript - ? getTypescriptImportIdentifierUsage( - parser, - tree.rootNode, - importSpecifier, - ) - : getJavascriptImportIdentifierUsage( - parser, - tree.rootNode, - importSpecifier, - ); - usages = usages.filter((usage) => { - return usage.id !== depImport.importIdentifier?.id; - }); - - indexesToRemove.push({ - startIndex: depImport.node.startIndex, - endIndex: depImport.node.endIndex, - }); - usages.forEach((usage) => { - indexesToRemove.push({ - startIndex: usage.startIndex, - endIndex: usage.endIndex, - }); - }); - } - }); - } - }); - - const updatedSourceCode = removeIndexesFromSourceCode( - sourceCode, - indexesToRemove, - ); - - return updatedSourceCode; -} - -export function cleanupUnusedJavascriptImports( - parser: Parser, - sourceCode: string, - isTypescript = false, -) { - const tree = parser.parse(sourceCode); - - const imports = isTypescript - ? getTypescriptImports(parser, tree.rootNode) - : getJavascriptImports(parser, tree.rootNode); - - const indexesToRemove: { startIndex: number; endIndex: number }[] = []; - - imports.forEach((depImport) => { - const importSpecifierToRemove: Parser.SyntaxNode[] = []; - depImport.importSpecifierIdentifiers.forEach((importSpecifier) => { - let usages = isTypescript - ? getTypescriptImportIdentifierUsage( - parser, - tree.rootNode, - importSpecifier, - ) - : getJavascriptImportIdentifierUsage( - parser, - tree.rootNode, - importSpecifier, - ); - usages = usages.filter((usage) => { - return usage.id !== importSpecifier.id; - }); - - if (usages.length === 0) { - importSpecifierToRemove.push(importSpecifier); - } - }); - - let removeDefaultImport = false; - if (depImport.importIdentifier) { - let usages = isTypescript - ? getTypescriptImportIdentifierUsage( - parser, - tree.rootNode, - depImport.importIdentifier, - ) - : getJavascriptImportIdentifierUsage( - parser, - tree.rootNode, - depImport.importIdentifier, - ); - usages = usages.filter((usage) => { - return usage.id !== depImport.importIdentifier?.id; - }); - - if (usages.length === 0) { - removeDefaultImport = true; - } - } - - let removeNameSpaceImport = false; - if (depImport.namespaceImport) { - let usages = isTypescript - ? getTypescriptImportIdentifierUsage( - parser, - tree.rootNode, - depImport.namespaceImport, - ) - : getJavascriptImportIdentifierUsage( - parser, - tree.rootNode, - depImport.namespaceImport, - ); - usages = usages.filter((usage) => { - return usage.id !== depImport.importIdentifier?.id; - }); - if (usages.length === 0) { - removeNameSpaceImport = true; - } - } - - if ( - importSpecifierToRemove.length === - depImport.importSpecifierIdentifiers.length && - (removeDefaultImport || removeNameSpaceImport) - ) { - indexesToRemove.push({ - startIndex: depImport.node.startIndex, - endIndex: depImport.node.endIndex, - }); - } else { - importSpecifierToRemove.forEach((importSpecifier) => { - indexesToRemove.push({ - startIndex: importSpecifier.startIndex, - endIndex: importSpecifier.endIndex, - }); - }); - } - }); - - const updatedSourceCode = removeIndexesFromSourceCode( - sourceCode, - indexesToRemove, - ); - - return updatedSourceCode; -} diff --git a/packages/cli/src/helper/languages/javascript/exports.ts b/packages/cli/src/helper/languages/javascript/exports.ts deleted file mode 100644 index 3513d4d..0000000 --- a/packages/cli/src/helper/languages/javascript/exports.ts +++ /dev/null @@ -1,89 +0,0 @@ -import Parser from "tree-sitter"; - -export function getJavascriptExports( - parser: Parser, - node: Parser.SyntaxNode, - isTypescript = false, -) { - const exportQuery = new Parser.Query( - parser.getLanguage(), - ` - ( - (export_statement) @export - (#not-match? @export "export default") - ) - `, - ); - - const exportCaptures = exportQuery.captures(node); - - const namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[] = []; - exportCaptures.forEach((capture) => { - const identifierQuery = new Parser.Query( - parser.getLanguage(), - isTypescript - ? ` - declaration: ([ - (_ - name: ([(identifier) (type_identifier)]) @identifier - ) - (_ - (_ - name: ([(identifier) (type_identifier)]) @identifier - ) - ) - ]) - ` - : ` - declaration: ([ - (_ - name: (identifier) @identifier - ) - (_ - (_ - name: (identifier) @identifier - ) - ) - ]) - `, - ); - - const identifierCaptures = identifierQuery.captures(capture.node); - if (identifierCaptures.length === 0) { - throw new Error("No identifier found in export statement"); - } - identifierCaptures.forEach((capture) => { - namedExports.push({ - exportNode: capture.node, - identifierNode: capture.node, - }); - }); - }); - - const defaultExportQuery = new Parser.Query( - parser.getLanguage(), - ` - ( - (export_statement) @export - (#match? @export "export default") - ) - `, - ); - const defaultExportCaptures = defaultExportQuery.captures(node); - if (defaultExportCaptures.length > 1) { - throw new Error("Found multiple default export. Only one is allowed"); - } - if (defaultExportCaptures.length === 1) { - return { - namedExports, - defaultExport: defaultExportCaptures[0].node, - }; - } - - return { - namedExports, - }; -} diff --git a/packages/cli/src/helper/languages/javascript/imports.ts b/packages/cli/src/helper/languages/javascript/imports.ts deleted file mode 100644 index a81b257..0000000 --- a/packages/cli/src/helper/languages/javascript/imports.ts +++ /dev/null @@ -1,319 +0,0 @@ -import Parser from "tree-sitter"; - -function getImportStatements(parser: Parser, node: Parser.SyntaxNode) { - const imports: { - node: Parser.SyntaxNode; - source: string; - importSpecifierIdentifiers: Parser.SyntaxNode[]; - importIdentifier?: Parser.SyntaxNode; - namespaceImport?: Parser.SyntaxNode; - }[] = []; - - const importStatementQuery = new Parser.Query( - parser.getLanguage(), - ` - (import_statement) @import - `, - ); - const importStatementCaptures = importStatementQuery.captures(node); - importStatementCaptures.forEach((capture) => { - const importSourceQuery = new Parser.Query( - parser.getLanguage(), - ` - source: (string - (string_fragment) @source - ) - `, - ); - const importSourceCaptures = importSourceQuery.captures(capture.node); - if (importSourceCaptures.length === 0) { - throw new Error("Could not find import source"); - } - if (importSourceCaptures.length > 1) { - throw new Error("Found multiple import sources"); - } - const source = importSourceCaptures[0].node.text; - - const importSpecifierIdentifierQuery = new Parser.Query( - parser.getLanguage(), - ` - (import_specifier - (identifier) @identifier - ) - `, - ); - const importSpecifierCaptures = importSpecifierIdentifierQuery.captures( - capture.node, - ); - const importSpecifierIdentifiers = importSpecifierCaptures.map( - (capture) => { - return capture.node; - }, - ); - - const importClauseIdentifierQuery = new Parser.Query( - parser.getLanguage(), - ` - (import_clause - (identifier) @identifier - ) - `, - ); - const importClauseIdentifierCaptures = importClauseIdentifierQuery.captures( - capture.node, - ); - if (importClauseIdentifierCaptures.length > 1) { - throw new Error("Found multiple import clause identifier"); - } - const importIdentifier = importClauseIdentifierCaptures.length - ? importClauseIdentifierCaptures[0].node - : undefined; - - const nameSpaceimportClauseIdentifierQuery = new Parser.Query( - parser.getLanguage(), - ` - (import_clause - (namespace_import - (identifier) @identifier - ) - ) - `, - ); - const nameSpaceimportClauseIdentifierCaptures = - nameSpaceimportClauseIdentifierQuery.captures(capture.node); - if (nameSpaceimportClauseIdentifierCaptures.length > 1) { - throw new Error("Found multiple namespace import clause identifier"); - } - const namespaceImport = nameSpaceimportClauseIdentifierCaptures.length - ? nameSpaceimportClauseIdentifierCaptures[0].node - : undefined; - - imports.push({ - node: capture.node, - source, - importSpecifierIdentifiers, - importIdentifier, - namespaceImport, - }); - }); - - return imports; -} - -// function getRequireAndDynamicImports(parser: Parser, node: Parser.SyntaxNode) { -// const imports: { -// node: Parser.SyntaxNode; -// source: string; -// importSpecifierIdentifiers: Parser.SyntaxNode[]; -// importIdentifier?: Parser.SyntaxNode; -// namespaceImport?: undefined; -// }[] = []; - -// const requireStatementQuery = new Parser.Query( -// parser.getLanguage(), -// ` -// ([ -// (lexical_declaration -// (variable_declarator -// (call_expression -// ( -// ([(identifier) (import)]) @call_expression -// (#match? @call_expression "^(require|import)$") -// ) -// ) -// ) -// ) -// (variable_declaration -// (variable_declarator -// (call_expression -// ( -// ([(identifier) (import)]) @call_expression -// (#match? @call_expression "^(require|import)$") -// ) -// ) -// ) -// ) -// ]) @import -// `, -// ); -// let requireStatementCaptures = requireStatementQuery.captures(node); -// requireStatementCaptures = requireStatementCaptures.filter( -// (capture) => capture.name === "import", -// ); -// requireStatementCaptures.forEach((capture) => { -// const requireSourceQuery = new Parser.Query( -// parser.getLanguage(), -// ` -// ([ -// (lexical_declaration -// (variable_declarator -// value: (call_expression -// arguments: (arguments -// (string -// (string_fragment) @source -// ) -// ) -// ) -// ) -// ) -// (variable_declaration -// (variable_declarator -// value: (call_expression -// arguments: (arguments -// (string -// (string_fragment) @source -// ) -// ) -// ) -// ) -// ) -// ]) -// `, -// ); -// const requireSourceCaptures = requireSourceQuery.captures(capture.node); -// if (requireSourceCaptures.length === 0) { -// throw new Error("Could not find require source"); -// } -// if (requireSourceCaptures.length > 1) { -// throw new Error("Found multiple require sources"); -// } -// const source = requireSourceCaptures[0].node.text; - -// const importSpecifierIdentifierQuery = new Parser.Query( -// parser.getLanguage(), -// ` -// ([ -// (lexical_declaration -// (variable_declarator -// name: (object_pattern -// (shorthand_property_identifier_pattern) @identifier -// ) -// ) -// ) -// (variable_declaration -// (variable_declarator -// name: (object_pattern -// (shorthand_property_identifier_pattern) @identifier -// ) -// ) -// ) -// ]) -// `, -// ); - -// const importSpecifierCaptures = importSpecifierIdentifierQuery.captures( -// capture.node, -// ); -// const importSpecifierIdentifiers = importSpecifierCaptures.map( -// (capture) => { -// return capture.node; -// }, -// ); - -// const importClauseIdentifierQuery = new Parser.Query( -// parser.getLanguage(), -// ` -// ([ -// (lexical_declaration -// (variable_declarator -// name: (identifier) @identifier -// ) -// ) -// (variable_declaration -// (variable_declarator -// name: (identifier) @identifier -// ) -// ) -// ]) -// `, -// ); -// const importClauseIdentifierCaptures = importClauseIdentifierQuery.captures( -// capture.node, -// ); -// if (importClauseIdentifierCaptures.length > 1) { -// throw new Error("Found multiple import clause identifier"); -// } -// const importIdentifier = importClauseIdentifierCaptures.length -// ? importClauseIdentifierCaptures[0].node -// : undefined; - -// imports.push({ -// node: capture.node, -// source, -// importSpecifierIdentifiers, -// importIdentifier, -// namespaceImport: undefined, -// }); -// }); - -// return imports; -// } - -export function getJavascriptImports(parser: Parser, node: Parser.SyntaxNode) { - const imports = getImportStatements(parser, node); - // TODO splitting is not reliable enought with require and dynamic imports. - // For now we do not use this. - // imports.push(...getRequireAndDynamicImports(parser, node)); - - return imports; -} - -export function getJavascriptImportIdentifierUsage( - parser: Parser, - node: Parser.SyntaxNode, - identifier: Parser.SyntaxNode, - isTypescript = false, -) { - const usageNodes: Parser.SyntaxNode[] = []; - const identifierQuery = new Parser.Query( - parser.getLanguage(), - isTypescript - ? ` - ( - ([ - (identifier) - (type_identifier) - ]) @identifier - (#eq? @identifier "${identifier.text}") - ) - ` - : ` - ( - (identifier) @identifier - (#eq? @identifier "${identifier.text}") - ) - `, - ); - const identifierCaptures = identifierQuery.captures(node); - identifierCaptures.forEach((capture) => { - if (capture.node.id === identifier.id) { - return; - } - - let targetNode = capture.node; - while (true) { - // we can remove from the array - if (targetNode.parent && targetNode.parent.type === "array") { - break; - } - - if ( - targetNode.parent && - targetNode.parent.type === "expression_statement" - ) { - break; - } - - // TODO: add more cases as we encounter them - - if (!targetNode.parent) { - break; - } - targetNode = targetNode.parent; - } - - return usageNodes.push(targetNode); - }); - - return usageNodes; -} diff --git a/packages/cli/src/helper/languages/python/imports.ts b/packages/cli/src/helper/languages/python/imports.ts deleted file mode 100644 index 525524b..0000000 --- a/packages/cli/src/helper/languages/python/imports.ts +++ /dev/null @@ -1,57 +0,0 @@ -import Parser from "tree-sitter"; - -export function getPythonImports(parser: Parser, node: Parser.SyntaxNode) { - const imports: { - node: Parser.SyntaxNode; - source: string; - importSpecifierIdentifiers: Parser.SyntaxNode[]; - }[] = []; - - const importStatementQuery = new Parser.Query( - parser.getLanguage(), - ` - (import_from_statement) @import - `, - ); - const importStatementCaptures = importStatementQuery.captures(node); - importStatementCaptures.forEach((capture) => { - const importSourceQuery = new Parser.Query( - parser.getLanguage(), - ` - module_name: (dotted_name) @source - `, - ); - const importSourceCaptures = importSourceQuery.captures(capture.node); - if (importSourceCaptures.length === 0) { - throw new Error("Could not find import source"); - } - if (importSourceCaptures.length > 1) { - throw new Error("Found multiple import sources"); - } - // need to replace all dots to / so it is a valid path - const source = importSourceCaptures[0].node.text.replaceAll(".", "/"); - - const importSpecifierIdentifierQuery = new Parser.Query( - parser.getLanguage(), - ` - name: (dotted_name) @identifier - `, - ); - const importSpecifierCaptures = importSpecifierIdentifierQuery.captures( - capture.node, - ); - const importSpecifierIdentifiers = importSpecifierCaptures.map( - (capture) => { - return capture.node; - }, - ); - - imports.push({ - node: capture.node, - source, - importSpecifierIdentifiers, - }); - }); - - return imports; -} diff --git a/packages/cli/src/helper/languages/typescript/annotations.ts b/packages/cli/src/helper/languages/typescript/annotations.ts deleted file mode 100644 index 4cccbf9..0000000 --- a/packages/cli/src/helper/languages/typescript/annotations.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Parser from "tree-sitter"; -import { getJavascriptAnnotationNodes } from "../javascript/annotations"; - -export function getTypescriptAnnotationNodes( - parser: Parser, - node: Parser.SyntaxNode, -) { - return getJavascriptAnnotationNodes(parser, node); -} diff --git a/packages/cli/src/helper/languages/typescript/cleanup.ts b/packages/cli/src/helper/languages/typescript/cleanup.ts deleted file mode 100644 index 6618987..0000000 --- a/packages/cli/src/helper/languages/typescript/cleanup.ts +++ /dev/null @@ -1,46 +0,0 @@ -import Parser from "tree-sitter"; -import { Group } from "../../types"; -import { - cleanupJavascriptAnnotations, - cleanupJavascriptInvalidImports, - cleanupUnusedJavascriptImports, -} from "../javascript/cleanup"; - -export function cleanupTypescriptAnnotations( - parser: Parser, - sourceCode: string, - groupToKeep: Group, -): string { - return cleanupJavascriptAnnotations(parser, sourceCode, groupToKeep, true); -} - -export function cleanupTypescriptInvalidImports( - parser: Parser, - filePath: string, - sourceCode: string, - exportIdentifiersMap: Map< - string, - { - namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[]; - defaultExport?: Parser.SyntaxNode; - } - >, -) { - return cleanupJavascriptInvalidImports( - parser, - filePath, - sourceCode, - exportIdentifiersMap, - true, - ); -} - -export function cleanupUnusedTypescriptImports( - parser: Parser, - sourceCode: string, -) { - return cleanupUnusedJavascriptImports(parser, sourceCode, true); -} diff --git a/packages/cli/src/helper/languages/typescript/exports.ts b/packages/cli/src/helper/languages/typescript/exports.ts deleted file mode 100644 index e293c31..0000000 --- a/packages/cli/src/helper/languages/typescript/exports.ts +++ /dev/null @@ -1,6 +0,0 @@ -import Parser from "tree-sitter"; -import { getJavascriptExports } from "../javascript/exports"; - -export function getTypescriptExports(parser: Parser, node: Parser.SyntaxNode) { - return getJavascriptExports(parser, node, true); -} diff --git a/packages/cli/src/helper/languages/typescript/imports.ts b/packages/cli/src/helper/languages/typescript/imports.ts deleted file mode 100644 index 33fd1e1..0000000 --- a/packages/cli/src/helper/languages/typescript/imports.ts +++ /dev/null @@ -1,17 +0,0 @@ -import Parser from "tree-sitter"; -import { - getJavascriptImportIdentifierUsage, - getJavascriptImports, -} from "../javascript/imports"; - -export function getTypescriptImports(parser: Parser, node: Parser.SyntaxNode) { - return getJavascriptImports(parser, node); -} - -export function getTypescriptImportIdentifierUsage( - parser: Parser, - node: Parser.SyntaxNode, - identifier: Parser.SyntaxNode, -) { - return getJavascriptImportIdentifierUsage(parser, node, identifier, true); -} diff --git a/packages/cli/src/helper/split.ts b/packages/cli/src/helper/split.ts deleted file mode 100644 index 2d4bff6..0000000 --- a/packages/cli/src/helper/split.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { Group } from "./types"; -import { - cleanupAnnotations, - getExportMap, - cleanupInvalidImports, - cleanupUnusedImports, - cleanupUnusedFiles, - cleanupErrors, - cleanupUnusedExports, -} from "./cleanup"; -import DependencyTreeManager from "./dependencyTree"; -import Parser from "tree-sitter"; - -class SplitRunner { - private dependencyTreeManager: DependencyTreeManager; - private group: Group; - private files: { path: string; sourceCode: string }[]; - - constructor(dependencyTreeManager: DependencyTreeManager, group: Group) { - this.dependencyTreeManager = dependencyTreeManager; - this.group = group; - this.files = dependencyTreeManager.getFiles(); - } - - #getExportMap() { - return getExportMap(this.files); - } - - #removeAnnotationFromOtherGroups() { - this.files = this.files.map((file) => { - const updatedSourceCode = cleanupAnnotations( - file.path, - file.sourceCode, - this.group, - ); - return { ...file, sourceCode: updatedSourceCode }; - }); - } - - #removeInvalidImportsAndUsages( - exportMap: Map< - string, - { - namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[]; - defaultExport?: Parser.SyntaxNode; - } - >, - ) { - this.files = this.files.map((file) => { - const updatedSourceCode = cleanupInvalidImports( - file.path, - file.sourceCode, - exportMap, - ); - return { ...file, sourceCode: updatedSourceCode }; - }); - } - - #removeUnusedImports() { - this.files = this.files.map((file) => { - const updatedSourceCode = cleanupUnusedImports( - file.path, - file.sourceCode, - ); - return { ...file, sourceCode: updatedSourceCode }; - }); - } - - #removeUnusedFiles() { - this.files = cleanupUnusedFiles( - this.dependencyTreeManager.dependencyTree.path, - this.files, - ); - } - - #removeUnusedExports( - exportMap: Map< - string, - { - namedExports: { - exportNode: Parser.SyntaxNode; - identifierNode: Parser.SyntaxNode; - }[]; - defaultExport?: Parser.SyntaxNode; - } - >, - ) { - this.files = cleanupUnusedExports(this.files, exportMap); - } - - #removeErrors() { - this.files = this.files.map((file) => { - const updatedSourceCode = cleanupErrors(file.path, file.sourceCode); - return { ...file, sourceCode: updatedSourceCode }; - }); - } - - run() { - console.log("\n"); - console.time("Splitting"); - - console.time("remove annotation from other groups"); - this.#removeAnnotationFromOtherGroups(); - console.timeEnd("remove annotation from other groups"); - - console.time("Get export map"); - const exportMap = this.#getExportMap(); - console.timeEnd("Get export map"); - - console.time("Remove invalid imports and usages"); - this.#removeInvalidImportsAndUsages(exportMap); - console.timeEnd("Remove invalid imports and usages"); - - console.time("Remove unused imports"); - this.#removeUnusedImports(); - console.timeEnd("Remove unused imports"); - - console.time("Remove unused files"); - this.#removeUnusedFiles(); - console.timeEnd("Remove unused files"); - - console.time("Remove unused exports"); - this.#removeUnusedExports(exportMap); - console.timeEnd("Remove unused exports"); - - console.time("Remove errors"); - this.#removeErrors(); - console.timeEnd("Remove errors"); - - console.timeEnd("Splitting"); - - return this.files; - } -} - -export default SplitRunner; diff --git a/packages/cli/src/helper/treeSitter.ts b/packages/cli/src/helper/treeSitter.ts deleted file mode 100644 index 3f61a05..0000000 --- a/packages/cli/src/helper/treeSitter.ts +++ /dev/null @@ -1,18 +0,0 @@ -import Javascript from "tree-sitter-javascript"; -import Typescript from "tree-sitter-typescript"; -import Python from "tree-sitter-python"; - -export function getParserLanguageFromFile(filePath: string) { - const ext = filePath.split(".").pop(); - - switch (ext) { - case "js": - return Javascript; - case "ts": - return Typescript.typescript; - case "py": - return Python; - default: - throw new Error(`Unsupported file type: ${ext}`); - } -} diff --git a/packages/cli/src/languages/index.ts b/packages/cli/src/languages/index.ts new file mode 100644 index 0000000..71b64be --- /dev/null +++ b/packages/cli/src/languages/index.ts @@ -0,0 +1,18 @@ +import JavascriptPlugin from "./javascript"; +import { LanguagePlugin } from "./types"; +import TypescriptPlugin from "./typescript"; + +export function getLanguagePluginFromFilePath( + filePath: string, +): LanguagePlugin { + const ext = filePath.split(".").pop(); + + switch (ext) { + case "js": + return new JavascriptPlugin(); + case "ts": + return new TypescriptPlugin(); + default: + throw new Error(`Unsupported file type: ${ext}`); + } +} diff --git a/packages/cli/src/languages/javascript.ts b/packages/cli/src/languages/javascript.ts new file mode 100644 index 0000000..55fc8f7 --- /dev/null +++ b/packages/cli/src/languages/javascript.ts @@ -0,0 +1,459 @@ +import Parser from "tree-sitter"; +import { LanguagePlugin, Import } from "./types"; +import { Group } from "../dependencyManager/types"; +import AnnotationManager from "../annotationManager"; +import { removeIndexesFromSourceCode, resolveFilePath } from "../helper/file"; +import Javascript from "tree-sitter-javascript"; +import { ExportMap } from "../splitRunner/types"; + +class JavascriptPlugin implements LanguagePlugin { + parser: Parser; + + constructor() { + this.parser = new Parser(); + this.parser.setLanguage(Javascript); + } + + commentPrefix = "//"; + annotationRegex = /\/\/( *)@nanoapi/; + + getCommentNodes(node: Parser.SyntaxNode) { + const commentQuery = new Parser.Query( + this.parser.getLanguage(), + "(comment) @comment", + ); + const commentCaptures = commentQuery.captures(node); + + return commentCaptures.map((capture) => { + return capture.node; + }); + } + + removeAnnotationFromOtherGroups(sourceCode: string, groupToKeep: Group) { + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + + const tree = this.parser.parse(sourceCode); + + const commentNodes = this.getCommentNodes(tree.rootNode); + + commentNodes.forEach((node) => { + try { + const annotationManager = new AnnotationManager(node.text, this); + + // keep annotation if in the group + if (annotationManager.isInGroup(groupToKeep)) { + return; + } + + // remove the other annotations + let nextNode = node.nextNamedSibling; + // We need to remove all decorators too + while (nextNode && nextNode.type === "decorator") { + nextNode = nextNode.nextNamedSibling; + } + if (!nextNode) { + throw new Error("Could not find next node"); + } + + // Remove this node (comment) and the next node(s) (api endpoint) + indexesToRemove.push({ + startIndex: node.startIndex, + endIndex: nextNode.endIndex, + }); + } catch { + return; + } + }); + + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + + return updatedSourceCode; + } + + getImports(node: Parser.SyntaxNode) { + const imports: Import[] = []; + + const importStatementQuery = new Parser.Query( + this.parser.getLanguage(), + ` + (import_statement) @import + `, + ); + const importStatementCaptures = importStatementQuery.captures(node); + importStatementCaptures.forEach((capture) => { + const importSourceQuery = new Parser.Query( + this.parser.getLanguage(), + ` + source: (string + (string_fragment) @source + ) + `, + ); + const importSourceCaptures = importSourceQuery.captures(capture.node); + if (importSourceCaptures.length === 0) { + throw new Error("Could not find import source"); + } + if (importSourceCaptures.length > 1) { + throw new Error("Found multiple import sources"); + } + const source = importSourceCaptures[0].node.text; + + const importSpecifierIdentifierQuery = new Parser.Query( + this.parser.getLanguage(), + ` + (import_specifier + (identifier) @identifier + ) + `, + ); + const importSpecifierCaptures = importSpecifierIdentifierQuery.captures( + capture.node, + ); + const importSpecifierIdentifiers = importSpecifierCaptures.map( + (capture) => { + return capture.node; + }, + ); + + const importClauseIdentifierQuery = new Parser.Query( + this.parser.getLanguage(), + ` + (import_clause + (identifier) @identifier + ) + `, + ); + const importClauseIdentifierCaptures = + importClauseIdentifierQuery.captures(capture.node); + if (importClauseIdentifierCaptures.length > 1) { + throw new Error("Found multiple import clause identifier"); + } + const importIdentifier = importClauseIdentifierCaptures.length + ? importClauseIdentifierCaptures[0].node + : undefined; + + const nameSpaceimportClauseIdentifierQuery = new Parser.Query( + this.parser.getLanguage(), + ` + (import_clause + (namespace_import + (identifier) @identifier + ) + ) + `, + ); + const nameSpaceimportClauseIdentifierCaptures = + nameSpaceimportClauseIdentifierQuery.captures(capture.node); + if (nameSpaceimportClauseIdentifierCaptures.length > 1) { + throw new Error("Found multiple namespace import clause identifier"); + } + const namespaceImport = nameSpaceimportClauseIdentifierCaptures.length + ? nameSpaceimportClauseIdentifierCaptures[0].node + : undefined; + + imports.push({ + node: capture.node, + source, + importSpecifierIdentifiers, + importIdentifier, + namespaceImport, + }); + }); + + return imports; + } + + _getIdentifierUsagesQuery(identifier: Parser.SyntaxNode) { + return new Parser.Query( + this.parser.getLanguage(), + ` + ( + (identifier) @identifier + (#eq? @identifier "${identifier.text}") + ) + `, + ); + } + + _getImportIdentifiersUsages( + node: Parser.SyntaxNode, + identifier: Parser.SyntaxNode, + ) { + const usageNodes: Parser.SyntaxNode[] = []; + const identifierQuery = this._getIdentifierUsagesQuery(identifier); + + const identifierCaptures = identifierQuery.captures(node); + identifierCaptures.forEach((capture) => { + if (capture.node.id === identifier.id) { + return; + } + + let targetNode = capture.node; + while (true) { + // we can remove from the array + if (targetNode.parent && targetNode.parent.type === "array") { + break; + } + + if ( + targetNode.parent && + targetNode.parent.type === "expression_statement" + ) { + break; + } + + // TODO: add more cases as we encounter them + + if (!targetNode.parent) { + break; + } + targetNode = targetNode.parent; + } + + return usageNodes.push(targetNode); + }); + + return usageNodes; + } + + _getExportIdentifierQuery() { + return new Parser.Query( + this.parser.getLanguage(), + ` + declaration: ([ + (_ + name: (identifier) @identifier + ) + (_ + (_ + name: (identifier) @identifier + ) + ) + ]) + `, + ); + } + + getExports(node: Parser.SyntaxNode) { + const exportQuery = new Parser.Query( + this.parser.getLanguage(), + ` + ( + (export_statement) @export + (#not-match? @export "export default") + ) + `, + ); + + const exportCaptures = exportQuery.captures(node); + const namedExports: { + exportNode: Parser.SyntaxNode; + identifierNode: Parser.SyntaxNode; + }[] = []; + + exportCaptures.forEach((capture) => { + const identifierQuery = this._getExportIdentifierQuery(); + + const identifierCaptures = identifierQuery.captures(capture.node); + if (identifierCaptures.length === 0) { + throw new Error("No identifier found in export statement"); + } + identifierCaptures.forEach((capture) => { + namedExports.push({ + exportNode: capture.node, + identifierNode: capture.node, + }); + }); + }); + + const defaultExportQuery = new Parser.Query( + this.parser.getLanguage(), + ` + ( + (export_statement) @export + (#match? @export "export default") + ) + `, + ); + const defaultExportCaptures = defaultExportQuery.captures(node); + if (defaultExportCaptures.length > 1) { + throw new Error("Found multiple default export. Only one is allowed"); + } + if (defaultExportCaptures.length === 1) { + return { + namedExports, + defaultExport: defaultExportCaptures[0].node, + }; + } + + return { + namedExports, + }; + } + + cleanupInvalidImports( + filePath: string, + sourceCode: string, + exportMap: ExportMap, + ) { + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + + const tree = this.parser.parse(sourceCode); + + const imports = this.getImports(tree.rootNode); + + imports.forEach((depImport) => { + // check if the import is a file, do not process external dependencies + if (depImport.source.startsWith(".")) { + const resolvedPath = resolveFilePath(depImport.source, filePath); + if (!resolvedPath) { + throw new Error("Could not resolve path"); + } + + const exportsForFile = exportMap.get(resolvedPath); + if (!exportsForFile) { + throw new Error("Could not find exports"); + } + + if (depImport.importIdentifier && !exportsForFile.defaultExport) { + let usages = this._getImportIdentifiersUsages( + tree.rootNode, + depImport.importIdentifier, + ); + usages = usages.filter((usage) => { + return usage.id !== depImport.importIdentifier?.id; + }); + indexesToRemove.push({ + startIndex: depImport.node.startIndex, + endIndex: depImport.node.endIndex, + }); + usages.forEach((usage) => { + indexesToRemove.push({ + startIndex: usage.startIndex, + endIndex: usage.endIndex, + }); + }); + } + + depImport.importSpecifierIdentifiers.forEach((importSpecifier) => { + if ( + !exportsForFile.namedExports.find( + (namedExport) => + namedExport.identifierNode.text === importSpecifier.text, + ) + ) { + let usages = this._getImportIdentifiersUsages( + tree.rootNode, + importSpecifier, + ); + usages = usages.filter((usage) => { + return usage.id !== depImport.importIdentifier?.id; + }); + + indexesToRemove.push({ + startIndex: depImport.node.startIndex, + endIndex: depImport.node.endIndex, + }); + usages.forEach((usage) => { + indexesToRemove.push({ + startIndex: usage.startIndex, + endIndex: usage.endIndex, + }); + }); + } + }); + } + }); + + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + + return updatedSourceCode; + } + + cleanupUnusedImports(sourceCode: string) { + const tree = this.parser.parse(sourceCode); + + const imports = this.getImports(tree.rootNode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + + imports.forEach((depImport) => { + const importSpecifierToRemove: Parser.SyntaxNode[] = []; + depImport.importSpecifierIdentifiers.forEach((importSpecifier) => { + let usages = this._getImportIdentifiersUsages( + tree.rootNode, + importSpecifier, + ); + usages = usages.filter((usage) => { + return usage.id !== importSpecifier.id; + }); + + if (usages.length === 0) { + importSpecifierToRemove.push(importSpecifier); + } + }); + + let removeDefaultImport = false; + if (depImport.importIdentifier) { + let usages = this._getImportIdentifiersUsages( + tree.rootNode, + depImport.importIdentifier, + ); + usages = usages.filter((usage) => { + return usage.id !== depImport.importIdentifier?.id; + }); + + if (usages.length === 0) { + removeDefaultImport = true; + } + } + + let removeNameSpaceImport = false; + if (depImport.namespaceImport) { + let usages = this._getImportIdentifiersUsages( + tree.rootNode, + depImport.namespaceImport, + ); + usages = usages.filter((usage) => { + return usage.id !== depImport.importIdentifier?.id; + }); + if (usages.length === 0) { + removeNameSpaceImport = true; + } + } + + if ( + importSpecifierToRemove.length === + depImport.importSpecifierIdentifiers.length && + (removeDefaultImport || removeNameSpaceImport) + ) { + indexesToRemove.push({ + startIndex: depImport.node.startIndex, + endIndex: depImport.node.endIndex, + }); + } else { + importSpecifierToRemove.forEach((importSpecifier) => { + indexesToRemove.push({ + startIndex: importSpecifier.startIndex, + endIndex: importSpecifier.endIndex, + }); + }); + } + }); + + const updatedSourceCode = removeIndexesFromSourceCode( + sourceCode, + indexesToRemove, + ); + + return updatedSourceCode; + } +} + +export default JavascriptPlugin; diff --git a/packages/cli/src/helper/languages/javascript/annotations.test.ts b/packages/cli/src/languages/javascript/annotations.test.ts similarity index 100% rename from packages/cli/src/helper/languages/javascript/annotations.test.ts rename to packages/cli/src/languages/javascript/annotations.test.ts diff --git a/packages/cli/src/languages/javascript/annotations.ts b/packages/cli/src/languages/javascript/annotations.ts new file mode 100644 index 0000000..91a972d --- /dev/null +++ b/packages/cli/src/languages/javascript/annotations.ts @@ -0,0 +1,3 @@ +export const javascriptCommentPrefix = "// "; + +export const javascriptAnnotationRegex = /\/\/( *)@nanoapi/; diff --git a/packages/cli/src/helper/languages/javascript/exports.test.ts b/packages/cli/src/languages/javascript/exports.test.ts similarity index 100% rename from packages/cli/src/helper/languages/javascript/exports.test.ts rename to packages/cli/src/languages/javascript/exports.test.ts diff --git a/packages/cli/src/helper/languages/javascript/imports.test.ts b/packages/cli/src/languages/javascript/imports.test.ts similarity index 100% rename from packages/cli/src/helper/languages/javascript/imports.test.ts rename to packages/cli/src/languages/javascript/imports.test.ts diff --git a/packages/cli/src/languages/types.ts b/packages/cli/src/languages/types.ts new file mode 100644 index 0000000..cd1cd45 --- /dev/null +++ b/packages/cli/src/languages/types.ts @@ -0,0 +1,44 @@ +import { Group } from "../dependencyManager/types"; +import Parser from "tree-sitter"; + +export interface Import { + node: Parser.SyntaxNode; + source: string; + importSpecifierIdentifiers: Parser.SyntaxNode[]; + importIdentifier?: Parser.SyntaxNode; + namespaceImport?: Parser.SyntaxNode; +} + +export interface Export { + namedExports: { + exportNode: Parser.SyntaxNode; + identifierNode: Parser.SyntaxNode; + }[]; + defaultExport?: Parser.SyntaxNode; +} + +export interface LanguagePlugin { + parser: Parser; + + commentPrefix: string; + annotationRegex: RegExp; + + getCommentNodes(node: Parser.SyntaxNode): Parser.SyntaxNode[]; + + removeAnnotationFromOtherGroups( + sourceCode: string, + groupToKeep: Group, + ): string; + + getImports(node: Parser.SyntaxNode): Import[]; + + getExports(node: Parser.SyntaxNode): Export; + + cleanupInvalidImports( + filePath: string, + sourceCode: string, + exportMap: Map, + ): string; + + cleanupUnusedImports(sourceCode: string): string; +} diff --git a/packages/cli/src/languages/typescript.ts b/packages/cli/src/languages/typescript.ts new file mode 100644 index 0000000..b8ce4a6 --- /dev/null +++ b/packages/cli/src/languages/typescript.ts @@ -0,0 +1,45 @@ +import Parser from "tree-sitter"; +import Typescript from "tree-sitter-typescript"; +import JavascriptPlugin from "./javascript"; + +class TypescriptPlugin extends JavascriptPlugin { + constructor() { + super(); + this.parser.setLanguage(Typescript.typescript); + } + + _getIdentifierUsagesQuery(identifier: Parser.SyntaxNode) { + return new Parser.Query( + this.parser.getLanguage(), + ` + ( + ([ + (identifier) + (type_identifier) + ]) @identifier + (#eq? @identifier "${identifier.text}") + ) + `, + ); + } + + _getExportIdentifierQuery() { + return new Parser.Query( + this.parser.getLanguage(), + ` + declaration: ([ + (_ + name: ([(identifier) (type_identifier)]) @identifier + ) + (_ + (_ + name: ([(identifier) (type_identifier)]) @identifier + ) + ) + ]) + `, + ); + } +} + +export default TypescriptPlugin; diff --git a/packages/cli/src/helper/languages/typescript/annotations.test.ts b/packages/cli/src/languages/typescript/annotations.test.ts similarity index 100% rename from packages/cli/src/helper/languages/typescript/annotations.test.ts rename to packages/cli/src/languages/typescript/annotations.test.ts diff --git a/packages/cli/src/languages/typescript/annotations.ts b/packages/cli/src/languages/typescript/annotations.ts new file mode 100644 index 0000000..ec904ad --- /dev/null +++ b/packages/cli/src/languages/typescript/annotations.ts @@ -0,0 +1,8 @@ +import { + javascriptAnnotationRegex, + javascriptCommentPrefix, +} from "../javascript/annotations"; + +export const typescriptCommentPrefix = javascriptCommentPrefix; + +export const typscriptAnnotationRegex = javascriptAnnotationRegex; diff --git a/packages/cli/src/helper/languages/typescript/exports.test.ts b/packages/cli/src/languages/typescript/exports.test.ts similarity index 100% rename from packages/cli/src/helper/languages/typescript/exports.test.ts rename to packages/cli/src/languages/typescript/exports.test.ts diff --git a/packages/cli/src/helper/languages/typescript/imports.test.ts b/packages/cli/src/languages/typescript/imports.test.ts similarity index 100% rename from packages/cli/src/helper/languages/typescript/imports.test.ts rename to packages/cli/src/languages/typescript/imports.test.ts diff --git a/packages/cli/src/splitRunner/splitRunner.ts b/packages/cli/src/splitRunner/splitRunner.ts new file mode 100644 index 0000000..819aa99 --- /dev/null +++ b/packages/cli/src/splitRunner/splitRunner.ts @@ -0,0 +1,190 @@ +import { Group } from "../dependencyManager/types"; +import { removeIndexesFromSourceCode } from "../helper/file"; +import DependencyTreeManager from "../dependencyManager/dependencyManager"; +import { ExportMap, File } from "./types"; +import Parser from "tree-sitter"; +import assert from "assert"; +import { resolveFilePath } from "../helper/file"; +import { getLanguagePluginFromFilePath } from "../languages"; + +class SplitRunner { + private dependencyTreeManager: DependencyTreeManager; + private group: Group; + private files: File[]; + + constructor(dependencyTreeManager: DependencyTreeManager, group: Group) { + this.dependencyTreeManager = dependencyTreeManager; + this.group = group; + this.files = dependencyTreeManager.getFiles(); + } + + #removeAnnotationFromOtherGroups() { + this.files = this.files.map((file) => { + const languagePlugin = getLanguagePluginFromFilePath(file.path); + + const updatedSourceCode = languagePlugin.removeAnnotationFromOtherGroups( + file.sourceCode, + this.group, + ); + return { ...file, sourceCode: updatedSourceCode }; + }); + } + + #getExportMap() { + const exportMap: ExportMap = new Map(); + + this.files.forEach((file) => { + const languagePlugin = getLanguagePluginFromFilePath(file.path); + + const tree = languagePlugin.parser.parse(file.sourceCode); + + const exports = languagePlugin.getExports(tree.rootNode); + + exportMap.set(file.path, exports); + }); + + return exportMap; + } + + #removeInvalidImportsAndUsages(exportMap: ExportMap) { + this.files = this.files.map((file) => { + const languagePlugin = getLanguagePluginFromFilePath(file.path); + + const updatedSourceCode = languagePlugin.cleanupInvalidImports( + file.path, + file.sourceCode, + exportMap, + ); + + return { ...file, sourceCode: updatedSourceCode }; + }); + } + + #removeUnusedImports() { + this.files = this.files.map((file) => { + const languagePlugin = getLanguagePluginFromFilePath(file.path); + + const updatedSourceCode = languagePlugin.cleanupUnusedImports( + file.sourceCode, + ); + + return { ...file, sourceCode: updatedSourceCode }; + }); + } + + #removeUnusedFiles() { + let fileRemoved = true; + while (fileRemoved) { + fileRemoved = false; + + // We always want to keep the entrypoint file. + // It will never be imported anywhere, so we add it now. + const filesToKeep = new Set(); + filesToKeep.add(this.dependencyTreeManager.dependencyTree.path); + + this.files.forEach((file) => { + const languagePlugin = getLanguagePluginFromFilePath(file.path); + + const tree = languagePlugin.parser.parse(file.sourceCode); + + let dependencies = languagePlugin.getImports(tree.rootNode); + dependencies = dependencies.filter((dep) => dep.source.startsWith(".")); + + dependencies.forEach((dep) => { + const resolvedPath = resolveFilePath(dep.source, file.path); + if (resolvedPath) { + filesToKeep.add(resolvedPath); + } + }); + }); + + const previousFilesLength = this.files.length; + + this.files = this.files.filter((file) => { + return filesToKeep.has(file.path); + }); + + if (this.files.length !== previousFilesLength) { + fileRemoved = true; + } + } + } + + #removeUnusedExports(exportMap: ExportMap) { + // TODO + // Step 1, create variable to track which export is user + // Step 2, iterate over all file imports. If the import is used, mark the export as used + // Step 3, iterate over each file, and remove the unused exports + + // Repeat above step until no more unused exports are found + assert(exportMap); + } + + #removeErrors() { + this.files = this.files.map((file) => { + const languagePlugin = getLanguagePluginFromFilePath(file.path); + + const tree = languagePlugin.parser.parse(file.sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + + const query = new Parser.Query( + languagePlugin.parser.getLanguage(), + "(ERROR) @error", + ); + const errorCaptures = query.captures(tree.rootNode); + errorCaptures.forEach((capture) => { + indexesToRemove.push({ + startIndex: capture.node.startIndex, + endIndex: capture.node.endIndex, + }); + }); + + const updatedSourceCode = removeIndexesFromSourceCode( + file.sourceCode, + indexesToRemove, + ); + + return { ...file, sourceCode: updatedSourceCode }; + }); + } + + run() { + console.info("\n"); + console.time("Splitting"); + + console.time("remove annotation from other groups"); + this.#removeAnnotationFromOtherGroups(); + console.timeEnd("remove annotation from other groups"); + + console.time("Get export map"); + const exportMap = this.#getExportMap(); + console.timeEnd("Get export map"); + + console.time("Remove invalid imports and usages"); + this.#removeInvalidImportsAndUsages(exportMap); + console.timeEnd("Remove invalid imports and usages"); + + console.time("Remove unused imports"); + this.#removeUnusedImports(); + console.timeEnd("Remove unused imports"); + + console.time("Remove unused files"); + this.#removeUnusedFiles(); + console.timeEnd("Remove unused files"); + + console.time("Remove unused exports"); + this.#removeUnusedExports(exportMap); + console.timeEnd("Remove unused exports"); + + console.time("Remove errors"); + this.#removeErrors(); + console.timeEnd("Remove errors"); + + console.timeEnd("Splitting"); + + return this.files; + } +} + +export default SplitRunner; diff --git a/packages/cli/src/splitRunner/types.ts b/packages/cli/src/splitRunner/types.ts new file mode 100644 index 0000000..508e9df --- /dev/null +++ b/packages/cli/src/splitRunner/types.ts @@ -0,0 +1,17 @@ +import Parser from "tree-sitter"; + +export interface File { + path: string; + sourceCode: string; +} + +export type ExportMap = Map< + string, + { + namedExports: { + exportNode: Parser.SyntaxNode; + identifierNode: Parser.SyntaxNode; + }[]; + defaultExport?: Parser.SyntaxNode; + } +>;