Skip to content

Commit e2bbc8b

Browse files
florianbgtflorianbgt
committed
improve dependency tree and file edits
1 parent 4bbfc78 commit e2bbc8b

File tree

18 files changed

+420
-1085
lines changed

18 files changed

+420
-1085
lines changed

‎package-lock.json

Lines changed: 52 additions & 691 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/cli/package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@
7575
},
7676
"dependencies": {
7777
"axios": "^1.7.7",
78-
"dependency-tree": "^11.0.1",
7978
"express": "^4.21.1",
8079
"http-proxy-middleware": "^3.0.3",
8180
"openai": "^4.67.2",

‎packages/cli/src/api/scan.ts

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
import { z } from "zod";
2-
import { getDependencyTree } from "../helper/dependencies";
3-
import { ScanCodebaseResponsePayload } from "../helper/payloads";
4-
import { getEndpontsFromTree } from "../helper/tree";
2+
import {
3+
getDependencyTree,
4+
getEndpontsFromTree,
5+
} from "../helper/dependencyTree";
56
import { scanSchema } from "./helpers/validation";
67

7-
export function scan(
8-
payload: z.infer<typeof scanSchema>,
9-
): ScanCodebaseResponsePayload {
8+
export function scan(payload: z.infer<typeof scanSchema>) {
109
const tree = getDependencyTree(payload.entrypointPath);
1110

1211
const endpoints = getEndpontsFromTree(tree);

‎packages/cli/src/api/split.ts

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
import fs from "fs";
22
import path from "path";
3-
import { getDependencyTree } from "../helper/dependencies";
3+
import {
4+
getDependencyTree,
5+
getEndpontsFromTree,
6+
getGroupsFromEndpoints,
7+
} from "../helper/dependencyTree";
48
import { cleanupOutputDir, createOutputDir } from "../helper/file";
5-
import { SplitCodebaseResponsePayload } from "../helper/payloads";
6-
import { getEndpontsFromTree, splitPath } from "../helper/tree";
9+
import { createSplit } from "../helper/split";
710
import { splitSchema } from "./helpers/validation";
811
import { z } from "zod";
912
import { GroupMap } from "../helper/types";
10-
import { getGroupsFromEndpoints } from "../helper/groups";
1113

12-
export function split(
13-
payload: z.infer<typeof splitSchema>,
14-
): SplitCodebaseResponsePayload {
14+
export function split(payload: z.infer<typeof splitSchema>) {
1515
let groupIndex = 0;
1616
const groupMap: GroupMap = {};
1717

@@ -32,7 +32,7 @@ export function split(
3232

3333
// Process each endpoint for splitting
3434
for (const group of groups) {
35-
splitPath(
35+
createSplit(
3636
group,
3737
payload.outputDir,
3838
payload.entrypointPath,

‎packages/cli/src/api/sync.ts

Lines changed: 50 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
import { z } from "zod";
22
import { syncSchema } from "./helpers/validation";
33
import fs from "fs";
4-
import { Dependencies } from "../helper/types";
54
import {
65
getNanoApiAnnotationFromCommentValue,
7-
getParserLanguageFromFile,
86
replaceCommentFromAnnotation,
9-
} from "../helper/file";
10-
import { getDependencyTree } from "../helper/dependencies";
7+
} from "../helper/annotations";
8+
import {
9+
getDependencyTree,
10+
getEndpontsFromTree,
11+
} from "../helper/dependencyTree";
1112
import Parser from "tree-sitter";
12-
import { getEndpontsFromTree } from "../helper/tree";
13+
import { getParserLanguageFromFile } from "../helper/treeSitter";
14+
import { replaceIndexesFromSourceCode } from "../helper/cleanup";
1315

1416
export function sync(payload: z.infer<typeof syncSchema>) {
1517
const tree = getDependencyTree(payload.entrypointPath);
@@ -31,53 +33,53 @@ export function sync(payload: z.infer<typeof syncSchema>) {
3133
return endpoint;
3234
});
3335

34-
function iterateOverTreeAndUpdateContent(tree: Dependencies) {
35-
for (const [filePath, value] of Object.entries(tree)) {
36-
let sourceCode = fs.readFileSync(filePath, "utf-8");
37-
38-
updatedEndpoints.forEach((endpoint) => {
39-
const language = getParserLanguageFromFile(filePath);
40-
const parser = new Parser();
41-
parser.setLanguage(language);
42-
43-
const tree = parser.parse(sourceCode);
44-
45-
function traverse(node: Parser.SyntaxNode) {
46-
if (node.type === "comment") {
47-
const comment = node.text;
48-
49-
const annotation = getNanoApiAnnotationFromCommentValue(comment);
50-
51-
if (annotation) {
52-
if (
53-
annotation.path === endpoint.path &&
54-
annotation.method === endpoint.method
55-
) {
56-
annotation.group = endpoint.group;
57-
const updatedComment = replaceCommentFromAnnotation(
58-
comment,
59-
annotation,
60-
);
61-
// Replace the comment in the source code
62-
sourceCode = sourceCode.replace(comment, updatedComment);
63-
}
64-
}
65-
}
66-
node.children.forEach((child) => traverse(child));
67-
}
36+
updatedEndpoints.forEach((endpoint) => {
37+
const language = getParserLanguageFromFile(endpoint.filePath);
38+
const parser = new Parser();
39+
parser.setLanguage(language);
40+
41+
let sourceCode = fs.readFileSync(endpoint.filePath, "utf-8");
42+
43+
const tree = parser.parse(sourceCode);
44+
45+
const indexesToReplace: {
46+
startIndex: number;
47+
endIndex: number;
48+
text: string;
49+
}[] = [];
6850

69-
traverse(tree.rootNode);
70-
});
51+
function traverse(node: Parser.SyntaxNode) {
52+
if (node.type === "comment") {
53+
const comment = node.text;
7154

72-
// update the file
73-
fs.writeFileSync(filePath, sourceCode, "utf-8");
55+
const annotation = getNanoApiAnnotationFromCommentValue(comment);
7456

75-
// Recursively process the tree
76-
if (typeof value !== "string") {
77-
iterateOverTreeAndUpdateContent(value);
57+
if (annotation) {
58+
if (
59+
annotation.path === endpoint.path &&
60+
annotation.method === endpoint.method
61+
) {
62+
annotation.group = endpoint.group;
63+
const updatedComment = replaceCommentFromAnnotation(
64+
comment,
65+
annotation,
66+
);
67+
68+
indexesToReplace.push({
69+
startIndex: node.startIndex,
70+
endIndex: node.endIndex,
71+
text: updatedComment,
72+
});
73+
}
74+
}
7875
}
76+
node.children.forEach((child) => traverse(child));
7977
}
80-
}
8178

82-
iterateOverTreeAndUpdateContent(tree);
79+
traverse(tree.rootNode);
80+
81+
sourceCode = replaceIndexesFromSourceCode(sourceCode, indexesToReplace);
82+
83+
fs.writeFileSync(endpoint.filePath, sourceCode, "utf-8");
84+
});
8385
}

‎packages/cli/src/commands/annotate.ts

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import fs from "fs";
2-
import { getDependencyTree } from "../helper/dependencies";
2+
import { getDependencyTree } from "../helper/dependencyTree";
33
import OpenAI from "openai";
4-
import { Dependencies } from "../helper/types";
4+
import { DependencyTree } from "../helper/types";
55

66
export default async function annotateOpenAICommandHandler(
77
entrypoint: string, // Path to the entrypoint file
@@ -33,20 +33,22 @@ export default async function annotateOpenAICommandHandler(
3333
I want you to understand what the application is doing and where each endpoint is.`,
3434
});
3535

36-
function iterateOverTree(tree: Dependencies) {
37-
for (const [filePath, value] of Object.entries(tree)) {
38-
// TODO send chunks of each file for big files to respect the openAI limits
39-
const content = fs.readFileSync(filePath, "utf8");
36+
function iterateOverTree(tree: DependencyTree) {
37+
messages.push({
38+
role: "user",
39+
content: `File: ${tree.path}
40+
Content: ${tree.sourceCode}`,
41+
});
42+
43+
tree.children.forEach((child) => {
4044
messages.push({
4145
role: "user",
42-
content: `File: ${filePath}
43-
Content: ${content}`,
46+
content: `File: ${child.path}
47+
Content: ${child.sourceCode}`,
4448
});
4549

46-
if (typeof value !== "string") {
47-
iterateOverTree(value);
48-
}
49-
}
50+
iterateOverTree(child);
51+
});
5052
}
5153
iterateOverTree(tree);
5254

‎packages/cli/src/commands/split.ts

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
import path from "path";
22
import fs from "fs";
3-
import { getDependencyTree } from "../helper/dependencies";
3+
import {
4+
getDependencyTree,
5+
getEndpontsFromTree,
6+
getGroupsFromEndpoints,
7+
} from "../helper/dependencyTree";
48
import { cleanupOutputDir, createOutputDir } from "../helper/file";
5-
import { getEndpontsFromTree, splitPath } from "../helper/tree";
9+
import { createSplit } from "../helper/split";
610
import { GroupMap } from "../helper/types";
7-
import { getGroupsFromEndpoints } from "../helper/groups";
811

912
export default function splitCommandHandler(
1013
entrypointPath: string, // Path to the entrypoint file
@@ -26,7 +29,7 @@ export default function splitCommandHandler(
2629

2730
// Process each endpoint for splitting
2831
for (const group of groups) {
29-
splitPath(group, outputDir, entrypointPath, groupMap, groupIndex);
32+
createSplit(group, outputDir, entrypointPath, groupMap, groupIndex);
3033
groupIndex++;
3134
}
3235

‎packages/cli/src/helper/annotations.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import { NanoAPIAnnotation } from "./types";
2+
3+
export function getNanoApiAnnotationFromCommentValue(comment: string) {
4+
const nanoapiRegex = /@nanoapi|((method|path|group):([^ ]+))/g;
5+
const matches = comment.match(nanoapiRegex);
6+
// remove first match, which is the @nanoapi identifier
7+
matches?.shift();
8+
9+
if (matches && matches.length > 0) {
10+
return matches.reduce((acc, match) => {
11+
// key, first element when split with ":"
12+
const key = match.split(":")[0];
13+
// value, everything else
14+
const value = match.split(":").slice(1).join(":");
15+
return { ...acc, [key]: value };
16+
}, {} as NanoAPIAnnotation);
17+
}
18+
19+
return null;
20+
}
21+
22+
export function replaceCommentFromAnnotation(
23+
comment: string,
24+
annotation: NanoAPIAnnotation,
25+
) {
26+
const commentRegex = /@nanoapi\s*(.*)/g;
27+
28+
// Construct the new annotation string
29+
let newAnnotation = "@nanoapi";
30+
if (annotation.method) {
31+
newAnnotation += ` method:${annotation.method}`;
32+
}
33+
if (annotation.path) {
34+
newAnnotation += ` path:${annotation.path}`;
35+
}
36+
if (annotation.group) {
37+
newAnnotation += ` group:${annotation.group}`;
38+
}
39+
40+
// Replace the old annotation with the new annotation
41+
const updatedComment = comment.replace(commentRegex, newAnnotation);
42+
43+
return updatedComment;
44+
}

‎packages/cli/src/helper/cleanup.ts

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import fs from "fs";
2+
import Parser from "tree-sitter";
3+
import { cleanupJavascriptFile } from "./languages/javascript/cleanup";
4+
import { extractJavascriptFileImports } from "./languages/javascript/imports";
5+
6+
import { resolveFilePath } from "./file";
7+
import { Group } from "./types";
8+
import { getParserLanguageFromFile } from "./treeSitter";
9+
10+
export function cleanupFile(filePath: string, group: Group) {
11+
const language = getParserLanguageFromFile(filePath);
12+
const parser = new Parser();
13+
parser.setLanguage(language);
14+
15+
const sourceCode = fs.readFileSync(filePath, "utf8");
16+
17+
let dependencies: string[] = [];
18+
if (["javascript", "typescript"].includes(language.name)) {
19+
dependencies = extractJavascriptFileImports(parser, sourceCode);
20+
} else {
21+
throw new Error(`Unsupported language: ${language.language}`);
22+
}
23+
24+
// Check if we can resolve the path for each dependency. If we cannot, we need to remove it
25+
const invalidDependencies = dependencies.filter(
26+
(dep) => !resolveFilePath(dep, filePath),
27+
);
28+
29+
let updatedSourceCode: string;
30+
31+
if (["javascript", "typescript"].includes(language.name)) {
32+
updatedSourceCode = cleanupJavascriptFile(
33+
parser,
34+
sourceCode,
35+
group,
36+
invalidDependencies,
37+
);
38+
} else {
39+
throw new Error(`Unsupported language: ${language.language}`);
40+
}
41+
42+
fs.writeFileSync(filePath, updatedSourceCode, "utf8");
43+
}
44+
45+
export function removeIndexesFromSourceCode(
46+
sourceCode: string,
47+
indexesToRemove: { startIndex: number; endIndex: number }[],
48+
) {
49+
let newSourceCode = sourceCode;
50+
51+
// sort to start removing from the of the file end
52+
indexesToRemove.sort((a, b) => b.startIndex - a.startIndex);
53+
54+
indexesToRemove.forEach(({ startIndex, endIndex }) => {
55+
newSourceCode =
56+
newSourceCode.slice(0, startIndex) + newSourceCode.slice(endIndex);
57+
});
58+
59+
return newSourceCode;
60+
}
61+
62+
export function replaceIndexesFromSourceCode(
63+
sourceCode: string,
64+
indexesToReplace: { startIndex: number; endIndex: number; text: string }[],
65+
) {
66+
// sort to start removing from the end of the file
67+
indexesToReplace.sort((a, b) => b.startIndex - a.startIndex);
68+
69+
indexesToReplace.forEach(({ startIndex, endIndex, text }) => {
70+
sourceCode =
71+
sourceCode.slice(0, startIndex) + text + sourceCode.slice(endIndex);
72+
});
73+
74+
return sourceCode;
75+
}

0 commit comments

Comments
 (0)