Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit fecddba

Browse files
authoredJul 10, 2023
Merge pull request #398 from code-hike/lighter-annotations
Use lighter annotations
2 parents b016a0c + 4667c0f commit fecddba

20 files changed

+358
-598
lines changed
 

‎packages/mdx/dev/content/assets/bar.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
console.log("one")
22
console.log("two")
33
console.log("three")
4+
// mark
45
console.log("four")
56
console.log("five")
67
console.log("six")

‎packages/mdx/dev/content/comment-annotations.mdx

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,3 +112,50 @@ function adipiscing(...elit) {
112112
return elit.map(ipsum => ipsum.sit)
113113
}
114114
```
115+
116+
## Inline comments
117+
118+
```css
119+
body {
120+
margin: 12px; /* mark[11:14] */
121+
}
122+
```
123+
124+
## Other syntaxes
125+
126+
```html hello.html
127+
<!-- mark[6:14] -->
128+
<div>Code Hike</div>
129+
```
130+
131+
```mdx
132+
{/* mark(2) */}
133+
134+
# Lorem
135+
136+
## Foo
137+
138+
# Ipsum {/* mark */}
139+
```
140+
141+
```jsonc
142+
{
143+
// mark[12:20]
144+
"name": "Code Hike"
145+
}
146+
```
147+
148+
```lua
149+
-- mark[8:16]
150+
print("Code Hike")
151+
```
152+
153+
```matlab
154+
% mark[7:15]
155+
disp('Code Hike')
156+
```
157+
158+
```lisp
159+
; mark[9:17]
160+
(print "Code Hike")
161+
```

‎packages/mdx/dev/content/simple-code.mdx

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ graph LR
1414
--> End1(End)
1515
```
1616

17+
```foobar
18+
// unknown lang
19+
function lorem(ipsum, dolor = 1) {
20+
return ipsum + dolor;
21+
}
22+
```
23+
1724
```js
1825
function lorem(ipsum, dolor = 1) {}
1926
```

‎packages/mdx/dev/content/test.mdx

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,3 @@ function foobarloremipsumfoobarloremipsumsitametfoobarloremipsumfoobarloremipsum
77
return 8
88
}
99
```
10-
11-
<CH.Preview>
12-
13-
Hello
14-
15-
</CH.Preview>

‎packages/mdx/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
"coverage": "vitest run --coverage"
4545
},
4646
"dependencies": {
47-
"@code-hike/lighter": "0.7.0",
47+
"@code-hike/lighter": "0.7.3",
4848
"node-fetch": "^2.0.0"
4949
},
5050
"devDependencies": {

‎packages/mdx/src/highlighter/index.tsx

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎packages/mdx/src/index.tsx

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1 @@
11
export { attacher as remarkCodeHike } from "./remark/transform"
2-
3-
export { highlight } from "./highlighter"

‎packages/mdx/src/remark/__snapshots__/comment-data.test.ts.snap

Lines changed: 0 additions & 73 deletions
This file was deleted.
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import { CodeAnnotation } from "smooth-code"
2+
import {
3+
LighterAnnotation,
4+
extractLighterAnnotations,
5+
parseLighterAnnotations,
6+
} from "./lighter"
7+
import { annotationsMap } from "../mdx-client/annotations"
8+
9+
const annotationNames = Object.keys(annotationsMap)
10+
11+
export async function splitCodeAndAnnotations(
12+
rawCode: string,
13+
lang: string,
14+
config: { filepath?: string; autoLink?: boolean }
15+
): Promise<{
16+
code: string
17+
annotations: CodeAnnotation[]
18+
focus: string
19+
}> {
20+
let { code, annotations } =
21+
await extractLighterAnnotations(rawCode, lang, [
22+
...annotationNames,
23+
"from",
24+
"focus",
25+
])
26+
27+
// import external code if needed and re-run annotations extraction
28+
const fromAnnotations = annotations.filter(
29+
a => a.name === "from"
30+
)
31+
if (fromAnnotations.length === 1) {
32+
const fromData = fromAnnotations[0].query?.trim()
33+
const [codepath, range] = fromData?.split(/\s+/) || []
34+
const externalFileContent = await readFile(
35+
codepath,
36+
config.filepath,
37+
range
38+
)
39+
40+
const result = await extractLighterAnnotations(
41+
externalFileContent,
42+
lang,
43+
[...annotationNames, "focus"]
44+
)
45+
code = result.code
46+
annotations = result.annotations
47+
}
48+
49+
if (config.autoLink) {
50+
const autoLinkAnnotations = findLinkAnnotations(code)
51+
annotations = [...annotations, ...autoLinkAnnotations]
52+
}
53+
54+
return { code, ...parseLighterAnnotations(annotations) }
55+
}
56+
57+
async function readFile(
58+
externalCodePath: string,
59+
mdxFilePath: string,
60+
range: string | undefined
61+
) {
62+
const annotationContent =
63+
"from " + mdxFilePath + " " + (range || "")
64+
65+
let fs, path
66+
67+
try {
68+
fs = (await import("fs")).default
69+
path = (await import("path")).default
70+
if (!fs || !fs.readFileSync || !path || !path.resolve) {
71+
throw new Error("fs or path not found")
72+
}
73+
} catch (e) {
74+
e.message = `Code Hike couldn't resolve this annotation:
75+
${annotationContent}
76+
Looks like node "fs" and "path" modules are not available.`
77+
throw e
78+
}
79+
80+
// if we don't know the path of the mdx file:
81+
if (mdxFilePath == null) {
82+
throw new Error(
83+
`Code Hike couldn't resolve this annotation:
84+
${annotationContent}
85+
Someone is calling the mdx compile function without setting the path.
86+
Open an issue on CodeHike's repo for help.`
87+
)
88+
}
89+
90+
const dir = path.dirname(mdxFilePath)
91+
const absoluteCodepath = path.resolve(
92+
dir,
93+
externalCodePath
94+
)
95+
96+
let content: string
97+
try {
98+
content = fs.readFileSync(absoluteCodepath, "utf8")
99+
} catch (e) {
100+
e.message = `Code Hike couldn't resolve this annotation:
101+
${annotationContent}
102+
${absoluteCodepath} doesn't exist.`
103+
throw e
104+
}
105+
106+
if (range) {
107+
const [start, end] = range.split(":")
108+
const startLine = parseInt(start)
109+
const endLine = parseInt(end)
110+
if (isNaN(startLine) || isNaN(endLine)) {
111+
throw new Error(
112+
`Code Hike couldn't resolve this annotation:
113+
${annotationContent}
114+
The range is not valid. Should be something like:
115+
${externalCodePath} 2:5`
116+
)
117+
}
118+
const lines = content.split("\n")
119+
content = lines.slice(startLine - 1, endLine).join("\n")
120+
}
121+
122+
return content
123+
}
124+
125+
const urlRegex = /https?:\/\/[\w\-_.~:/?#[\]@!$&*+,;=%]+/g
126+
function findLinkAnnotations(
127+
code: string
128+
): LighterAnnotation[] {
129+
const lines = code.split("\n")
130+
131+
const annotations: LighterAnnotation[] = []
132+
133+
lines.forEach((line, i) => {
134+
let match: RegExpExecArray | null
135+
while ((match = urlRegex.exec(line)) !== null) {
136+
const url = match[0]
137+
const start = match.index
138+
const end = start + url.length
139+
140+
annotations.push({
141+
name: "link",
142+
query: url,
143+
ranges: [
144+
{
145+
lineNumber: i + 1,
146+
fromColumn: start + 1,
147+
toColumn: end + 1,
148+
},
149+
],
150+
})
151+
}
152+
})
153+
154+
return annotations
155+
}
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { CodeAnnotation } from "../smooth-code"
2+
import { annotationsMap } from "../mdx-client/annotations"
3+
4+
export function getAnnotationsFromMetastring(
5+
options: Record<string, string>
6+
) {
7+
const annotations = [] as CodeAnnotation[]
8+
Object.keys(options).forEach(key => {
9+
const Component = annotationsMap[key]
10+
if (Component) {
11+
annotations?.push({ focus: options[key], Component })
12+
}
13+
})
14+
return annotations
15+
}

‎packages/mdx/src/remark/annotations.ts

Lines changed: 0 additions & 139 deletions
This file was deleted.

‎packages/mdx/src/remark/code.ts

Lines changed: 22 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,11 @@
1-
import { highlight } from "../highlighter"
2-
import { extractLinks } from "./links"
1+
import { highlight } from "./lighter"
32
import { NodeInfo, splitChildren } from "./unist-utils"
43
import { CodeStep } from "../smooth-code"
54
import { EditorProps } from "../mini-editor"
6-
import {
7-
getAnnotationsFromMetastring,
8-
extractAnnotationsFromCode,
9-
extractJSXAnnotations,
10-
} from "./annotations"
11-
import { Code, mergeFocus } from "../utils"
5+
import { getAnnotationsFromMetastring } from "./annotations.metastring"
126
import { CodeNode, SuperNode } from "./nodes"
137
import { CodeHikeConfig } from "./config"
14-
import { getCommentData } from "./comment-data"
8+
import { splitCodeAndAnnotations } from "./annotations.comments"
159

1610
export function isEditorNode(
1711
node: SuperNode,
@@ -120,52 +114,45 @@ async function mapFile(
120114

121115
const lang = (node.lang as string) || "text"
122116

123-
let code = await highlight({
124-
code: node.value as string,
117+
const {
118+
code,
119+
annotations: commentAnnotations,
120+
focus: commentFocus,
121+
} = await splitCodeAndAnnotations(
122+
node.value as string,
123+
lang,
124+
config
125+
)
126+
127+
let highlightedCode = await highlight({
128+
code,
125129
lang,
126130
theme,
127131
})
128132

129-
// if the code is a single line with a "from" annotation
130-
code = await getCodeFromExternalFileIfNeeded(code, config)
131-
132-
const [commentAnnotations, commentFocus] =
133-
extractAnnotationsFromCode(code, config)
134-
135133
const options = parseMetastring(
136134
typeof node.meta === "string" ? node.meta : ""
137135
)
138-
139136
const metaAnnotations = getAnnotationsFromMetastring(
140137
options as any
141138
)
142139

143-
// const linkAnnotations = extractLinks(
144-
// node,
145-
// index,
146-
// parent,
147-
// nodeValue as string
148-
// )
149-
150-
const jsxAnnotations = extractJSXAnnotations(
151-
node,
152-
index,
153-
parent
154-
)
155-
156-
const file = {
140+
return {
157141
...options,
142+
code: highlightedCode,
158143
focus: mergeFocus(options.focus, commentFocus),
159-
code,
160144
name: options.name || "",
161145
annotations: [
162146
...metaAnnotations,
163147
...commentAnnotations,
164-
...jsxAnnotations,
165148
],
166149
}
150+
}
167151

168-
return file
152+
function mergeFocus(fs1: string, fs2: string) {
153+
if (!fs1) return fs2 || ""
154+
if (!fs2) return fs1 || ""
155+
return `${fs1},${fs2}`
169156
}
170157

171158
type FileOptions = {
@@ -192,87 +179,3 @@ function parseMetastring(
192179
})
193180
return { name: name || "", ...options }
194181
}
195-
196-
async function getCodeFromExternalFileIfNeeded(
197-
code: Code,
198-
config: CodeHikeConfig
199-
) {
200-
if (code?.lines?.length != 1) {
201-
return code
202-
}
203-
204-
const firstLine = code.lines[0]
205-
const commentData = getCommentData(firstLine, code.lang)
206-
207-
if (!commentData || commentData.key != "from") {
208-
return code
209-
}
210-
211-
const fileText = firstLine.tokens
212-
.map(t => t.content)
213-
.join("")
214-
215-
const [codepath, range] = commentData.data
216-
?.trim()
217-
.split(/\s+/)
218-
219-
let fs, path
220-
221-
try {
222-
fs = (await import("fs")).default
223-
path = (await import("path")).default
224-
if (!fs || !fs.readFileSync || !path || !path.resolve) {
225-
throw new Error("fs or path not found")
226-
}
227-
} catch (e) {
228-
e.message = `Code Hike couldn't resolve this annotation:
229-
${fileText}
230-
Looks like node "fs" and "path" modules are not available.`
231-
throw e
232-
}
233-
234-
// if we don't know the path of the mdx file:
235-
if (config.filepath === undefined) {
236-
throw new Error(
237-
`Code Hike couldn't resolve this annotation:
238-
${fileText}
239-
Someone is calling the mdx compile function without setting the path.
240-
Open an issue on CodeHike's repo for help.`
241-
)
242-
}
243-
244-
const dir = path.dirname(config.filepath)
245-
const absoluteCodepath = path.resolve(dir, codepath)
246-
247-
let content: string
248-
try {
249-
content = fs.readFileSync(absoluteCodepath, "utf8")
250-
} catch (e) {
251-
e.message = `Code Hike couldn't resolve this annotation:
252-
${fileText}
253-
${absoluteCodepath} doesn't exist.`
254-
throw e
255-
}
256-
257-
if (range) {
258-
const [start, end] = range.split(":")
259-
const startLine = parseInt(start)
260-
const endLine = parseInt(end)
261-
if (isNaN(startLine) || isNaN(endLine)) {
262-
throw new Error(
263-
`Code Hike couldn't resolve this annotation:
264-
${fileText}
265-
The range is not valid. Should be something like:
266-
${codepath} 2:5`
267-
)
268-
}
269-
const lines = content.split("\n")
270-
content = lines.slice(startLine - 1, endLine).join("\n")
271-
}
272-
273-
return await highlight({
274-
code: content,
275-
lang: code.lang,
276-
theme: config.theme,
277-
})
278-
}

‎packages/mdx/src/remark/comment-data.test.ts

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎packages/mdx/src/remark/comment-data.ts

Lines changed: 0 additions & 91 deletions
This file was deleted.

‎packages/mdx/src/remark/lighter.ts

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import {
2+
highlight as light,
3+
extractAnnotations,
4+
Annotation,
5+
LANG_NAMES,
6+
} from "@code-hike/lighter"
7+
import { Code } from "../utils"
8+
import { CodeAnnotation } from "../smooth-code"
9+
import { annotationsMap } from "../mdx-client/annotations"
10+
11+
export type LighterAnnotation = Annotation
12+
13+
export async function extractLighterAnnotations(
14+
codeWithAnnotations: string,
15+
lang: string,
16+
annotationNames: string[]
17+
) {
18+
return await extractAnnotations(
19+
codeWithAnnotations,
20+
warnIfUnknownLang(lang),
21+
annotationNames
22+
)
23+
}
24+
25+
export function parseLighterAnnotations(
26+
annotations: LighterAnnotation[]
27+
) {
28+
const focusList = [] as string[]
29+
const codeAnnotations = [] as CodeAnnotation[]
30+
annotations.forEach(({ name, query, ranges }) => {
31+
ranges.forEach(range => {
32+
const focus = rangeString(range)
33+
if (name === "focus") {
34+
focusList.push(focus)
35+
} else {
36+
const Component = annotationsMap[name]
37+
if (Component) {
38+
codeAnnotations.push({
39+
Component,
40+
focus: focus,
41+
data: query,
42+
})
43+
} else {
44+
// this shouldn't happen
45+
throw new Error(`Unknown annotation ${name}`)
46+
}
47+
}
48+
})
49+
})
50+
return {
51+
annotations: codeAnnotations,
52+
focus: focusList.join(","),
53+
}
54+
}
55+
56+
function rangeString(range: Annotation["ranges"][0]) {
57+
if ("lineNumber" in range) {
58+
return `${range.lineNumber}[${range.fromColumn}:${range.toColumn}]`
59+
} else if (range.fromLineNumber === range.toLineNumber) {
60+
return range.fromLineNumber.toString()
61+
} else {
62+
return `${range.fromLineNumber}:${range.toLineNumber}`
63+
}
64+
}
65+
66+
const warnings = new Set()
67+
68+
function warnIfUnknownLang(lang: string) {
69+
if (!LANG_NAMES.includes(lang)) {
70+
if (!warnings.has(lang)) {
71+
console.warn(
72+
"[Code Hike warning]",
73+
`${lang} isn't a valid language, no syntax highlighting will be applied.`
74+
)
75+
warnings.add(lang)
76+
}
77+
return "text"
78+
}
79+
return lang
80+
}
81+
82+
export async function highlight({
83+
code,
84+
lang,
85+
theme,
86+
}: {
87+
code: string
88+
lang: string
89+
theme: any // TODO type this
90+
}): Promise<Code> {
91+
const r = await light(
92+
code,
93+
warnIfUnknownLang(lang),
94+
theme
95+
)
96+
97+
const lines = r.lines.map(line => ({
98+
tokens: line.map(token => ({
99+
content: token.content,
100+
props: { style: token.style },
101+
})),
102+
}))
103+
104+
return { lines, lang: r.lang }
105+
}

‎packages/mdx/src/remark/links.tsx

Lines changed: 0 additions & 63 deletions
This file was deleted.

‎packages/mdx/src/remark/transform.inline-code.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { visitAsync, toJSX } from "./unist-utils"
2-
import { highlight } from "../highlighter"
2+
import { highlight } from "./lighter"
33
import { EditorStep } from "../mini-editor"
44
import { Code } from "../utils"
55
import { SuperNode, visit } from "./nodes"

‎packages/mdx/src/utils/focus.ts

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -323,12 +323,3 @@ function toFocusString(focusMap: FocusMap) {
323323
})
324324
return parts.join(",")
325325
}
326-
327-
export function mergeFocus(
328-
fs1: FocusString,
329-
fs2: FocusString
330-
) {
331-
if (!fs1) return fs2 || ""
332-
if (!fs2) return fs1 || ""
333-
return `${fs1},${fs2}`
334-
}

‎yarn.lock

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -455,10 +455,10 @@
455455
"@babel/helper-validator-identifier" "^7.19.1"
456456
to-fast-properties "^2.0.0"
457457

458-
"@code-hike/lighter@0.7.0":
459-
version "0.7.0"
460-
resolved "https://registry.yarnpkg.com/@code-hike/lighter/-/lighter-0.7.0.tgz#7bb7d59631237d7d2e82434c3ea6fe1875813cb0"
461-
integrity sha512-64O07rIORKQLB+5T/GKAmKcD9sC0N9yHFJXa0Hs+0Aee1G+I4bSXxTccuDFP6c/G/3h5Pk7yv7PoX9/SpzaeiQ==
458+
"@code-hike/lighter@0.7.3":
459+
version "0.7.3"
460+
resolved "https://registry.yarnpkg.com/@code-hike/lighter/-/lighter-0.7.3.tgz#729a7ab484f11069d258c34dfe39be918f093e39"
461+
integrity sha512-IW3nBrnQRSoYDY2suXUjNZegCGVH3ljEH5w51chazy30s103CdpwSwvFya3e6GbY/xfbyAVj+Uyg6HH4L5T6tw==
462462

463463
"@codesandbox/sandpack-client@^0.19.0":
464464
version "0.19.0"

0 commit comments

Comments
 (0)
Please sign in to comment.