Merge pull request #398 from code-hike/lighter-annotations

Use lighter annotations

Author: pomber

.github/workflows/bundle_analysis.yml renamed to .github/workflows/bundle_analysis.yml.disabled

@@ -1,6 +1,7 @@
 console.log("one")
 console.log("two")
 console.log("three")
+// mark
 console.log("four")
 console.log("five")
 console.log("six")

packages/mdx/dev/content/assets/bar.js

@@ -112,3 +112,50 @@ function adipiscing(...elit) {
   return elit.map(ipsum => ipsum.sit)
 }
 ```
+
+## Inline comments
+
+```css
+body {
+  margin: 12px; /* mark[11:14] */
+}
+```
+
+## Other syntaxes
+
+```html hello.html
+<!-- mark[6:14] -->
+<div>Code Hike</div>
+```
+
+```mdx
+{/* mark(2) */}
+
+# Lorem
+
+## Foo
+
+# Ipsum {/* mark */}
+```
+
+```jsonc
+{
+  // mark[12:20]
+  "name": "Code Hike"
+}
+```
+
+```lua
+-- mark[8:16]
+print("Code Hike")
+```
+
+```matlab
+% mark[7:15]
+disp('Code Hike')
+```
+
+```lisp
+; mark[9:17]
+(print "Code Hike")
+```

packages/mdx/dev/content/comment-annotations.mdx

@@ -14,6 +14,13 @@ graph LR
     --> End1(End)
 ```
 
+```foobar
+// unknown lang
+function lorem(ipsum, dolor = 1) {
+  return ipsum + dolor;
+}
+```
+
 ```js
 function lorem(ipsum, dolor = 1) {}
 ```

packages/mdx/dev/content/simple-code.mdx

@@ -7,9 +7,3 @@ function foobarloremipsumfoobarloremipsumsitametfoobarloremipsumfoobarloremipsum
   return 8
 }
 ```
-
-<CH.Preview>
-
-Hello
-
-</CH.Preview>

packages/mdx/dev/content/test.mdx

@@ -44,7 +44,7 @@
     "coverage": "vitest run --coverage"
   },
   "dependencies": {
-    "@code-hike/lighter": "0.7.0",
+    "@code-hike/lighter": "0.7.3",
     "node-fetch": "^2.0.0"
   },
   "devDependencies": {

packages/mdx/package.json

@@ -1,3 +1 @@
 export { attacher as remarkCodeHike } from "./remark/transform"
-
-export { highlight } from "./highlighter"

packages/mdx/src/highlighter/index.tsx

@@ -0,0 +1,155 @@
+import { CodeAnnotation } from "smooth-code"
+import {
+  LighterAnnotation,
+  extractLighterAnnotations,
+  parseLighterAnnotations,
+} from "./lighter"
+import { annotationsMap } from "../mdx-client/annotations"
+
+const annotationNames = Object.keys(annotationsMap)
+
+export async function splitCodeAndAnnotations(
+  rawCode: string,
+  lang: string,
+  config: { filepath?: string; autoLink?: boolean }
+): Promise<{
+  code: string
+  annotations: CodeAnnotation[]
+  focus: string
+}> {
+  let { code, annotations } =
+    await extractLighterAnnotations(rawCode, lang, [
+      ...annotationNames,
+      "from",
+      "focus",
+    ])
+
+  // import external code if needed and re-run annotations extraction
+  const fromAnnotations = annotations.filter(
+    a => a.name === "from"
+  )
+  if (fromAnnotations.length === 1) {
+    const fromData = fromAnnotations[0].query?.trim()
+    const [codepath, range] = fromData?.split(/\s+/) || []
+    const externalFileContent = await readFile(
+      codepath,
+      config.filepath,
+      range
+    )
+
+    const result = await extractLighterAnnotations(
+      externalFileContent,
+      lang,
+      [...annotationNames, "focus"]
+    )
+    code = result.code
+    annotations = result.annotations
+  }
+
+  if (config.autoLink) {
+    const autoLinkAnnotations = findLinkAnnotations(code)
+    annotations = [...annotations, ...autoLinkAnnotations]
+  }
+
+  return { code, ...parseLighterAnnotations(annotations) }
+}
+
+async function readFile(
+  externalCodePath: string,
+  mdxFilePath: string,
+  range: string | undefined
+) {
+  const annotationContent =
+    "from " + mdxFilePath + " " + (range || "")
+
+  let fs, path
+
+  try {
+    fs = (await import("fs")).default
+    path = (await import("path")).default
+    if (!fs || !fs.readFileSync || !path || !path.resolve) {
+      throw new Error("fs or path not found")
+    }
+  } catch (e) {
+    e.message = `Code Hike couldn't resolve this annotation:
+${annotationContent}
+Looks like node "fs" and "path" modules are not available.`
+    throw e
+  }
+
+  // if we don't know the path of the mdx file:
+  if (mdxFilePath == null) {
+    throw new Error(
+      `Code Hike couldn't resolve this annotation:
+  ${annotationContent}
+  Someone is calling the mdx compile function without setting the path.
+  Open an issue on CodeHike's repo for help.`
+    )
+  }
+
+  const dir = path.dirname(mdxFilePath)
+  const absoluteCodepath = path.resolve(
+    dir,
+    externalCodePath
+  )
+
+  let content: string
+  try {
+    content = fs.readFileSync(absoluteCodepath, "utf8")
+  } catch (e) {
+    e.message = `Code Hike couldn't resolve this annotation:
+${annotationContent}
+${absoluteCodepath} doesn't exist.`
+    throw e
+  }
+
+  if (range) {
+    const [start, end] = range.split(":")
+    const startLine = parseInt(start)
+    const endLine = parseInt(end)
+    if (isNaN(startLine) || isNaN(endLine)) {
+      throw new Error(
+        `Code Hike couldn't resolve this annotation:
+${annotationContent}
+The range is not valid. Should be something like:
+ ${externalCodePath} 2:5`
+      )
+    }
+    const lines = content.split("\n")
+    content = lines.slice(startLine - 1, endLine).join("\n")
+  }
+
+  return content
+}
+
+const urlRegex = /https?:\/\/[\w\-_.~:/?#[\]@!$&*+,;=%]+/g
+function findLinkAnnotations(
+  code: string
+): LighterAnnotation[] {
+  const lines = code.split("\n")
+
+  const annotations: LighterAnnotation[] = []
+
+  lines.forEach((line, i) => {
+    let match: RegExpExecArray | null
+    while ((match = urlRegex.exec(line)) !== null) {
+      const url = match[0]
+      const start = match.index
+      const end = start + url.length
+
+      annotations.push({
+        name: "link",
+        query: url,
+        ranges: [
+          {
+            lineNumber: i + 1,
+            fromColumn: start + 1,
+            toColumn: end + 1,
+          },
+        ],
+      })
+    }
+  })
+
+  return annotations
+}

packages/mdx/src/index.tsx

@@ -0,0 +1,15 @@
+import { CodeAnnotation } from "../smooth-code"
+import { annotationsMap } from "../mdx-client/annotations"
+
+export function getAnnotationsFromMetastring(
+  options: Record<string, string>
+) {
+  const annotations = [] as CodeAnnotation[]
+  Object.keys(options).forEach(key => {
+    const Component = annotationsMap[key]
+    if (Component) {
+      annotations?.push({ focus: options[key], Component })
+    }
+  })
+  return annotations
+}

packages/mdx/src/remark/__snapshots__/comment-data.test.ts.snap

@@ -1,17 +1,11 @@
-import { highlight } from "../highlighter"
-import { extractLinks } from "./links"
+import { highlight } from "./lighter"
 import { NodeInfo, splitChildren } from "./unist-utils"
 import { CodeStep } from "../smooth-code"
 import { EditorProps } from "../mini-editor"
-import {
-  getAnnotationsFromMetastring,
-  extractAnnotationsFromCode,
-  extractJSXAnnotations,
-} from "./annotations"
-import { Code, mergeFocus } from "../utils"
+import { getAnnotationsFromMetastring } from "./annotations.metastring"
 import { CodeNode, SuperNode } from "./nodes"
 import { CodeHikeConfig } from "./config"
-import { getCommentData } from "./comment-data"
+import { splitCodeAndAnnotations } from "./annotations.comments"
 
 export function isEditorNode(
   node: SuperNode,
@@ -120,52 +114,45 @@ async function mapFile(
 
   const lang = (node.lang as string) || "text"
 
-  let code = await highlight({
-    code: node.value as string,
+  const {
+    code,
+    annotations: commentAnnotations,
+    focus: commentFocus,
+  } = await splitCodeAndAnnotations(
+    node.value as string,
+    lang,
+    config
+  )
+
+  let highlightedCode = await highlight({
+    code,
     lang,
     theme,
   })
 
-  // if the code is a single line with a "from" annotation
-  code = await getCodeFromExternalFileIfNeeded(code, config)
-
-  const [commentAnnotations, commentFocus] =
-    extractAnnotationsFromCode(code, config)
-
   const options = parseMetastring(
     typeof node.meta === "string" ? node.meta : ""
   )
-
   const metaAnnotations = getAnnotationsFromMetastring(
     options as any
   )
 
-  // const linkAnnotations = extractLinks(
-  //   node,
-  //   index,
-  //   parent,
-  //   nodeValue as string
-  // )
-
-  const jsxAnnotations = extractJSXAnnotations(
-    node,
-    index,
-    parent
-  )
-
-  const file = {
+  return {
     ...options,
+    code: highlightedCode,
     focus: mergeFocus(options.focus, commentFocus),
-    code,
     name: options.name || "",
     annotations: [
       ...metaAnnotations,
       ...commentAnnotations,
-      ...jsxAnnotations,
     ],
   }
+}
 
-  return file
+function mergeFocus(fs1: string, fs2: string) {
+  if (!fs1) return fs2 || ""
+  if (!fs2) return fs1 || ""
+  return `${fs1},${fs2}`
 }
 
 type FileOptions = {
@@ -192,87 +179,3 @@ function parseMetastring(
   })
   return { name: name || "", ...options }
 }
-
-async function getCodeFromExternalFileIfNeeded(
-  code: Code,
-  config: CodeHikeConfig
-) {
-  if (code?.lines?.length != 1) {
-    return code
-  }
-
-  const firstLine = code.lines[0]
-  const commentData = getCommentData(firstLine, code.lang)
-
-  if (!commentData || commentData.key != "from") {
-    return code
-  }
-
-  const fileText = firstLine.tokens
-    .map(t => t.content)
-    .join("")
-
-  const [codepath, range] = commentData.data
-    ?.trim()
-    .split(/\s+/)
-
-  let fs, path
-
-  try {
-    fs = (await import("fs")).default
-    path = (await import("path")).default
-    if (!fs || !fs.readFileSync || !path || !path.resolve) {
-      throw new Error("fs or path not found")
-    }
-  } catch (e) {
-    e.message = `Code Hike couldn't resolve this annotation:
-${fileText}
-Looks like node "fs" and "path" modules are not available.`
-    throw e
-  }
-
-  // if we don't know the path of the mdx file:
-  if (config.filepath === undefined) {
-    throw new Error(
-      `Code Hike couldn't resolve this annotation:
-  ${fileText}
-  Someone is calling the mdx compile function without setting the path.
-  Open an issue on CodeHike's repo for help.`
-    )
-  }
-
-  const dir = path.dirname(config.filepath)
-  const absoluteCodepath = path.resolve(dir, codepath)
-
-  let content: string
-  try {
-    content = fs.readFileSync(absoluteCodepath, "utf8")
-  } catch (e) {
-    e.message = `Code Hike couldn't resolve this annotation:
-${fileText}
-${absoluteCodepath} doesn't exist.`
-    throw e
-  }
-
-  if (range) {
-    const [start, end] = range.split(":")
-    const startLine = parseInt(start)
-    const endLine = parseInt(end)
-    if (isNaN(startLine) || isNaN(endLine)) {
-      throw new Error(
-        `Code Hike couldn't resolve this annotation:
-${fileText}
-The range is not valid. Should be something like:
- ${codepath} 2:5`
-      )
-    }
-    const lines = content.split("\n")
-    content = lines.slice(startLine - 1, endLine).join("\n")
-  }
-
-  return await highlight({
-    code: content,
-    lang: code.lang,
-    theme: config.theme,
-  })
-}

packages/mdx/src/remark/annotations.comments.ts

@@ -0,0 +1,105 @@
+import {
+  highlight as light,
+  extractAnnotations,
+  Annotation,
+  LANG_NAMES,
+} from "@code-hike/lighter"
+import { Code } from "../utils"
+import { CodeAnnotation } from "../smooth-code"
+import { annotationsMap } from "../mdx-client/annotations"
+
+export type LighterAnnotation = Annotation
+
+export async function extractLighterAnnotations(
+  codeWithAnnotations: string,
+  lang: string,
+  annotationNames: string[]
+) {
+  return await extractAnnotations(
+    codeWithAnnotations,
+    warnIfUnknownLang(lang),
+    annotationNames
+  )
+}
+
+export function parseLighterAnnotations(
+  annotations: LighterAnnotation[]
+) {
+  const focusList = [] as string[]
+  const codeAnnotations = [] as CodeAnnotation[]
+  annotations.forEach(({ name, query, ranges }) => {
+    ranges.forEach(range => {
+      const focus = rangeString(range)
+      if (name === "focus") {
+        focusList.push(focus)
+      } else {
+        const Component = annotationsMap[name]
+        if (Component) {
+          codeAnnotations.push({
+            Component,
+            focus: focus,
+            data: query,
+          })
+        } else {
+          // this shouldn't happen
+          throw new Error(`Unknown annotation ${name}`)
+        }
+      }
+    })
+  })
+  return {
+    annotations: codeAnnotations,
+    focus: focusList.join(","),
+  }
+}
+
+function rangeString(range: Annotation["ranges"][0]) {
+  if ("lineNumber" in range) {
+    return `${range.lineNumber}[${range.fromColumn}:${range.toColumn}]`
+  } else if (range.fromLineNumber === range.toLineNumber) {
+    return range.fromLineNumber.toString()
+  } else {
+    return `${range.fromLineNumber}:${range.toLineNumber}`
+  }
+}
+
+const warnings = new Set()
+
+function warnIfUnknownLang(lang: string) {
+  if (!LANG_NAMES.includes(lang)) {
+    if (!warnings.has(lang)) {
+      console.warn(
+        "[Code Hike warning]",
+        `${lang} isn't a valid language, no syntax highlighting will be applied.`
+      )
+      warnings.add(lang)
+    }
+    return "text"
+  }
+  return lang
+}
+
+export async function highlight({
+  code,
+  lang,
+  theme,
+}: {
+  code: string
+  lang: string
+  theme: any // TODO type this
+}): Promise<Code> {
+  const r = await light(
+    code,
+    warnIfUnknownLang(lang),
+    theme
+  )
+
+  const lines = r.lines.map(line => ({
+    tokens: line.map(token => ({
+      content: token.content,
+      props: { style: token.style },
+    })),
+  }))
+
+  return { lines, lang: r.lang }
+}

packages/mdx/src/remark/annotations.metastring.ts

@@ -1,5 +1,5 @@
 import { visitAsync, toJSX } from "./unist-utils"
-import { highlight } from "../highlighter"
+import { highlight } from "./lighter"
 import { EditorStep } from "../mini-editor"
 import { Code } from "../utils"
 import { SuperNode, visit } from "./nodes"

packages/mdx/src/remark/annotations.ts

@@ -323,12 +323,3 @@ function toFocusString(focusMap: FocusMap) {
   })
   return parts.join(",")
 }
-
-export function mergeFocus(
-  fs1: FocusString,
-  fs2: FocusString
-) {
-  if (!fs1) return fs2 || ""
-  if (!fs2) return fs1 || ""
-  return `${fs1},${fs2}`
-}

packages/mdx/src/remark/code.ts

@@ -455,10 +455,10 @@
     "@babel/helper-validator-identifier" "^7.19.1"
     to-fast-properties "^2.0.0"
 
-"@code-hike/lighter@0.7.0":
-  version "0.7.0"
-  resolved "https://registry.yarnpkg.com/@code-hike/lighter/-/lighter-0.7.0.tgz#7bb7d59631237d7d2e82434c3ea6fe1875813cb0"
-  integrity sha512-64O07rIORKQLB+5T/GKAmKcD9sC0N9yHFJXa0Hs+0Aee1G+I4bSXxTccuDFP6c/G/3h5Pk7yv7PoX9/SpzaeiQ==
+"@code-hike/lighter@0.7.3":
+  version "0.7.3"
+  resolved "https://registry.yarnpkg.com/@code-hike/lighter/-/lighter-0.7.3.tgz#729a7ab484f11069d258c34dfe39be918f093e39"
+  integrity sha512-IW3nBrnQRSoYDY2suXUjNZegCGVH3ljEH5w51chazy30s103CdpwSwvFya3e6GbY/xfbyAVj+Uyg6HH4L5T6tw==
 
 "@codesandbox/sandpack-client@^0.19.0":
   version "0.19.0"