diff --git a/.gitignore b/.gitignore index d49650146..0149327e1 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,19 @@ web-build/ **/ios/generated **/android/generated +# Vendored tree-sitter sources (machine-generated C, ~178 MB total). Nothing here +# is tracked: the runtime is fetched from the pinned GitHub release tarball, the +# grammar parse tables come from the grammar devDependencies, and the default +# registry is codegen'd from them. All three are restored by +# vendor/vendor-grammars.mjs via the `prepare` hook (`yarn prepare`) and baked +# into the npm tarball on prepack. See docs and vendor/grammar-versions.json. +packages/core/cpp/highlight/vendor/tree-sitter/ +packages/core/cpp/highlight/vendor/grammars/ +packages/core/cpp/highlight/vendor/generated/ +# Registry for a custom ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES set (iOS); kept +# out of the committed generated/ dir so a custom build never clobbers the default. +packages/core/cpp/highlight/vendor/generated-custom/ + # React Native Nitro Modules nitrogen/ diff --git a/.maestro/enrichedMarkdownText/screenshots/android/blockquote_code_block_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/android/blockquote_code_block_combo_display.png index ff6448911..68fe46408 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/android/blockquote_code_block_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/android/blockquote_code_block_combo_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/android/code_block_display.png b/.maestro/enrichedMarkdownText/screenshots/android/code_block_display.png index 30ca57f4d..e69de29bb 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/android/code_block_display.png and b/.maestro/enrichedMarkdownText/screenshots/android/code_block_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/android/code_block_math_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/android/code_block_math_combo_display.png index 9610fc76b..51e51dfb6 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/android/code_block_math_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/android/code_block_math_combo_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/android/header_code_block_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/android/header_code_block_combo_display.png index 751304f0b..02b4ffc8e 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/android/header_code_block_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/android/header_code_block_combo_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_loose_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_loose_combo_display.png index ec53b78d4..aba1250a9 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_loose_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_loose_combo_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_tight_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_tight_combo_display.png index ec53b78d4..aba1250a9 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_tight_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/android/list_code_block_tight_combo_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/android/task_list_code_block_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/android/task_list_code_block_combo_display.png index e1dcc8927..e69de29bb 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/android/task_list_code_block_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/android/task_list_code_block_combo_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/ios/code_block_math_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/ios/code_block_math_combo_display.png index 5c4c6bc35..1540d5a9e 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/ios/code_block_math_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/ios/code_block_math_combo_display.png differ diff --git a/.maestro/enrichedMarkdownText/screenshots/ios/header_code_block_combo_display.png b/.maestro/enrichedMarkdownText/screenshots/ios/header_code_block_combo_display.png index 01ab42bc5..e64dc8a9c 100644 Binary files a/.maestro/enrichedMarkdownText/screenshots/ios/header_code_block_combo_display.png and b/.maestro/enrichedMarkdownText/screenshots/ios/header_code_block_combo_display.png differ diff --git a/.yarnrc.yml b/.yarnrc.yml index a95225543..b0863d344 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -2,3 +2,24 @@ nodeLinker: node-modules nmHoistingLimits: workspaces yarnPath: .yarn/releases/yarn-4.11.0.cjs + +# The tree-sitter grammars are vendored from source (see vendor/vendor-grammars.mjs); +# we never load their compiled Node bindings. @tree-sitter-grammars/tree-sitter-markdown +# and tree-sitter-swift ship a typo'd peerDependenciesMeta key ("tree_sitter" instead of +# "tree-sitter"), so Yarn treats their optional "tree-sitter" peer as required and warns. +# Re-declare it optional. (Their native builds are disabled via dependenciesMeta in package.json.) +packageExtensions: + '@tree-sitter-grammars/tree-sitter-markdown@*': + peerDependenciesMeta: + tree-sitter: + optional: true + 'tree-sitter-swift@*': + peerDependenciesMeta: + tree-sitter: + optional: true + +# The only packages with builds disabled above are those vendored tree-sitter grammars, +# so silence the per-package "build has been explicitly disabled" notice (YN0005). +logFilters: + - code: YN0005 + level: discard diff --git a/apps/react-native-example/.rnstorybook/stories/components/EnrichedMarkdownText/block/CodeHighlight.stories.tsx b/apps/react-native-example/.rnstorybook/stories/components/EnrichedMarkdownText/block/CodeHighlight.stories.tsx index 876b4989b..e665748bb 100644 --- a/apps/react-native-example/.rnstorybook/stories/components/EnrichedMarkdownText/block/CodeHighlight.stories.tsx +++ b/apps/react-native-example/.rnstorybook/stories/components/EnrichedMarkdownText/block/CodeHighlight.stories.tsx @@ -5,79 +5,146 @@ import { githubFlavorArgTypes } from '../shared/storybookMarkdownStyles'; import { splitStyleControls } from '../shared/storybookStyleBuilders'; import type { TextStory } from '../shared/storyTypes'; +// A per-language sample split into two labelled sections so a tester can verify +// language coverage at a glance (see docs/TESTING_CODE_HIGHLIGHT_VENDORING.md, +// step 4): the "Defaults" section lists every language in the default registry, +// which highlights out of the box; the "Extended List" section lists the +// non-default grammars, which stay plain unless a build opts them in via the +// language-subset override (steps 4a/5a). One short block per language. const MARKDOWN = [ + '## Defaults', + '', '```javascript', - 'const greet = (name) => `hi ${name}`; // arrow fn', - 'export default greet(42);', + 'const sum = (a, b) => a + b; // add two numbers', '```', '', '```typescript', - 'type Pair = { left: T; right: T };', - 'function swap(p: Pair): Pair {', - ' return { left: p.right, right: p.left };', - '}', + 'const id: number = 7; type Name = string;', + '```', + '', + '```tsx', + 'const El = () => ;', '```', '', '```python', - 'def fib(n: int) -> int:', - ' return n if n < 2 else fib(n - 1) + fib(n - 2) # recursion', + 'def greet(name): return f"hi {name}" # f-string', '```', '', '```json', - '{ "id": 7, "tags": ["a", "b"], "active": true }', + '{ "id": 7, "tags": ["a"], "active": true }', + '```', + '', + '```yaml', + 'name: demo', + 'port: 8080 # comment', + '```', + '', + '```bash', + 'echo "hi $USER" # greet', '```', '', '```go', - 'package main', - 'func main() { println("hello") }', + 'func main() { x := 3; _ = x }', '```', '', - '```rust', - 'fn main() { let x: u32 = 3; println!("{x}"); }', + '```java', + 'class A { int x = 3; }', '```', '', '```c', - '#include ', 'int main(void) { return 0; }', '```', '', - '```java', - 'record Point(int x, int y) {}', + '```rust', + 'fn main() { let x: u32 = 3; }', '```', '', - '```bash', - 'for f in *.ts; do echo "$f"; done', + '```html', + 'go', '```', '', '```css', - '.title { color: #cf222e; font-weight: 600; }', + '.title { color: #cf222e; }', '```', '', - '```html', - 'go', + '```markdown', + '# Heading with **bold** and _em_ [link](/x)', '```', '', - '```yaml', - 'name: build', - 'on: [push]', + '## Extended List', + '', + '```cpp', + 'int main() { auto x = 3; return x; }', + '```', + '', + '```swift', + 'let x: Int = 3; print(x)', + '```', + '', + '```php', + '', + '```', + '', + '```ruby', + 'def greet(name) = "hi #{name}"', + '```', + '', + '```csharp', + 'class A { int X = 3; }', '```', ].join('\n'); +// Two blocks that together exercise every token type except embedded (which +// needs language injection, unsupported by the single-grammar highlighter). The +// Rust block covers comment, attribute (#[derive]), keyword, type, function, +// property (fields), variable, number, string, constant, operator and +// punctuation; the HTML block adds tag and attribute (plus doctype -> constant). +const MARKDOWN_ALL_TOKENS = [ + '```rust', + '// distance between two points', + '#[derive(Debug, Clone)]', + 'struct Point {', + ' x: f64,', + ' y: f64,', + '}', + '', + 'impl Point {', + ' fn dist(&self, other: &Point) -> f64 {', + ' let dx = self.x - other.x;', + ' (dx * dx).sqrt()', + ' }', + '}', + '', + 'fn main() {', + ' const SCALE: u32 = 2;', + ' let p = Point { x: 1.5, y: 3.0 };', + ' println!("dist = {}", p.dist(&p) * SCALE as f64);', + '}', + '```', + '', + '```html', + '', + '', + 'go', + '```', +].join('\n'); + +// GitHub-dark palette; the four "inherit" tokens use the code block base color. const BASE_TEXT_COLOR = '#f3f4f6'; const syntaxColorDefaults = { - keyword: '#cf222e', + keyword: '#ff7b72', operatorColor: BASE_TEXT_COLOR, punctuation: BASE_TEXT_COLOR, - string: '#0a3069', - number: '#0550ae', - constant: '#0550ae', - comment: '#6e7781', - function: '#8250df', - type: '#953800', + string: '#a5d6ff', + number: '#79c0ff', + constant: '#79c0ff', + comment: '#8b949e', + function: '#d2a8ff', + type: '#ffa657', variable: BASE_TEXT_COLOR, - property: '#0550ae', - tag: '#116329', - attribute: '#0550ae', + property: '#79c0ff', + tag: '#7ee787', + attribute: '#79c0ff', embedded: BASE_TEXT_COLOR, }; @@ -122,7 +189,27 @@ export const Default: TextStory = { return ( + ); + }, +}; + +export const AllTokens: TextStory = { + args: { + markdown: MARKDOWN_ALL_TOKENS, + flavor: 'github', + ...syntaxColorDefaults, + }, + argTypes, + render: (args) => { + const { controls, rest } = splitStyleControls(args, syntaxColorDefaults); + return ( + diff --git a/apps/react-native-example/android/gradle.properties b/apps/react-native-example/android/gradle.properties index c224c4fc4..ab4e003de 100644 --- a/apps/react-native-example/android/gradle.properties +++ b/apps/react-native-example/android/gradle.properties @@ -45,3 +45,5 @@ edgeToEdgeEnabled=false # Set to false to disable LaTeX math rendering and remove the RaTeX dependency. #enrichedMarkdown.enableMath=false + +enrichedMarkdown.codeHighlightLanguages=json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,php,ruby,c-sharp diff --git a/apps/react-native-example/ios/Podfile b/apps/react-native-example/ios/Podfile index b16535e14..acbb6c258 100644 --- a/apps/react-native-example/ios/Podfile +++ b/apps/react-native-example/ios/Podfile @@ -5,6 +5,10 @@ ENV['RCT_NEW_ARCH_ENABLED'] = '1' # ENV['ENRICHED_MARKDOWN_ENABLE_MATH'] = '0' ENV['USE_FRAMEWORKS'] = 'dynamic' if ENV['ENRICHED_MARKDOWN_ENABLE_MATH'] != '0' +# Keep project default to full list of languages +ENV['ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES'] = 'json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,php,ruby,c-sharp' +# ENV['ENRICHED_MARKDOWN_ENABLE_CODE_HIGHLIGHT'] = '0' + # Resolve react_native_pods.rb with node to allow for hoisting require Pod::Executable.execute_command('node', ['-p', 'require.resolve( diff --git a/apps/react-native-example/ios/Podfile.lock b/apps/react-native-example/ios/Podfile.lock index 468f6c714..453d8d555 100644 --- a/apps/react-native-example/ios/Podfile.lock +++ b/apps/react-native-example/ios/Podfile.lock @@ -2557,9 +2557,9 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native/ReactCommon/yoga" SPEC CHECKSUMS: - EnrichedMarkdownCore: 35aa4b07e4f23204609527c0e2440bbded65f23d + EnrichedMarkdownCore: 77755092110fabe3ea3a522b86b29ae98bf4d21a FBLazyVector: b3e7ad108f0d882e30445c5527d774e3fd432f3d - hermes-engine: 4a0ceceb51f483aef72bfeb288e965aff6defa54 + hermes-engine: 1622a566d44d0dd4d377a1532e3ac82be7a0443c RCTDeprecation: 2a74a2c57675e64419bd89078efde81f7c1de90b RCTRequired: 30451112e6fef4e6f31b4e7eee0845156e35e4b0 RCTSwiftUI: 5aaf0b07e747ba749dc6acc94d8bd41eea4b570f @@ -2568,7 +2568,7 @@ SPEC CHECKSUMS: React: 2574546f2d017abd14d0c9b48cf2b6a0547c2591 React-callinvoker: 03cd4b931d1d583d87aae99b8f7b6fe26bf571ee React-Core: 1c824d9c7dd8aa760b5f1b50d5a54c2a3f598f87 - React-Core-prebuilt: 13924a267683b3d6fa4bde9c80380becf83a9c5c + React-Core-prebuilt: 7da85c26e5616d52f119b81ce5e3e6fc5c9bda49 React-CoreModules: 2f9ed75bca7f6dea2b70e8a1f4a5ca9b6be52d76 React-cxxreact: 7103d5ba69848c039e11079e74ceede05efe795e React-debug: e47fe49e67816470d183595746b1d0e20cd09fab @@ -2644,6 +2644,6 @@ SPEC CHECKSUMS: RNWorklets: ae7f2b95d75b903387e6359583f2fd67bc9a93ff Yoga: c30c859b7e9b75e547b600b486fe33165f60de00 -PODFILE CHECKSUM: ef24009d29b47aa09e94351250ad7451d534a441 +PODFILE CHECKSUM: 1958dc707e4ee490c590b8544a91b81340ad9af8 COCOAPODS: 1.16.2 diff --git a/docs/CODE_HIGHLIGHT.md b/docs/CODE_HIGHLIGHT.md new file mode 100644 index 000000000..74c971cd1 --- /dev/null +++ b/docs/CODE_HIGHLIGHT.md @@ -0,0 +1,146 @@ +# Code-block syntax highlighting + +Fenced code blocks are syntax-highlighted natively via [tree-sitter](https://tree-sitter.github.io/). +Highlighting is **foreground-only** (it recolors tokens and never changes text metrics), so a code +block's measured height always matches its drawn height. It is enabled by default on iOS and Android +with a curated set of languages, and can be trimmed or disabled to reduce binary size. + +## Usage + +Highlighting activates automatically for a fenced block whose info string names a supported language: + +````tsx + str: + return f"Hello, {name}!" # a comment +\`\`\` +`} + markdownStyle={{ + codeBlock: { + syntaxColors: { + keyword: '#C678DD', + string: '#98C379', + number: '#D19A66', + comment: '#7F848E', + function: '#61AFEF', + type: '#E5C07B', + // ...any of the 14 token types + }, + }, + }} +/> +```` + +Token colors are set through `codeBlock.syntaxColors`. The 14 token types are: `keyword`, +`operatorColor`, `punctuation`, `string`, `number`, `constant`, `comment`, `function`, `type`, +`variable`, `property`, `tag`, `attribute`, `embedded`. Any type left unset is drawn in the normal +code color. + +## Supported languages + +Fence info strings map to a grammar (for example `js`, `jsx` -> JavaScript). The **curated default +set** is compiled in unless you override it. It is defined by `default:true` in +`vendor/grammar-versions.json` (the single source of truth the iOS podspec and Android build both +derive from), so the table below tracks that manifest: + +| Default (on) | Opt-in (heavier) | +|---|---| +| json, html, css, markdown, yaml, go, java, javascript, python, c, rust, bash, typescript, tsx | cpp, swift, php, ruby, c-sharp | + +The default set is the smaller-footprint tier (~32 MB of grammar C source across the whole set). +The opt-in grammars are larger (17-29 MB each) and are only compiled when you list them explicitly. +A block whose language is not compiled in simply renders as plain (uncolored) code. + +### TODO: Kotlin + +Kotlin is not currently supported. The `@tree-sitter-grammars/tree-sitter-kotlin` package +ships a parser but no `queries/highlights.scm`, so there is nothing to highlight with, and a +grammar without a highlights query cannot be compiled into the registry. Kotlin was therefore +removed from the manifest, the fence alias table, and this list; `kotlin` fences render as plain +code. To add support, vendor a compatible `highlights.scm` (for example an MIT-licensed one from +nvim-treesitter, matched to the grammar version), drop it alongside the grammar, and re-add the +`kotlin` entry to `vendor/grammar-versions.json` and the `CodeBlockLanguages.cpp` alias table. + +## Choosing languages / reducing binary size + +Only the grammars you compile end up in your binary, so trimming the list is the main size lever. +The seam degrades to plain code whenever a grammar is absent, so nothing breaks when you remove one. + +### iOS + +Add to your `Podfile` and re-run `pod install`: + +```ruby +# Compile a custom set (comma-separated; adds tsx to the trimmed set below): +ENV['ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES'] = 'javascript,tsx,json,bash' + +# ...or disable highlighting entirely (no tree-sitter code linked): +ENV['ENRICHED_MARKDOWN_ENABLE_CODE_HIGHLIGHT'] = '0' +``` + +### Android + +Add to your project's `gradle.properties`: + +```properties +# Compile a custom set: +enrichedMarkdown.codeHighlightLanguages=javascript,tsx,json,bash + +# ...or disable highlighting entirely: +enrichedMarkdown.enableCodeHighlight=false +``` + +Rebuild the app after changing either value. + +### Expo config plugin + +Configure both platforms at once in `app.json` / `app.config.js`: + +```json +{ + "expo": { + "plugins": [ + [ + "react-native-enriched-markdown", + { + "codeHighlight": { + "enabled": true, + "languages": ["javascript", "tsx", "json", "bash"] + } + } + ] + ] + } +} +``` + +Set `"enabled": false` to disable it. Changes are applied during `npx expo prebuild`; if you change +the set later, run `npx expo prebuild --clean` and rebuild. + +## How it works + +Grammars are **vendored** into `packages/core/cpp/highlight/vendor/` (only each grammar's +`parser.c`/`scanner.c` + `highlights.scm`, never whole npm packages), so the native build itself is +fully offline and deterministic. The stable tree-sitter runtime is vendored the same way and compiled +with WebAssembly support left out. A build-time codegen emits a registry for exactly the selected +languages, so the binary and link step only ever reference compiled grammars. + +The entire `vendor/` tree is **gitignored** to keep the repo and PRs small — nothing generated lives +in git. `vendor/vendor-grammars.mjs` restores all of it from the pins in `vendor/grammar-versions.json`: +the tree-sitter runtime (`vendor/tree-sitter/`) is fetched and sha256-verified from the pinned GitHub +release tarball, the ~178 MB of grammar `parser.c` tables (`vendor/grammars/`) are copied from the +pinned grammar devDependencies, and the default registry (`vendor/generated/`) is codegen'd from them. +It is wired into the package `prepare` script (so a plain `yarn install` restores everything, with +`.stamp` guards making repeats a no-op) and into `prepack` (so the published npm tarball still ships +the full set — consumers install it prebaked and never fetch anything). + +Highlighting runs synchronously when a code block is applied and is cached per block, with a size cap +(~50 KB / ~2000 lines) that falls back to plain rendering for pathological inputs. Maintainers re-pin +by editing `vendor/grammar-versions.json` (for a runtime bump, also update `runtime.sha256` — a full +`node vendor/vendor-grammars.mjs --force` run prints the correct digest on mismatch) and re-running +the script; there is nothing generated to commit. To vendor the runtime from a local tree-sitter +checkout instead of the network, pass `--runtime-src ` (or set +`TREE_SITTER_SRC`). diff --git a/package.json b/package.json index cc3e86297..2ea79568c 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,65 @@ "typescript": "^6.0.2" }, "packageManager": "yarn@4.11.0", + "dependenciesMeta": { + "@tree-sitter-grammars/tree-sitter-markdown": { + "built": false + }, + "@tree-sitter-grammars/tree-sitter-yaml": { + "built": false + }, + "tree-sitter-bash": { + "built": false + }, + "tree-sitter-c": { + "built": false + }, + "tree-sitter-c-sharp": { + "built": false + }, + "tree-sitter-cli": { + "built": false + }, + "tree-sitter-cpp": { + "built": false + }, + "tree-sitter-css": { + "built": false + }, + "tree-sitter-go": { + "built": false + }, + "tree-sitter-html": { + "built": false + }, + "tree-sitter-java": { + "built": false + }, + "tree-sitter-javascript": { + "built": false + }, + "tree-sitter-json": { + "built": false + }, + "tree-sitter-php": { + "built": false + }, + "tree-sitter-python": { + "built": false + }, + "tree-sitter-ruby": { + "built": false + }, + "tree-sitter-rust": { + "built": false + }, + "tree-sitter-swift": { + "built": false + }, + "tree-sitter-typescript": { + "built": false + } + }, "commitlint": { "extends": [ "@commitlint/config-conventional" diff --git a/packages/core/EnrichedMarkdownCore.podspec b/packages/core/EnrichedMarkdownCore.podspec index 2ac467536..12f1b5441 100644 --- a/packages/core/EnrichedMarkdownCore.podspec +++ b/packages/core/EnrichedMarkdownCore.podspec @@ -1,6 +1,8 @@ require "json" +require File.join(__dir__, "cpp/highlight/code_highlight_podspec.rb") package = JSON.parse(File.read(File.join(__dir__, "package.json"))) +code_highlight = EnrichedMarkdownCodeHighlight.config(__dir__) Pod::Spec.new do |s| s.name = "EnrichedMarkdownCore" @@ -13,12 +15,18 @@ Pod::Spec.new do |s| s.platforms = { :ios => min_ios_version_supported, :osx => "14.0" } - s.source_files = "cpp/md4c/*.{c,h}", "cpp/parser/*.{hpp,cpp}", "cpp/highlight/*.{hpp,cpp}" + s.source_files = ["cpp/md4c/*.{c,h}", "cpp/parser/*.{hpp,cpp}", "cpp/highlight/*.{hpp,cpp}"] + code_highlight[:source_files] s.private_header_files = "cpp/**/*.{h,hpp}" + # Include-only vendored sources (schema.*.c, tree_sitter/*.h, other runtime .c) + # must ship even though they are not compiled, so quoted relative includes resolve. + s.preserve_paths = "cpp/highlight/vendor/**/*" if code_highlight[:enabled] + + header_paths = ['"$(PODS_TARGET_SRCROOT)/cpp/md4c"', '"$(PODS_TARGET_SRCROOT)/cpp/parser"', '"$(PODS_TARGET_SRCROOT)/cpp/highlight"'] + header_paths += code_highlight[:header_paths].map { |p| "\"$(PODS_TARGET_SRCROOT)/#{p}\"" } s.pod_target_xcconfig = { - "HEADER_SEARCH_PATHS" => '"$(PODS_TARGET_SRCROOT)/cpp/md4c" "$(PODS_TARGET_SRCROOT)/cpp/parser" "$(PODS_TARGET_SRCROOT)/cpp/highlight"', - "GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) MD4C_USE_UTF8=1", + "HEADER_SEARCH_PATHS" => header_paths.join(" "), + "GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) MD4C_USE_UTF8=1#{code_highlight[:defines]}", "CLANG_CXX_LANGUAGE_STANDARD" => "c++17" } end diff --git a/packages/core/cpp/highlight/CodeBlockHighlighter.cpp b/packages/core/cpp/highlight/CodeBlockHighlighter.cpp index bf27281e5..c3a97f6b0 100644 --- a/packages/core/cpp/highlight/CodeBlockHighlighter.cpp +++ b/packages/core/cpp/highlight/CodeBlockHighlighter.cpp @@ -14,4 +14,271 @@ std::vector highlightCode(const std::string & /*code*/, const st } // namespace Markdown +#else + +#include "CodeBlockLanguages.hpp" +#include "HighlightGrammars.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace Markdown { + +namespace { + +// Bounds the synchronous main-thread parse against pathological inputs. Typical +// blocks parse in sub-millisecond to low-millisecond time; anything past these +// caps falls back to plain rendering instead of risking a frame hang. +constexpr size_t kMaxBytes = 50u * 1024u; +constexpr size_t kMaxLines = 2000u; + +// tree-sitter highlight capture names, flattened to their leading dotted +// prefix. The longest matching prefix wins, so "variable.member" resolves to +// Property while a bare "variable" resolves to Variable. Names with no entry +// (for example "spell", "none") are skipped and leave their bytes uncolored. +struct CaptureMapping { + const char *prefix; + HighlightTokenType type; +}; + +constexpr CaptureMapping kCaptureMappings[] = { + {"keyword", HighlightTokenType::Keyword}, + {"operator", HighlightTokenType::Operator}, + {"punctuation", HighlightTokenType::Punctuation}, + {"string", HighlightTokenType::String}, + {"escape", HighlightTokenType::String}, + {"character", HighlightTokenType::String}, + {"number", HighlightTokenType::Number}, + {"float", HighlightTokenType::Number}, + {"boolean", HighlightTokenType::Constant}, + {"constant", HighlightTokenType::Constant}, + {"comment", HighlightTokenType::Comment}, + {"function", HighlightTokenType::Function}, + {"method", HighlightTokenType::Function}, + {"constructor", HighlightTokenType::Type}, + {"type", HighlightTokenType::Type}, + {"module", HighlightTokenType::Type}, + {"namespace", HighlightTokenType::Type}, + {"variable", HighlightTokenType::Variable}, + {"parameter", HighlightTokenType::Variable}, + {"variable.member", HighlightTokenType::Property}, + {"property", HighlightTokenType::Property}, + {"field", HighlightTokenType::Property}, + {"tag", HighlightTokenType::Tag}, + {"attribute", HighlightTokenType::Attribute}, + {"embedded", HighlightTokenType::Embedded}, + {"injection", HighlightTokenType::Embedded}, +}; + +// A capture name matches a prefix only on dotted boundaries so "type" never +// swallows "typename"-style names. Returns false when nothing maps. +bool mapCaptureName(const char *name, uint32_t length, HighlightTokenType &out) { + size_t best = 0; + bool found = false; + for (const CaptureMapping &m : kCaptureMappings) { + size_t plen = std::strlen(m.prefix); + if (plen > length || plen < best) { + continue; + } + if (std::strncmp(name, m.prefix, plen) != 0) { + continue; + } + if (plen != length && name[plen] != '.') { + continue; + } + best = plen; + out = m.type; + found = true; + } + return found; +} + +// Compiled queries are immutable after creation and reused across calls. Access +// is serialized on the main thread today; the mutex is cheap insurance should +// highlighting ever move off-main. A cached nullptr marks a query that failed to +// compile so it is not retried. +TSQuery *queryForLanguage(const TSLanguage *language, const char *source) { + static std::mutex mutex; + static std::unordered_map cache; + + std::lock_guard lock(mutex); + auto it = cache.find(language); + if (it != cache.end()) { + return it->second; + } + + uint32_t errorOffset = 0; + TSQueryError errorType = TSQueryErrorNone; + TSQuery *query = ts_query_new(language, source, static_cast(std::strlen(source)), &errorOffset, &errorType); + cache.emplace(language, query); + return query; +} + +// Precomputes, for every byte offset that starts a code point, the number of +// UTF-16 code units preceding it. Token boundaries always land on code-point +// boundaries, so mapping a byte offset is a single lookup. Astral code points +// (4-byte UTF-8) contribute a surrogate pair (+2). +std::vector buildUtf16PrefixMap(const std::string &code) { + const size_t n = code.size(); + std::vector prefix(n + 1, 0); + uint32_t units = 0; + size_t i = 0; + while (i < n) { + unsigned char c = static_cast(code[i]); + size_t len; + uint32_t width; + if (c < 0x80) { + len = 1; + width = 1; + } else if ((c >> 5) == 0x6) { + len = 2; + width = 1; + } else if ((c >> 4) == 0xE) { + len = 3; + width = 1; + } else if ((c >> 3) == 0x1E) { + len = 4; + width = 2; + } else { + len = 1; + width = 1; + } + units += width; + for (size_t k = 1; k <= len && i + k <= n; ++k) { + prefix[i + k] = units; + } + i += len; + } + return prefix; +} + +using ParserPtr = std::unique_ptr; +using TreePtr = std::unique_ptr; +using CursorPtr = std::unique_ptr; + +} // namespace + +// Resolves the language to a compiled grammar, parses the code, runs the +// grammar's highlight query, resolves overlapping captures (innermost span +// wins per byte), and returns coalesced foreground-only tokens with UTF-16 +// offsets. Any failure along the way degrades to an empty vector (plain +// rendering). +std::vector highlightCode(const std::string &code, const std::string &language) { + try { + if (code.empty()) { + return {}; + } + + std::string grammarId = canonicalGrammarId(language); + if (grammarId.empty()) { + return {}; + } + const GrammarEntry *grammar = findGrammar(grammarId.c_str()); + if (grammar == nullptr) { + return {}; + } + + if (code.size() > kMaxBytes) { + return {}; + } + size_t lines = 1; + for (char c : code) { + if (c == '\n' && ++lines > kMaxLines) { + return {}; + } + } + + ParserPtr parser(ts_parser_new(), ts_parser_delete); + if (!parser) { + return {}; + } + const TSLanguage *tsLanguage = grammar->language(); + if (tsLanguage == nullptr || !ts_parser_set_language(parser.get(), tsLanguage)) { + return {}; + } + + TSQuery *query = queryForLanguage(tsLanguage, grammar->highlightsQuery); + if (query == nullptr) { + return {}; + } + + TreePtr tree(ts_parser_parse_string(parser.get(), nullptr, code.c_str(), static_cast(code.size())), + ts_tree_delete); + if (!tree) { + return {}; + } + + CursorPtr cursor(ts_query_cursor_new(), ts_query_cursor_delete); + if (!cursor) { + return {}; + } + ts_query_cursor_exec(cursor.get(), query, ts_tree_root_node(tree.get())); + + // Innermost-wins overlap resolution: paint each capture's byte range, + // widest span first, so tighter captures overwrite. A -1 byte is uncolored. + struct Capture { + uint32_t start; + uint32_t end; + HighlightTokenType type; + }; + std::vector captures; + TSQueryMatch match; + uint32_t captureIndex = 0; + while (ts_query_cursor_next_capture(cursor.get(), &match, &captureIndex)) { + const TSQueryCapture &capture = match.captures[captureIndex]; + uint32_t nameLength = 0; + const char *name = ts_query_capture_name_for_id(query, capture.index, &nameLength); + HighlightTokenType type; + if (name == nullptr || !mapCaptureName(name, nameLength, type)) { + continue; + } + uint32_t start = ts_node_start_byte(capture.node); + uint32_t end = ts_node_end_byte(capture.node); + if (end > start && end <= code.size()) { + captures.push_back({start, end, type}); + } + } + if (captures.empty()) { + return {}; + } + + std::stable_sort(captures.begin(), captures.end(), + [](const Capture &a, const Capture &b) { return (a.end - a.start) > (b.end - b.start); }); + + std::vector byteType(code.size(), -1); + for (const Capture &capture : captures) { + for (uint32_t b = capture.start; b < capture.end; ++b) { + byteType[b] = static_cast(capture.type); + } + } + + std::vector u16 = buildUtf16PrefixMap(code); + std::vector tokens; + size_t b = 0; + const size_t n = code.size(); + while (b < n) { + if (byteType[b] < 0) { + ++b; + continue; + } + int8_t type = byteType[b]; + size_t runStart = b; + while (b < n && byteType[b] == type) { + ++b; + } + tokens.push_back({u16[runStart], u16[b], static_cast(type)}); + } + return tokens; + } catch (...) { + return {}; + } +} + +} // namespace Markdown + #endif diff --git a/packages/core/cpp/highlight/CodeBlockHighlighter.hpp b/packages/core/cpp/highlight/CodeBlockHighlighter.hpp index cf0d23c66..23770b3f3 100644 --- a/packages/core/cpp/highlight/CodeBlockHighlighter.hpp +++ b/packages/core/cpp/highlight/CodeBlockHighlighter.hpp @@ -26,17 +26,11 @@ // Token types follow tree-sitter's standard highlight capture names, // flattened to one level. Values are explicit because they cross the JNI // boundary as plain integers. - -// TODO: phase 2 adds the real implementation next to this file: the -// tree-sitter runtime plus a vendored set of language grammars and their -// highlight queries, compiled only when ENRICHED_MARKDOWN_CODE_HIGHLIGHT is -// defined by the gradle property / podspec option. // -// TODO: only the github flavor calls this seam; the commonmark flavor renders -// code blocks uncolored. To hook it up, the commonmark code block renderers -// (CodeBlockRenderer.kt / CodeBlockRenderer.m) would call highlightCode with -// the shared node helpers (CodeBlockNode.kt / ENRMCodeBlockContent.h) and -// apply the tokens to their content range through the platform adapters. +// Both markdown flavors call this seam: the github flavor highlights the code +// string in its container view, and the commonmark renderers +// (CodeBlockRenderer.kt / CodeBlockRenderer.m) apply the tokens to their +// content range through the platform adapters. namespace Markdown { diff --git a/packages/core/cpp/highlight/CodeBlockLanguages.cpp b/packages/core/cpp/highlight/CodeBlockLanguages.cpp index 9faac40f2..031d25a35 100644 --- a/packages/core/cpp/highlight/CodeBlockLanguages.cpp +++ b/packages/core/cpp/highlight/CodeBlockLanguages.cpp @@ -12,77 +12,127 @@ namespace { struct LanguageName { const char *key; const char *name; + // Canonical tree-sitter grammar id (vendored directory name), or "" when no + // grammar covers this fence language. + const char *grammar; }; -// Sorted by key; displayNameForLanguage binary-searches this table. -constexpr std::array kLanguageNames{{ - {"bash", "Bash"}, - {"c", "C"}, - {"cc", "C++"}, - {"cpp", "C++"}, - {"cs", "C#"}, - {"csharp", "C#"}, - {"css", "CSS"}, - {"cxx", "C++"}, - {"dockerfile", "Dockerfile"}, - {"go", "Go"}, - {"golang", "Go"}, - {"graphql", "GraphQL"}, - {"html", "HTML"}, - {"java", "Java"}, - {"javascript", "JavaScript"}, - {"js", "JavaScript"}, - {"json", "JSON"}, - {"jsx", "JSX"}, - {"kotlin", "Kotlin"}, - {"kt", "Kotlin"}, - {"markdown", "Markdown"}, - {"md", "Markdown"}, - {"objc", "Objective-C"}, - {"objectivec", "Objective-C"}, - {"php", "PHP"}, - {"py", "Python"}, - {"python", "Python"}, - {"rb", "Ruby"}, - {"ruby", "Ruby"}, - {"rs", "Rust"}, - {"rust", "Rust"}, - {"scss", "SCSS"}, - {"sh", "Shell"}, - {"shell", "Shell"}, - {"sql", "SQL"}, - {"swift", "Swift"}, - {"toml", "TOML"}, - {"ts", "TypeScript"}, - {"tsx", "TSX"}, - {"typescript", "TypeScript"}, - {"xml", "XML"}, - {"yaml", "YAML"}, - {"yml", "YAML"}, - {"zsh", "Zsh"}, +// Sorted by key; both lookups below binary-search this table. +constexpr std::array kLanguageNames{{ + {"bash", "Bash", "bash"}, + {"c", "C", "c"}, + {"cc", "C++", "cpp"}, + {"cpp", "C++", "cpp"}, + {"cs", "C#", "c-sharp"}, + {"csharp", "C#", "c-sharp"}, + {"css", "CSS", "css"}, + {"cxx", "C++", "cpp"}, + {"dockerfile", "Dockerfile", ""}, + {"go", "Go", "go"}, + {"golang", "Go", "go"}, + {"graphql", "GraphQL", ""}, + {"html", "HTML", "html"}, + {"java", "Java", "java"}, + {"javascript", "JavaScript", "javascript"}, + {"js", "JavaScript", "javascript"}, + {"json", "JSON", "json"}, + {"jsx", "JSX", "javascript"}, + {"markdown", "Markdown", "markdown"}, + {"md", "Markdown", "markdown"}, + {"objc", "Objective-C", ""}, + {"objectivec", "Objective-C", ""}, + {"php", "PHP", "php"}, + {"py", "Python", "python"}, + {"python", "Python", "python"}, + {"rb", "Ruby", "ruby"}, + {"rs", "Rust", "rust"}, + {"ruby", "Ruby", "ruby"}, + {"rust", "Rust", "rust"}, + {"scss", "SCSS", ""}, + {"sh", "Shell", "bash"}, + {"shell", "Shell", "bash"}, + {"sql", "SQL", ""}, + {"swift", "Swift", "swift"}, + {"toml", "TOML", ""}, + {"ts", "TypeScript", "typescript"}, + {"tsx", "TSX", "tsx"}, + {"typescript", "TypeScript", "typescript"}, + {"xml", "XML", ""}, + {"yaml", "YAML", "yaml"}, + {"yml", "YAML", "yaml"}, + {"zsh", "Zsh", "bash"}, }}; -} // namespace +// findLanguage binary-searches kLanguageNames, so the table must be strictly +// ascending by key under C strcmp order. Enforce it at compile time: a +// misordered entry (e.g. "ruby" before "rs") would otherwise silently fail the +// lookup and leave that language unhighlighted. +constexpr bool keyLess(const char *a, const char *b) { + for (std::size_t i = 0;; ++i) { + unsigned char ca = static_cast(a[i]); + unsigned char cb = static_cast(b[i]); + if (ca != cb) { + return ca < cb; + } + if (ca == '\0') { + return false; + } + } +} -std::string displayNameForLanguage(const std::string &language) { - if (language.empty()) { - return ""; +constexpr bool languageTableIsSorted() { + for (std::size_t i = 1; i < kLanguageNames.size(); ++i) { + if (!keyLess(kLanguageNames[i - 1].key, kLanguageNames[i].key)) { + return false; + } } + return true; +} - std::string lower = language; - for (char &c : lower) { +static_assert(languageTableIsSorted(), "kLanguageNames must be sorted by key for binary search"); + +// Lowercases language and binary-searches the table. Returns nullptr on miss. +const LanguageName *findLanguage(const std::string &language, std::string &lowerOut) { + lowerOut = language; + for (char &c : lowerOut) { c = static_cast(std::tolower(static_cast(c))); } auto it = - std::lower_bound(kLanguageNames.begin(), kLanguageNames.end(), lower.c_str(), + std::lower_bound(kLanguageNames.begin(), kLanguageNames.end(), lowerOut.c_str(), [](const LanguageName &entry, const char *key) { return std::strcmp(entry.key, key) < 0; }); - if (it != kLanguageNames.end() && lower == it->key) { - return it->name; + if (it != kLanguageNames.end() && lowerOut == it->key) { + return &(*it); + } + return nullptr; +} + +} // namespace + +std::string displayNameForLanguage(const std::string &language) { + if (language.empty()) { + return ""; + } + + std::string lower; + if (const LanguageName *entry = findLanguage(language, lower)) { + return entry->name; } lower[0] = static_cast(std::toupper(static_cast(lower[0]))); return lower; } +std::string canonicalGrammarId(const std::string &language) { + if (language.empty()) { + return ""; + } + + std::string lower; + if (const LanguageName *entry = findLanguage(language, lower)) { + return entry->grammar; + } + return ""; +} + } // namespace Markdown diff --git a/packages/core/cpp/highlight/CodeBlockLanguages.hpp b/packages/core/cpp/highlight/CodeBlockLanguages.hpp index dbb1ca6e8..9e55e8a6f 100644 --- a/packages/core/cpp/highlight/CodeBlockLanguages.hpp +++ b/packages/core/cpp/highlight/CodeBlockLanguages.hpp @@ -15,4 +15,11 @@ namespace Markdown { // returns an empty string. std::string displayNameForLanguage(const std::string& language); +// Maps a fence info string to the canonical tree-sitter grammar id that can +// highlight it (for example "js" and "jsx" both map to "javascript"), or an +// empty string when no grammar covers the language. The id is the vendored +// grammar directory name; whether that grammar is actually compiled into the +// build is answered separately by findGrammar. +std::string canonicalGrammarId(const std::string& language); + } // namespace Markdown diff --git a/packages/core/cpp/highlight/HighlightGrammars.hpp b/packages/core/cpp/highlight/HighlightGrammars.hpp new file mode 100644 index 000000000..5245e4f60 --- /dev/null +++ b/packages/core/cpp/highlight/HighlightGrammars.hpp @@ -0,0 +1,28 @@ +#pragma once + +// Registry of the tree-sitter grammars compiled into this build. The table and +// the findGrammar definition live in generated_registry.cpp, emitted by +// vendor/gen-registry.mjs for exactly the selected language subset, so this +// header never references symbols for grammars that were not compiled. +// +// Only meaningful when ENRICHED_MARKDOWN_CODE_HIGHLIGHT is defined; without the +// flag neither this header nor generated_registry.cpp is compiled. + +struct TSLanguage; + +namespace Markdown { + +struct GrammarEntry { + // Canonical grammar id, matching canonicalGrammarId() in CodeBlockLanguages. + const char *canonicalId; + // tree-sitter language constructor (extern "C" tree_sitter_). + const TSLanguage *(*language)(void); + // The grammar's highlights.scm, inheritance already inlined, as a C string. + const char *highlightsQuery; +}; + +// Returns the entry for canonicalId, or nullptr when no grammar for it was +// compiled into this build (unselected language) or canonicalId is empty. +const GrammarEntry *findGrammar(const char *canonicalId); + +} // namespace Markdown diff --git a/packages/core/cpp/highlight/code_highlight_podspec.rb b/packages/core/cpp/highlight/code_highlight_podspec.rb new file mode 100644 index 000000000..cee030f1a --- /dev/null +++ b/packages/core/cpp/highlight/code_highlight_podspec.rb @@ -0,0 +1,111 @@ +# Shared tree-sitter code-highlighting wiring for both podspecs +# (EnrichedMarkdownCore in the monorepo, ReactNativeEnrichedMarkdown when +# published without the core pod). Podspecs are imperative Ruby that own their +# own source_files/defines/header paths, so unlike Android there is no build +# ownership problem: this computes exactly what to add. +# +# Only lib.c (the runtime) and each selected grammar's parser.c/scanner.c are +# compiled; every other vendored .c/.h is include-only and resolves through +# quoted relative includes, so the shared HEADER_SEARCH_PATHS never carries a +# grammar dir or tree-sitter/src (which would collide grammar parser.h files). +# +# The default grammar set is NOT hardcoded here: it is derived from the single +# source of truth, vendor/grammar-versions.json (every grammar with default:true), +# so the compiled sources always match the registry that gen-registry generates +# from the same manifest. The Android build derives it the same way. + +require 'json' + +module EnrichedMarkdownCodeHighlight + # Locate grammar-versions.json in both layouts, mirroring how gen-registry.mjs + # is resolved below: "/../../vendor" in the monorepo, and the copy + # dropped into "/cpp/highlight" by prepare-npm-publish.sh when + # published. + def self.manifest_path(podspec_dir) + [ + File.join(podspec_dir, '../../vendor/grammar-versions.json'), + File.join(podspec_dir, 'cpp/highlight/grammar-versions.json'), + ].find { |p| File.exist?(p) } + end + + # The default language set: every grammar flagged default:true in the manifest. + def self.default_languages(podspec_dir) + manifest = manifest_path(podspec_dir) + raise '[code-highlight] grammar-versions.json not found; cannot resolve the ' \ + 'default language set. Build from the monorepo or a published tarball.' unless manifest + grammars = JSON.parse(File.read(manifest))['grammars'] || {} + grammars.select { |_id, spec| spec['default'] }.keys + end + + # podspec_dir is the directory of the including podspec; cpp is reached at + # "/cpp" (a symlink in the monorepo, real files when published). + def self.config(podspec_dir) + return disabled if ENV['ENRICHED_MARKDOWN_ENABLE_CODE_HIGHLIGHT'] == '0' + + defaults = default_languages(podspec_dir) + langs = (ENV['ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES'] || '') + .split(',').map(&:strip).reject(&:empty?) + langs = defaults.dup if langs.empty? + return disabled if langs.empty? + + vendor = File.join(podspec_dir, 'cpp/highlight/vendor') + # A custom language set regenerates its registry into a SEPARATE dir so it never + # clobbers the committed default-set registry in vendor/generated. That committed + # dir is the shared source of truth other default builds -- and the Android build + # -- rely on staying the default set; overwriting it in place with a custom set + # leaves the next default build linking a mismatched grammar list. + custom = langs.sort != defaults.sort + generated_rel = custom ? 'cpp/highlight/vendor/generated-custom' : 'cpp/highlight/vendor/generated' + generated = File.join(podspec_dir, generated_rel) + ensure_registry(podspec_dir, vendor, generated, langs, custom) + + sources = [ + 'cpp/highlight/vendor/tree-sitter/src/lib.c', + "#{generated_rel}/generated_registry.cpp", + ] + langs.each do |lang| + sources << "cpp/highlight/vendor/grammars/#{lang}/parser.c" + if File.exist?(File.join(vendor, "grammars/#{lang}/scanner.c")) + sources << "cpp/highlight/vendor/grammars/#{lang}/scanner.c" + end + end + + { + enabled: true, + source_files: sources, + defines: ' ENRICHED_MARKDOWN_CODE_HIGHLIGHT=1', + header_paths: [ + 'cpp/highlight/vendor/tree-sitter/include', + generated_rel, + ], + } + end + + def self.disabled + { enabled: false, source_files: [], defines: '', header_paths: [] } + end + + # The committed default-set registry ships in-tree, so the default case needs no + # codegen. A custom set (or a missing default registry) regenerates into `generated` + # from the vendored .scm files. + def self.ensure_registry(podspec_dir, vendor, generated, langs, custom) + return if !custom && File.exist?(File.join(generated, 'generated_registry.cpp')) + + script = [ + File.join(podspec_dir, '../../vendor/gen-registry.mjs'), + File.join(podspec_dir, 'cpp/highlight/gen-registry.mjs'), + ].find { |p| File.exist?(p) } + raise '[code-highlight] ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES is customized but ' \ + 'vendor/gen-registry.mjs was not found. Use the default set or build from the monorepo.' unless script + + # Canonicalize before handing the path to node: in the monorepo the package is + # a workspace symlink, so the "../.." above resolves correctly for File.exist? + # (which follows the symlink physically) but node would normalize ".." lexically + # from the symlink location and miss the file. Android already does this via + # File#canonicalPath. + script = File.realpath(script) + + ok = system('node', script, '--vendor-dir', vendor, '--languages', langs.join(','), '--out', generated) + raise '[code-highlight] gen-registry.mjs failed' unless ok + end +end diff --git a/packages/react-native-enriched-markdown/ReactNativeEnrichedMarkdown.podspec b/packages/react-native-enriched-markdown/ReactNativeEnrichedMarkdown.podspec index 05a2aef98..c8281c0d8 100644 --- a/packages/react-native-enriched-markdown/ReactNativeEnrichedMarkdown.podspec +++ b/packages/react-native-enriched-markdown/ReactNativeEnrichedMarkdown.podspec @@ -5,6 +5,11 @@ package = JSON.parse(File.read(File.join(__dir__, "package.json"))) monorepo = File.exist?(File.expand_path("../core/EnrichedMarkdownCore.podspec", __dir__)) cpp_root = monorepo ? "$(PODS_TARGET_SRCROOT)/../core/cpp" : "$(PODS_TARGET_SRCROOT)/cpp" +require File.join(__dir__, "cpp/highlight/code_highlight_podspec.rb") +# In the monorepo the C++ (including tree-sitter highlighting) compiles in the +# EnrichedMarkdownCore pod; only the published, core-less build compiles it here. +code_highlight = monorepo ? EnrichedMarkdownCodeHighlight.disabled : EnrichedMarkdownCodeHighlight.config(__dir__) + Pod::Spec.new do |s| s.name = "ReactNativeEnrichedMarkdown" s.version = package["version"] @@ -22,7 +27,8 @@ Pod::Spec.new do |s| s.dependency "EnrichedMarkdownCore" else s.private_header_files = "ios/**/*.h", "cpp/**/*.{h,hpp}" - s.source_files = "ios/**/*.{h,m,mm,cpp,swift}", "cpp/md4c/*.{c,h}", "cpp/parser/*.{hpp,cpp}", "cpp/highlight/*.{hpp,cpp}" + s.source_files = ["ios/**/*.{h,m,mm,cpp,swift}", "cpp/md4c/*.{c,h}", "cpp/parser/*.{hpp,cpp}", "cpp/highlight/*.{hpp,cpp}"] + code_highlight[:source_files] + s.preserve_paths = "cpp/highlight/vendor/**/*" if code_highlight[:enabled] end # To disable LaTeX math rendering (RaTeX, iOS only), add ENV['ENRICHED_MARKDOWN_ENABLE_MATH'] = '0' to your Podfile. @@ -44,7 +50,7 @@ Pod::Spec.new do |s| s.exclude_files = "ios/math/**/*.swift" end - preprocessor_defs = '$(inherited) MD4C_USE_UTF8=1' + preprocessor_defs = "$(inherited) MD4C_USE_UTF8=1#{code_highlight[:defines]}" if enable_math preprocessor_defs += ' ENRICHED_MARKDOWN_MATH=1' spm_dependency(s, @@ -55,7 +61,10 @@ Pod::Spec.new do |s| end pod_xcconfig = { - 'HEADER_SEARCH_PATHS' => "\"#{cpp_root}/md4c\" \"#{cpp_root}/parser\" \"#{cpp_root}/highlight\" \"$(PODS_TARGET_SRCROOT)/ios/internals\" \"$(PODS_TARGET_SRCROOT)/ios/input/internals\"", + 'HEADER_SEARCH_PATHS' => ([ + "\"#{cpp_root}/md4c\"", "\"#{cpp_root}/parser\"", "\"#{cpp_root}/highlight\"", + "\"$(PODS_TARGET_SRCROOT)/ios/internals\"", "\"$(PODS_TARGET_SRCROOT)/ios/input/internals\"" + ] + code_highlight[:header_paths].map { |p| "\"$(PODS_TARGET_SRCROOT)/#{p}\"" }).join(" "), 'GCC_PREPROCESSOR_DEFINITIONS' => preprocessor_defs, 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17', 'DEFINES_MODULE' => 'YES' diff --git a/packages/react-native-enriched-markdown/android/build.gradle b/packages/react-native-enriched-markdown/android/build.gradle index 9bc6b22aa..e08b11181 100644 --- a/packages/react-native-enriched-markdown/android/build.gradle +++ b/packages/react-native-enriched-markdown/android/build.gradle @@ -25,6 +25,25 @@ def getExtOrIntegerDefault(name) { def enableMath = (rootProject.findProperty("enrichedMarkdown.enableMath") ?: findProperty("enrichedMarkdown.enableMath") ?: "true").toString().toBoolean() +// Default grammar set derived from the single source of truth, +// vendor/grammar-versions.json (every grammar with default:true), so the compiled +// grammars always match the registry gen-registry generates from the same +// manifest. Resolved in both layouts like gen-registry.mjs below: +// "../../../vendor" in the monorepo, "../cpp/highlight" when published. +def grammarManifest = [ + new File(projectDir, "../../../vendor/grammar-versions.json"), + new File(projectDir, "../cpp/highlight/grammar-versions.json"), +].find { it.exists() } +if (grammarManifest == null) { + throw new GradleException("[code-highlight] grammar-versions.json not found; cannot resolve the default language set. Build from the monorepo or a published tarball.") +} +def defaultCodeHighlightLanguages = new groovy.json.JsonSlurper().parse(grammarManifest) + .grammars.findAll { id, spec -> spec['default'] }.collect { id, spec -> id } +def enableCodeHighlight = (rootProject.findProperty("enrichedMarkdown.enableCodeHighlight") ?: findProperty("enrichedMarkdown.enableCodeHighlight") ?: "true").toString().toBoolean() +def codeHighlightLanguagesProp = rootProject.findProperty("enrichedMarkdown.codeHighlightLanguages") ?: findProperty("enrichedMarkdown.codeHighlightLanguages") +def codeHighlightLanguages = codeHighlightLanguagesProp ? codeHighlightLanguagesProp.toString().split(",").collect { it.trim() }.findAll { it } : defaultCodeHighlightLanguages +def codeHighlightActive = enableCodeHighlight && !codeHighlightLanguages.isEmpty() + android { namespace "com.swmansion.enriched.markdown" @@ -35,6 +54,7 @@ android { targetSdkVersion getExtOrIntegerDefault("targetSdkVersion") consumerProguardFiles "consumer-rules.pro" buildConfigField "boolean", "ENABLE_MATH", "${enableMath}" + buildConfigField "boolean", "ENABLE_CODE_HIGHLIGHT", "${codeHighlightActive}" } buildFeatures { @@ -69,6 +89,54 @@ android { } } +// Code-block syntax highlighting (tree-sitter) native wiring. This module owns +// no externalNativeBuild block: RN autolinking compiles src/main/jni/CMakeLists.txt +// as part of the consumer app's native build, so the app -- not this library -- +// owns the CMake arguments. The enable flag + selected grammars therefore reach +// CMake through a config file written here during configuration (before any +// native build task runs) and include()d by CMakeLists.txt. +def highlightConfigDir = new File(projectDir, "build/generated/highlight") +highlightConfigDir.mkdirs() +def highlightConfigFile = new File(highlightConfigDir, "highlight-config.cmake") +if (codeHighlightActive) { + def vendorDir = new File(projectDir, "../cpp/highlight/vendor").canonicalPath + def committedGenerated = new File(vendorDir, "generated") + def generatedDir + if (codeHighlightLanguages.toSorted() == defaultCodeHighlightLanguages.toSorted() && committedGenerated.exists()) { + // Common case: the committed default-set registry ships in-tree, no codegen. + generatedDir = committedGenerated + } else { + // Custom language set: regenerate the registry from the vendored .scm files. + generatedDir = new File(highlightConfigDir, "generated") + generatedDir.mkdirs() + def genScript = [ + new File(projectDir, "../../../vendor/gen-registry.mjs"), + new File(projectDir, "../cpp/highlight/gen-registry.mjs"), + ].find { it.exists() } + if (genScript == null) { + throw new GradleException("enrichedMarkdown.codeHighlightLanguages is customized but vendor/gen-registry.mjs was not found. Use the default language set or build from the monorepo.") + } + // Run node directly via ProcessBuilder rather than Project.exec {}, which was + // removed in Gradle 9. Runs at configuration time (before the native build), + // and surfaces gen-registry's own stderr on failure. + def genCmd = ["node", genScript.canonicalPath, "--vendor-dir", vendorDir, + "--languages", codeHighlightLanguages.join(","), "--out", generatedDir.canonicalPath] + def genProc = new ProcessBuilder(genCmd).redirectErrorStream(true).start() + def genOut = genProc.inputStream.text + def genExit = genProc.waitFor() + if (genExit != 0) { + throw new GradleException("[code-highlight] gen-registry.mjs failed (exit ${genExit}):\n${genOut}") + } + } + highlightConfigFile.text = """\ +set(ENRICHED_MARKDOWN_CODE_HIGHLIGHT ON) +set(ENRICHED_MARKDOWN_HIGHLIGHT_LANGUAGES "${codeHighlightLanguages.join(';')}") +set(ENRICHED_MARKDOWN_HIGHLIGHT_GENERATED_DIR "${generatedDir.canonicalPath}") +""" +} else { + highlightConfigFile.text = "set(ENRICHED_MARKDOWN_CODE_HIGHLIGHT OFF)\n" +} + repositories { mavenCentral() google() diff --git a/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/FeatureFlags.kt b/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/FeatureFlags.kt index 0c9c62ab8..84824ee8a 100644 --- a/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/FeatureFlags.kt +++ b/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/FeatureFlags.kt @@ -4,4 +4,5 @@ import com.swmansion.enriched.markdown.BuildConfig object FeatureFlags { const val IS_MATH_ENABLED: Boolean = BuildConfig.ENABLE_MATH + const val IS_CODE_HIGHLIGHT_ENABLED: Boolean = BuildConfig.ENABLE_CODE_HIGHLIGHT } diff --git a/packages/react-native-enriched-markdown/android/src/main/jni/CMakeLists.txt b/packages/react-native-enriched-markdown/android/src/main/jni/CMakeLists.txt index 1142614e9..2bdaa07a0 100644 --- a/packages/react-native-enriched-markdown/android/src/main/jni/CMakeLists.txt +++ b/packages/react-native-enriched-markdown/android/src/main/jni/CMakeLists.txt @@ -25,6 +25,14 @@ endif() set(ANDROID_CPP_DIR "${LIB_ANDROID_DIR}/src/main/cpp") +# Optional tree-sitter code highlighting. build.gradle writes this config during +# configuration (enable flag, selected languages, generated registry dir); absent +# on a bare CMake invocation, in which case highlighting stays compiled out. +set(HIGHLIGHT_CONFIG "${LIB_ANDROID_DIR}/build/generated/highlight/highlight-config.cmake") +if(EXISTS "${HIGHLIGHT_CONFIG}") + include("${HIGHLIGHT_CONFIG}") +endif() + file(GLOB MD4C_SOURCES "${CPP_ROOT}/md4c/*.c") file(GLOB PARSER_SOURCES "${CPP_ROOT}/parser/*.cpp") file(GLOB HIGHLIGHT_SOURCES "${CPP_ROOT}/highlight/*.cpp") @@ -85,3 +93,45 @@ target_include_directories( ) target_compile_reactnative_options(${LIB_TARGET_NAME} PRIVATE) + +# --- Code-block syntax highlighting (tree-sitter) --- +# CodeBlockHighlighter.cpp / CodeBlockLanguages.cpp are already in the target via +# HIGHLIGHT_SOURCES; without the define below CodeBlockHighlighter.cpp is the stub. +# When enabled we add the runtime, the generated registry, and one OBJECT lib per +# selected grammar (each isolated to its own dir so grammar parser.h files never +# collide), then define ENRICHED_MARKDOWN_CODE_HIGHLIGHT so the real seam compiles. +if(ENRICHED_MARKDOWN_CODE_HIGHLIGHT) + set(TS_VENDOR "${CPP_ROOT}/highlight/vendor") + + add_library(tree_sitter_runtime STATIC "${TS_VENDOR}/tree-sitter/src/lib.c") + target_include_directories(tree_sitter_runtime + PRIVATE "${TS_VENDOR}/tree-sitter/src" + PUBLIC "${TS_VENDOR}/tree-sitter/include") + set_target_properties(tree_sitter_runtime PROPERTIES + C_STANDARD 11 C_STANDARD_REQUIRED ON POSITION_INDEPENDENT_CODE ON) + target_compile_options(tree_sitter_runtime PRIVATE -fvisibility=hidden) + + set(GRAMMAR_OBJECTS "") + foreach(lang IN LISTS ENRICHED_MARKDOWN_HIGHLIGHT_LANGUAGES) + set(gdir "${TS_VENDOR}/grammars/${lang}") + set(gsrcs "${gdir}/parser.c") + if(EXISTS "${gdir}/scanner.c") + list(APPEND gsrcs "${gdir}/scanner.c") + endif() + add_library(ts_grammar_${lang} OBJECT ${gsrcs}) + target_include_directories(ts_grammar_${lang} PRIVATE "${gdir}") + set_target_properties(ts_grammar_${lang} PROPERTIES + C_STANDARD 11 C_STANDARD_REQUIRED ON POSITION_INDEPENDENT_CODE ON) + target_compile_options(ts_grammar_${lang} PRIVATE -fvisibility=hidden) + list(APPEND GRAMMAR_OBJECTS $) + endforeach() + + target_sources(${LIB_TARGET_NAME} PRIVATE + "${ENRICHED_MARKDOWN_HIGHLIGHT_GENERATED_DIR}/generated_registry.cpp" + ${GRAMMAR_OBJECTS}) + target_include_directories(${LIB_TARGET_NAME} PRIVATE + "${TS_VENDOR}/tree-sitter/include" + "${ENRICHED_MARKDOWN_HIGHLIGHT_GENERATED_DIR}") + target_compile_definitions(${LIB_TARGET_NAME} PRIVATE ENRICHED_MARKDOWN_CODE_HIGHLIGHT=1) + target_link_libraries(${LIB_TARGET_NAME} tree_sitter_runtime) +endif() diff --git a/packages/react-native-enriched-markdown/ios/utils/ENRMFeatureFlags.h b/packages/react-native-enriched-markdown/ios/utils/ENRMFeatureFlags.h index f57fa5707..684227e10 100644 --- a/packages/react-native-enriched-markdown/ios/utils/ENRMFeatureFlags.h +++ b/packages/react-native-enriched-markdown/ios/utils/ENRMFeatureFlags.h @@ -10,3 +10,10 @@ #if !defined(ENRICHED_MARKDOWN_MATH) #define ENRICHED_MARKDOWN_MATH 0 #endif + +// Code-block syntax highlighting is defined purely by the podspec (no external +// framework to auto-detect, unlike RaTeX). Default off when the podspec did not +// enable it; the C++ seam then degrades to no tokens. +#if !defined(ENRICHED_MARKDOWN_CODE_HIGHLIGHT) +#define ENRICHED_MARKDOWN_CODE_HIGHLIGHT 0 +#endif diff --git a/packages/react-native-enriched-markdown/package.json b/packages/react-native-enriched-markdown/package.json index 203a47ffe..1aeef1c05 100644 --- a/packages/react-native-enriched-markdown/package.json +++ b/packages/react-native-enriched-markdown/package.json @@ -39,10 +39,11 @@ "lint-clang:fix": "yarn lint-clang:ios:fix && yarn lint-clang:android:fix", "clean": "del-cli android/build lib plugin/build", "sync-md4c": "bash ../../scripts/fetch-md4c.sh", + "vendor-grammars": "node ../../vendor/vendor-grammars.mjs", "prepack": "bash ../../scripts/prepare-npm-publish.sh prepack", "postpack": "bash ../../scripts/prepare-npm-publish.sh postpack", "build:plugin": "tsc -p plugin/tsconfig.build.json", - "prepare": "bob build && yarn build:plugin" + "prepare": "yarn vendor-grammars && bob build && yarn build:plugin" }, "keywords": [ "react-native", @@ -85,6 +86,8 @@ "devDependencies": { "@expo/config-plugins": "^55.0.6", "@react-native/babel-preset": "0.85.0", + "@tree-sitter-grammars/tree-sitter-markdown": "0.3.2", + "@tree-sitter-grammars/tree-sitter-yaml": "0.7.1", "@types/node": "^22.0.0", "@types/react": "^19.2.0", "clang-format": "^1.8.0", @@ -92,6 +95,22 @@ "react": "19.2.3", "react-native": "0.85.0", "react-native-builder-bob": "^0.41.0", + "tree-sitter-bash": "0.23.3", + "tree-sitter-c": "0.23.5", + "tree-sitter-c-sharp": "0.23.1", + "tree-sitter-cpp": "0.23.4", + "tree-sitter-css": "0.23.2", + "tree-sitter-go": "0.23.4", + "tree-sitter-html": "0.23.2", + "tree-sitter-java": "0.23.5", + "tree-sitter-javascript": "0.23.1", + "tree-sitter-json": "0.24.8", + "tree-sitter-php": "0.24.2", + "tree-sitter-python": "0.23.6", + "tree-sitter-ruby": "0.23.1", + "tree-sitter-rust": "0.23.2", + "tree-sitter-swift": "0.7.0", + "tree-sitter-typescript": "0.23.2", "typescript": "^6.0.2" }, "react-native-builder-bob": { diff --git a/packages/react-native-enriched-markdown/plugin/src/withAndroidCodeHighlight.ts b/packages/react-native-enriched-markdown/plugin/src/withAndroidCodeHighlight.ts new file mode 100644 index 000000000..a462ee102 --- /dev/null +++ b/packages/react-native-enriched-markdown/plugin/src/withAndroidCodeHighlight.ts @@ -0,0 +1,38 @@ +import { withGradleProperties, type ConfigPlugin } from '@expo/config-plugins'; + +export type CodeHighlightOptions = { enabled?: boolean; languages?: string[] }; + +export const withAndroidCodeHighlight: ConfigPlugin = ( + config, + { enabled = true, languages } = {} +) => { + const hasLanguages = Array.isArray(languages) && languages.length > 0; + // Always run the mod so switching back to defaults strips previously-injected + // properties; an absent property already means enabled with the curated set. + return withGradleProperties(config, (gradleConfig) => { + const drop = new Set([ + 'enrichedMarkdown.enableCodeHighlight', + 'enrichedMarkdown.codeHighlightLanguages', + ]); + gradleConfig.modResults = gradleConfig.modResults.filter( + (prop) => prop.type !== 'property' || !drop.has(prop.key) + ); + + if (!enabled) { + gradleConfig.modResults.push({ + type: 'property', + key: 'enrichedMarkdown.enableCodeHighlight', + value: 'false', + }); + } + if (hasLanguages) { + gradleConfig.modResults.push({ + type: 'property', + key: 'enrichedMarkdown.codeHighlightLanguages', + value: languages!.join(','), + }); + } + + return gradleConfig; + }); +}; diff --git a/packages/react-native-enriched-markdown/plugin/src/withIosCodeHighlight.ts b/packages/react-native-enriched-markdown/plugin/src/withIosCodeHighlight.ts new file mode 100644 index 000000000..b91f99078 --- /dev/null +++ b/packages/react-native-enriched-markdown/plugin/src/withIosCodeHighlight.ts @@ -0,0 +1,45 @@ +import { withDangerousMod, type ConfigPlugin } from '@expo/config-plugins'; +import fs from 'fs'; +import path from 'path'; + +export type CodeHighlightOptions = { enabled?: boolean; languages?: string[] }; + +const ENABLE_KEY = 'ENRICHED_MARKDOWN_ENABLE_CODE_HIGHLIGHT'; +const LANGUAGES_KEY = 'ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES'; + +export const withIosCodeHighlight: ConfigPlugin = ( + config, + { enabled = true, languages } = {} +) => { + const hasLanguages = Array.isArray(languages) && languages.length > 0; + // Always run the mod so switching back to defaults strips previously-injected + // ENV lines; an absent ENV var already means enabled with the curated set. + return withDangerousMod(config, [ + 'ios', + async (modConfig) => { + const file = path.join( + modConfig.modRequest.platformProjectRoot, + 'Podfile' + ); + const contents = fs.readFileSync(file, 'utf8'); + const lines = contents + .split('\n') + .filter( + (line: string) => + !line.includes(ENABLE_KEY) && !line.includes(LANGUAGES_KEY) + ); + + const inject: string[] = []; + if (!enabled) { + inject.push(`ENV['${ENABLE_KEY}'] = '0'`); + } + if (hasLanguages) { + inject.push(`ENV['${LANGUAGES_KEY}'] = '${languages!.join(',')}'`); + } + lines.unshift(...inject); + + fs.writeFileSync(file, lines.join('\n')); + return modConfig; + }, + ]); +}; diff --git a/packages/react-native-enriched-markdown/plugin/src/withReactNativeEnrichedMarkdown.ts b/packages/react-native-enriched-markdown/plugin/src/withReactNativeEnrichedMarkdown.ts index fe96baabb..6a58d0855 100644 --- a/packages/react-native-enriched-markdown/plugin/src/withReactNativeEnrichedMarkdown.ts +++ b/packages/react-native-enriched-markdown/plugin/src/withReactNativeEnrichedMarkdown.ts @@ -1,8 +1,15 @@ import { type ConfigPlugin } from '@expo/config-plugins'; import { withIosMath } from './withIosMath'; import { withAndroidMath } from './withAndroidMath'; +import { withAndroidCodeHighlight } from './withAndroidCodeHighlight'; +import { withIosCodeHighlight } from './withIosCodeHighlight'; -const withEnrichedMarkdown: ConfigPlugin<{ enableMath?: boolean } | void> = ( +type EnrichedMarkdownProps = { + enableMath?: boolean; + codeHighlight?: { enabled?: boolean; languages?: string[] }; +}; + +const withEnrichedMarkdown: ConfigPlugin = ( config, props ) => { @@ -11,6 +18,10 @@ const withEnrichedMarkdown: ConfigPlugin<{ enableMath?: boolean } | void> = ( config = withAndroidMath(config, { enableMath }); config = withIosMath(config, { enableMath }); + const codeHighlight = props?.codeHighlight ?? {}; + config = withAndroidCodeHighlight(config, codeHighlight); + config = withIosCodeHighlight(config, codeHighlight); + return config; }; diff --git a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts index 65e665258..fcf1f01d2 100644 --- a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts +++ b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts @@ -33,20 +33,24 @@ const defaultTextColor = normalizeColor('#1F2937')!; const codeBlockTextColor = normalizeColor('#F3F4F6')!; +// GitHub-dark syntax palette (Primer prettylights), tuned for the dark code +// block background (#1F2937). It is the single source of truth for per-token +// code colors: native reads these resolved values and holds no default of its +// own. The four inheriting tokens resolve to the code block base color. const DEFAULT_CODE_BLOCK_SYNTAX_COLORS = { - keyword: normalizeColor('#CF222E')!, + keyword: normalizeColor('#FF7B72')!, operatorColor: codeBlockTextColor, punctuation: codeBlockTextColor, - string: normalizeColor('#0A3069')!, - number: normalizeColor('#0550AE')!, - constant: normalizeColor('#0550AE')!, - comment: normalizeColor('#6E7781')!, - function: normalizeColor('#8250DF')!, - type: normalizeColor('#953800')!, + string: normalizeColor('#A5D6FF')!, + number: normalizeColor('#79C0FF')!, + constant: normalizeColor('#79C0FF')!, + comment: normalizeColor('#8B949E')!, + function: normalizeColor('#D2A8FF')!, + type: normalizeColor('#FFA657')!, variable: codeBlockTextColor, - property: normalizeColor('#0550AE')!, - tag: normalizeColor('#116329')!, - attribute: normalizeColor('#0550AE')!, + property: normalizeColor('#79C0FF')!, + tag: normalizeColor('#7EE787')!, + attribute: normalizeColor('#79C0FF')!, embedded: codeBlockTextColor, }; diff --git a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts index 9bc8214b3..ae222993a 100644 --- a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts +++ b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts @@ -126,19 +126,19 @@ const DEFAULT_NORMALIZED_STYLE: MarkdownStyleInternal = Object.freeze({ borderWidth: 1, padding: 16, syntaxColors: { - keyword: '#CF222E', + keyword: '#FF7B72', operatorColor: '#F3F4F6', punctuation: '#F3F4F6', - string: '#0A3069', - number: '#0550AE', - constant: '#0550AE', - comment: '#6E7781', - function: '#8250DF', - type: '#953800', + string: '#A5D6FF', + number: '#79C0FF', + constant: '#79C0FF', + comment: '#8B949E', + function: '#D2A8FF', + type: '#FFA657', variable: '#F3F4F6', - property: '#0550AE', - tag: '#116329', - attribute: '#0550AE', + property: '#79C0FF', + tag: '#7EE787', + attribute: '#79C0FF', embedded: '#F3F4F6', }, }, diff --git a/scripts/prepare-npm-publish.sh b/scripts/prepare-npm-publish.sh index 8e23e88be..6e9331e07 100755 --- a/scripts/prepare-npm-publish.sh +++ b/scripts/prepare-npm-publish.sh @@ -11,15 +11,26 @@ mode="${1:-}" case "$mode" in prepack) if [[ ! -d "$CORE_CPP" ]]; then - echo "error: core cpp directory not found at $CORE_CPP" >&2 + echo "[react-native-enriched-markdown] error: core cpp directory not found at $CORE_CPP" >&2 exit 1 fi + # The whole vendor tree (runtime, grammar sources, default registry) is + # gitignored (see .gitignore); restore it so the published tarball ships the + # full vendored set. No-op when already up to date (stamp guards). + node "$REPO_ROOT/vendor/vendor-grammars.mjs" + cd "$RN_PKG" rm -rf cpp mkdir -p cpp cp -R "$CORE_CPP/." cpp/ + # Ship the registry codegen + manifest alongside the vendored grammars so a + # published consumer can compile a custom language set (the default set uses + # the restored cpp/highlight/vendor/generated registry and needs neither). + cp "$REPO_ROOT/vendor/gen-registry.mjs" cpp/highlight/gen-registry.mjs + cp "$REPO_ROOT/vendor/grammar-versions.json" cpp/highlight/grammar-versions.json + cp "$REPO_ROOT/README.md" README.md cp "$REPO_ROOT/LICENSE" LICENSE cp -R "$REPO_ROOT/docs" docs @@ -33,7 +44,7 @@ case "$mode" in rm -rf docs ;; *) - echo "usage: $0 prepack|postpack" >&2 + echo "[react-native-enriched-markdown] usage: $0 prepack|postpack" >&2 exit 1 ;; esac diff --git a/vendor/gen-registry.mjs b/vendor/gen-registry.mjs new file mode 100644 index 000000000..a18254208 --- /dev/null +++ b/vendor/gen-registry.mjs @@ -0,0 +1,155 @@ +#!/usr/bin/env node +// Build-time (and offline): emits the grammar registry for a selected language +// subset from the in-tree vendored files. No network, reads only --vendor-dir. +// Invoked by the Android CMake configure step and the iOS podspec, and by +// vendor-grammars.mjs to refresh the committed default set. +// +// node vendor/gen-registry.mjs --vendor-dir --languages a,b,c --out +// +// Writes /generated_queries.h and /generated_registry.cpp referencing +// only the selected grammars, so the link step stays valid no matter how many +// grammars are vendored. `; inherits:` directives are resolved by inlining the +// parent grammar's highlights (parents come first, child overrides). + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); + +const LOG_PREFIX = '[react-native-enriched-markdown]'; +const log = (message) => console.log(`${LOG_PREFIX} ${message}`); +const warn = (message) => console.warn(`${LOG_PREFIX} ${message}`); +const fail = (message) => { + console.error(`${LOG_PREFIX} ${message}`); + process.exit(1); +}; + +function parseArgs(argv) { + const args = { vendorDir: null, languages: [], out: null, manifest: path.join(here, 'grammar-versions.json') }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--vendor-dir') args.vendorDir = argv[++i]; + else if (a === '--languages') args.languages = argv[++i].split(',').map((s) => s.trim()).filter(Boolean); + else if (a === '--out') args.out = argv[++i]; + else if (a === '--manifest') args.manifest = argv[++i]; + } + if (!args.vendorDir || !args.out) { + fail('usage: --vendor-dir --languages a,b --out '); + } + return args; +} + +function identifier(id) { + return id.replace(/[^A-Za-z0-9]/g, '_'); +} + +function grammarFunction(id, manifest) { + const spec = manifest.grammars?.[id]; + return spec?.function ?? `tree_sitter_${identifier(id)}`; +} + +function highlightsPath(vendorDir, id) { + return path.join(vendorDir, 'grammars', id, 'highlights.scm'); +} + +const INHERITS_RE = /^[ \t]*;+[ \t]*inherits[ \t]*:[ \t]*([^\n]+)$/m; + +// Reads a grammar's highlights.scm and inlines any `; inherits:` parents that +// are also vendored. Cycle-guarded; a missing parent is skipped with a warning. +function resolveHighlights(vendorDir, id, seen = new Set()) { + const file = highlightsPath(vendorDir, id); + if (!fs.existsSync(file)) { + fail(`missing vendored highlights for '${id}': ${file}`); + } + if (seen.has(id)) return ''; + seen.add(id); + + const raw = fs.readFileSync(file, 'utf8'); + const match = raw.match(INHERITS_RE); + let inherited = ''; + if (match) { + for (const parent of match[1].split(',').map((s) => s.trim()).filter(Boolean)) { + if (fs.existsSync(highlightsPath(vendorDir, parent))) { + inherited += resolveHighlights(vendorDir, parent, seen) + '\n'; + } else { + warn(`${id} inherits '${parent}' which is not vendored; skipping inheritance`); + } + } + } + const body = raw.replace(INHERITS_RE, ''); + return inherited + body; +} + +function toCStringLines(text) { + const lines = text.split('\n'); + return lines + .map((line, i) => { + // Escape backslashes, quotes, and every '?' so sequences like "??=" in a + // query never form a C trigraph. + const escaped = line.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\?/g, '\\?'); + const newline = i < lines.length - 1 ? '\\n' : ''; + return ` "${escaped}${newline}"`; + }) + .join('\n'); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const manifest = JSON.parse(fs.readFileSync(args.manifest, 'utf8')); + fs.mkdirSync(args.out, { recursive: true }); + + const entries = args.languages.map((id) => ({ + id, + ident: identifier(id), + fn: grammarFunction(id, manifest), + query: resolveHighlights(args.vendorDir, id), + })); + + const queriesHeader = + '#pragma once\n' + + '// Generated by vendor/gen-registry.mjs. Do not edit.\n\n' + + 'namespace Markdown {\n\n' + + entries + .map((e) => `static const char kHighlightsQuery_${e.ident}[] =\n${toCStringLines(e.query)};\n`) + .join('\n') + + '\n} // namespace Markdown\n'; + + const header = + '// Generated by vendor/gen-registry.mjs. Do not edit.\n\n' + + '#include "HighlightGrammars.hpp"\n' + + '#include "generated_queries.h"\n\n' + + '#include \n\n'; + + let registryCpp; + if (entries.length === 0) { + registryCpp = + header + + 'namespace Markdown {\n\n' + + 'const GrammarEntry *findGrammar(const char * /*canonicalId*/) {\n return nullptr;\n}\n\n' + + '} // namespace Markdown\n'; + } else { + registryCpp = + header + + 'extern "C" {\n' + + entries.map((e) => `const TSLanguage *${e.fn}(void);`).join('\n') + + '\n}\n\n' + + 'namespace Markdown {\nnamespace {\n\n' + + 'constexpr GrammarEntry kGrammars[] = {\n' + + entries.map((e) => ` {"${e.id}", ${e.fn}, kHighlightsQuery_${e.ident}},`).join('\n') + + '\n};\n\n' + + '} // namespace\n\n' + + 'const GrammarEntry *findGrammar(const char *canonicalId) {\n' + + ' if (canonicalId == nullptr) {\n return nullptr;\n }\n' + + ' for (const GrammarEntry &entry : kGrammars) {\n' + + ' if (std::strcmp(entry.canonicalId, canonicalId) == 0) {\n return &entry;\n }\n }\n' + + ' return nullptr;\n}\n\n' + + '} // namespace Markdown\n'; + } + + fs.writeFileSync(path.join(args.out, 'generated_queries.h'), queriesHeader); + fs.writeFileSync(path.join(args.out, 'generated_registry.cpp'), registryCpp); + log(`wrote ${entries.length} grammar(s) to ${args.out}`); +} + +main(); diff --git a/vendor/grammar-versions.json b/vendor/grammar-versions.json new file mode 100644 index 000000000..3ed99907f --- /dev/null +++ b/vendor/grammar-versions.json @@ -0,0 +1,67 @@ +{ + "$comment": "Pins the tree-sitter runtime and every supported grammar for vendor-grammars.mjs. Grammars marked default:true form the curated set compiled unless a consumer overrides the language list. Each grammar package is a devDependency of react-native-enriched-markdown and is never shipped; only the copied .c/.h/.scm files are vendored. Versions must generate an ABI in [13,15] to match the runtime. Re-pin and re-run vendor-grammars.mjs to upgrade.", + "runtime": { + "$comment": "tree-sitter runtime C amalgamation. Fetched at install/prepack from the pinned GitHub release tarball (lib/src + lib/include) into the gitignored vendor/tree-sitter/; WASM is compiled out by leaving TREE_SITTER_FEATURE_WASM undefined (never strip wasm_store.c, it provides the no-op stubs parser.c/language.c call). To re-pin: bump ref, run `node vendor/vendor-grammars.mjs --force`, and paste the sha256 it prints on mismatch. tarball is optional; when absent it is derived from ref.", + "package": "tree-sitter", + "ref": "v0.26.3", + "tarball": "https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v0.26.3.tar.gz", + "sha256": "7f4a7cf0a2cd217444063fe2a4d800bc9d21ed609badc2ac20c0841d67166550", + "abi": { "version": 15, "minCompatible": 13 } + }, + "grammars": { + "json": { "package": "tree-sitter-json", "version": "0.24.8", "scanner": false, "default": true, "size": "27 KB" }, + "html": { "package": "tree-sitter-html", "version": "0.23.2", "scanner": true, "default": true, "size": "76 KB" }, + "css": { "package": "tree-sitter-css", "version": "0.23.2", "scanner": true, "default": true, "size": "0.5 MB" }, + "markdown": { + "package": "@tree-sitter-grammars/tree-sitter-markdown", + "version": "0.3.2", + "subPath": "tree-sitter-markdown", + "scanner": true, + "default": true, + "size": "1.5 MB", + "$comment": "Multi-grammar package; the block grammar (tree-sitter-markdown/) is vendored. The inline sub-grammar is not linked." + }, + "yaml": { "package": "@tree-sitter-grammars/tree-sitter-yaml", "version": "0.7.1", "scanner": true, "default": true, "size": "1 MB" }, + "go": { "package": "tree-sitter-go", "version": "0.23.4", "scanner": false, "default": true, "size": "1 MB" }, + "java": { "package": "tree-sitter-java", "version": "0.23.5", "scanner": false, "default": true, "size": "2.5 MB" }, + "javascript": { "package": "tree-sitter-javascript", "version": "0.23.1", "scanner": true, "default": true, "size": "2.8 MB" }, + "python": { "package": "tree-sitter-python", "version": "0.23.6", "scanner": true, "default": true, "size": "3.4 MB" }, + "c": { "package": "tree-sitter-c", "version": "0.23.5", "scanner": false, "default": true, "size": "3.8 MB" }, + "rust": { "package": "tree-sitter-rust", "version": "0.23.2", "scanner": true, "default": true, "size": "6.3 MB" }, + "bash": { "package": "tree-sitter-bash", "version": "0.23.3", "scanner": true, "default": true, "size": "9.7 MB" }, + + "typescript": { + "package": "tree-sitter-typescript", + "version": "0.23.2", + "subPath": "typescript", + "scanner": true, + "sharedSrc": ["../../common/scanner.h"], + "inherits": ["javascript"], + "default": true, + "size": "17 MB" + }, + "tsx": { + "package": "tree-sitter-typescript", + "version": "0.23.2", + "subPath": "tsx", + "scanner": true, + "sharedSrc": ["../../common/scanner.h"], + "inherits": ["javascript"], + "default": true, + "size": "17 MB" + }, + "cpp": { "package": "tree-sitter-cpp", "version": "0.23.4", "scanner": true, "inherits": ["c"], "default": false, "size": "17 MB" }, + "swift": { "package": "tree-sitter-swift", "version": "0.7.0", "scanner": true, "default": false, "size": "18 MB" }, + "php": { + "package": "tree-sitter-php", + "version": "0.24.2", + "subPath": "php", + "scanner": true, + "sharedSrc": ["../../common/scanner.h"], + "default": false, + "size": "14 MB" + }, + "ruby": { "package": "tree-sitter-ruby", "version": "0.23.1", "scanner": true, "default": false, "size": "15 MB" }, + "c-sharp": { "package": "tree-sitter-c-sharp", "version": "0.23.1", "scanner": true, "function": "tree_sitter_c_sharp", "default": false, "size": "29 MB" } + } +} diff --git a/vendor/vendor-grammars.mjs b/vendor/vendor-grammars.mjs new file mode 100644 index 000000000..35c4a27cd --- /dev/null +++ b/vendor/vendor-grammars.mjs @@ -0,0 +1,377 @@ +#!/usr/bin/env node +// Restores the tree-sitter runtime, each supported grammar's minimal source set, +// and the default-language registry into packages/core/cpp/highlight/vendor/. +// The whole vendor/ tree (runtime, grammars, generated registry) is GITIGNORED +// and reproduced on demand from the pins in grammar-versions.json, so nothing +// generated lives in git. This one command is wired into the package `prepare` +// hook (run `yarn prepare` to self-heal the working tree; note this repo is Yarn 4 +// (Berry), where a plain `yarn install` does NOT run the workspace `prepare` +// script) and into `prepack` (so the published npm tarball still ships the full +// set). +// +// node vendor/vendor-grammars.mjs +// Ensures the runtime (fetched from the pinned GitHub release tarball), every +// grammar (copied from the grammar devDependencies), and the default registry +// are present and up to date. Idempotent: .stamp fingerprints on the runtime +// ref+sha and on the pinned grammar versions make repeated runs a no-op, so +// re-running `prepare` never re-fetches or rewrites an up-to-date tree. +// +// Flags: +// --only json,css Restore just these grammars (dev/testing). Skips the +// default-registry refresh and the stamp, so it never +// clobbers the full vendored set. +// --force Re-fetch the runtime and rewrite every grammar regardless +// of the stamps. +// --runtime-src Use a local tree-sitter lib/ checkout instead of +// fetching the tarball (offline / maintainer override). +// +// Only parser.c, scanner.c (when present), tree_sitter/*.h, highlights.scm and +// LICENSE are copied per grammar; nothing else from the grammar packages ships. +// WASM is left out of the runtime by never defining TREE_SITTER_FEATURE_WASM at +// build time, so the full lib/src tree (including wasm_store.c and its no-op +// stubs) is vendored verbatim and only lib.c is compiled. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { spawnSync, } from 'node:child_process'; +import { createRequire } from 'node:module'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '..'); +const vendorOut = path.join(repoRoot, 'packages/core/cpp/highlight/vendor'); +const manifestPath = path.join(here, 'grammar-versions.json'); + +const LOG_PREFIX = '[react-native-enriched-markdown]'; +const log = (message) => console.log(`${LOG_PREFIX} ${message}`); +const warn = (message) => console.warn(`${LOG_PREFIX} ${message}`); + +// Grammar packages are devDependencies of react-native-enriched-markdown, so +// resolve them from that workspace regardless of hoisting. +const pkgRequire = createRequire( + path.join(repoRoot, 'packages/react-native-enriched-markdown/package.json') +); + +function parseArgs(argv) { + const args = { only: null, runtimeSrc: null, force: false }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--only') args.only = argv[++i].split(',').map((s) => s.trim()); + else if (argv[i] === '--runtime-src') args.runtimeSrc = argv[++i]; + else if (argv[i] === '--force') args.force = true; + // --grammars-only is accepted for backward compatibility; the single restore + // flow already covers grammars, so it is a no-op. + else if (argv[i] === '--grammars-only') continue; + } + return args; +} + +function fail(message) { + console.error(`${LOG_PREFIX} ${message}`); + process.exit(1); +} + +function copyFile(src, dest) { + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.copyFileSync(src, dest); +} + +function copyDir(src, dest) { + fs.mkdirSync(dest, { recursive: true }); + for (const entry of fs.readdirSync(src, { withFileTypes: true })) { + const from = path.join(src, entry.name); + const to = path.join(dest, entry.name); + if (entry.isDirectory()) copyDir(from, to); + else copyFile(from, to); + } +} + +function firstExisting(candidates) { + return candidates.find((p) => p && fs.existsSync(p)) ?? null; +} + +// Copies lib/src + lib/include/tree_sitter/api.h from a tree-sitter `lib` +// checkout into the vendor tree. Shared by the tarball and --runtime-src paths. +function copyRuntimeFromLib(libDir) { + const srcDir = path.join(libDir, 'src'); + const apiHeader = path.join(libDir, 'include/tree_sitter/api.h'); + if (!fs.existsSync(path.join(srcDir, 'lib.c')) || !fs.existsSync(apiHeader)) { + fail(`runtime source at ${libDir} is missing src/lib.c or include/tree_sitter/api.h`); + } + const outSrc = path.join(vendorOut, 'tree-sitter/src'); + fs.rmSync(outSrc, { recursive: true, force: true }); + copyDir(srcDir, outSrc); + copyFile(apiHeader, path.join(vendorOut, 'tree-sitter/include/tree_sitter/api.h')); +} + +function runtimeTarballUrl(runtime) { + return runtime.tarball ?? `https://github.com/tree-sitter/tree-sitter/archive/refs/tags/${runtime.ref}.tar.gz`; +} + +// Downloads the pinned runtime tarball, verifies its sha256 against the manifest, +// extracts it to a temp dir with the system `tar`, and returns the `lib/` path. +// A sha mismatch is fatal and prints the computed digest to paste back into the +// manifest on a deliberate re-pin. +async function fetchRuntimeLib(runtime) { + const url = runtimeTarballUrl(runtime); + if (!runtime.sha256) { + fail(`runtime.sha256 missing in grammar-versions.json; cannot verify ${url}`); + } + log(`fetching runtime ${runtime.ref} from ${url}`); + let buf; + try { + const res = await fetch(url); + if (!res.ok) fail(`runtime download failed: ${res.status} ${res.statusText} for ${url}`); + buf = Buffer.from(await res.arrayBuffer()); + } catch (err) { + fail(`runtime download failed for ${url}: ${err.message}. Pass --runtime-src to vendor from a local checkout offline.`); + } + const digest = crypto.createHash('sha256').update(buf).digest('hex'); + if (digest !== runtime.sha256) { + fail( + `runtime tarball sha256 mismatch for ${url}\n expected ${runtime.sha256}\n got ${digest}\n` + + 'If this is a deliberate re-pin, update runtime.sha256 in grammar-versions.json to the "got" value.' + ); + } + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ts-runtime-')); + const tgz = path.join(tmp, 'runtime.tar.gz'); + fs.writeFileSync(tgz, buf); + const untar = spawnSync('tar', ['-xzf', tgz, '-C', tmp], { stdio: 'inherit' }); + if (untar.status !== 0) fail(`tar failed to extract ${tgz} (status ${untar.status})`); + const top = fs + .readdirSync(tmp, { withFileTypes: true }) + .find((e) => e.isDirectory() && e.name.startsWith('tree-sitter-')); + if (!top) fail(`extracted runtime tarball has no tree-sitter-* directory in ${tmp}`); + return { libDir: path.join(tmp, top.name, 'lib'), tmp }; +} + +// Fingerprint of the pinned runtime: ref + sha (+ 'local' when sourced from a +// checkout). A re-pin invalidates it; matching stamp + present lib.c is a no-op. +function runtimeStampKey(runtime, fromLocal) { + return `${runtime.ref}|${runtime.sha256 ?? 'nosha'}|${fromLocal ? 'local' : 'tarball'}`; +} + +async function ensureRuntime(manifest, args) { + const runtime = manifest.runtime ?? {}; + const localLib = firstExisting([args.runtimeSrc, process.env.TREE_SITTER_SRC].filter(Boolean)); + const stampFile = path.join(vendorOut, 'tree-sitter/.stamp'); + const stampKey = runtimeStampKey(runtime, !!localLib); + const present = fs.existsSync(path.join(vendorOut, 'tree-sitter/src/lib.c')); + + if (!args.force && present && readStamp(stampFile) === stampKey) { + log('runtime already up to date; skipping.'); + return; + } + + if (localLib) { + copyRuntimeFromLib(localLib); + } else { + const { libDir, tmp } = await fetchRuntimeLib(runtime); + try { + copyRuntimeFromLib(libDir); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + } + fs.writeFileSync(stampFile, stampKey + '\n'); + log(`runtime -> ${path.relative(repoRoot, path.join(vendorOut, 'tree-sitter/src'))}`); +} + +function packageRoot(pkgName) { + try { + return path.dirname(pkgRequire.resolve(`${pkgName}/package.json`)); + } catch { + fail(`grammar package ${pkgName} not installed. Add it as a devDependency and run yarn install.`); + return ''; + } +} + +// Resolves a grammar's highlights.scm from its package (null if it ships none). +// A grammar with no highlights query cannot be compiled into the registry, so +// callers skip it. Kept separate so main() can pre-filter without side effects. +function grammarHighlights(spec) { + const pkgRoot = packageRoot(spec.package); + const base = spec.subPath ? path.join(pkgRoot, spec.subPath) : pkgRoot; + return firstExisting([ + path.join(base, 'queries/highlights.scm'), + path.join(pkgRoot, 'queries/highlights.scm'), + ]); +} + +function vendorGrammar(id, spec) { + const pkgRoot = packageRoot(spec.package); + const base = spec.subPath ? path.join(pkgRoot, spec.subPath) : pkgRoot; + const srcDir = path.join(base, 'src'); + const outDir = path.join(vendorOut, 'grammars', id); + + const parser = path.join(srcDir, 'parser.c'); + if (!fs.existsSync(parser)) fail(`${id}: parser.c not found at ${parser}`); + + if (spec.scanner && !fs.existsSync(path.join(srcDir, 'scanner.c'))) { + fail(`${id}: scanner:true but scanner.c missing at ${srcDir}`); + } + + // Resolve highlights before writing anything so a highlights-less grammar + // never leaves a partial output dir (main() filters these out up front). + const highlights = grammarHighlights(spec); + if (!highlights) fail(`${id}: queries/highlights.scm not found under ${base} or ${pkgRoot}`); + + fs.rmSync(outDir, { recursive: true, force: true }); + + // Copy every loose .c/.h in src/ (parser.c, scanner.c, plus siblings a scanner + // text-includes such as html's tag.h or yaml's schema.*.c). Only parser.c and + // scanner.c are compiled; the rest are include-only. node-types.json, + // grammar.json and other non-source files are left behind. + for (const entry of fs.readdirSync(srcDir, { withFileTypes: true })) { + if (entry.isFile() && /\.(c|h)$/.test(entry.name)) { + copyFile(path.join(srcDir, entry.name), path.join(outDir, entry.name)); + } + } + + const headerDir = path.join(srcDir, 'tree_sitter'); + if (fs.existsSync(headerDir)) copyDir(headerDir, path.join(outDir, 'tree_sitter')); + + copyFile(highlights, path.join(outDir, 'highlights.scm')); + + const license = firstExisting([ + path.join(pkgRoot, 'LICENSE'), + path.join(pkgRoot, 'LICENSE.md'), + path.join(pkgRoot, 'LICENSE.txt'), + ]); + if (license) copyFile(license, path.join(outDir, 'LICENSE')); + + // Localize any header a source file includes from outside its own src/ dir. + // tree-sitter-typescript's typescript and tsx grammars both text-include + // "../../common/scanner.h", a header shared at the package root that the + // per-grammar copy above (which only walks src/) leaves behind. Copy each into + // the grammar dir under its basename and rewrite the escaping include to that + // basename, so it resolves through the grammar's own quoted includes -- its + // tree_sitter/parser.h is vendored alongside -- with no shared search path. + for (const rel of spec.sharedSrc ?? []) { + const from = path.join(srcDir, rel); + if (!fs.existsSync(from)) fail(`${id}: sharedSrc '${rel}' not found at ${from}`); + const base = path.basename(rel); + copyFile(from, path.join(outDir, base)); + for (const entry of fs.readdirSync(outDir, { withFileTypes: true })) { + if (!entry.isFile() || !/\.(c|h)$/.test(entry.name)) continue; + const filePath = path.join(outDir, entry.name); + const text = fs.readFileSync(filePath, 'utf8'); + const rewritten = text.split(`"${rel}"`).join(`"${base}"`); + if (rewritten !== text) fs.writeFileSync(filePath, rewritten); + } + } + + log(`${id} -> ${path.relative(repoRoot, outDir)}`); +} + +function regenerateDefaultRegistry(manifest) { + const defaults = Object.entries(manifest.grammars) + .filter(([, spec]) => spec.default) + .map(([id]) => id); + const outDir = path.join(vendorOut, 'generated'); + const result = spawnSync( + process.execPath, + [ + path.join(here, 'gen-registry.mjs'), + '--vendor-dir', + vendorOut, + '--languages', + defaults.join(','), + '--out', + outDir, + ], + { stdio: 'inherit' } + ); + if (result.status !== 0) fail('gen-registry.mjs failed while refreshing the committed default set'); +} + +function requireSpec(manifest, id) { + const spec = manifest.grammars[id]; + if (!spec) fail(`unknown grammar '${id}' (not in grammar-versions.json)`); + return spec; +} + +// Fingerprint of the pinned grammar set: manifest spec plus each grammar +// package's installed version, so a re-pin OR a node_modules change invalidates. +function grammarStampKey(manifest, ids) { + const parts = ids.map((id) => { + const spec = requireSpec(manifest, id); + let version = 'missing'; + try { + version = pkgRequire(`${spec.package}/package.json`).version; + } catch { + /* resolved lazily during copy; a miss just forces a rebuild */ + } + return `${id}|${spec.package}@${version}|scanner:${!!spec.scanner}|sub:${spec.subPath ?? ''}|shared:${(spec.sharedSrc ?? []).join('+')}`; + }); + return crypto.createHash('sha256').update(JSON.stringify(parts)).digest('hex'); +} + +function everyGrammarPresent(ids) { + return ids.every((id) => fs.existsSync(path.join(vendorOut, 'grammars', id, 'parser.c'))); +} + +function readStamp(stampFile) { + return fs.existsSync(stampFile) ? fs.readFileSync(stampFile, 'utf8').trim() : null; +} + +// Drops grammars whose package ships no highlights.scm (they cannot be compiled +// into any registry). A default-set grammar missing one is a hard error; others +// are skipped with a warning and any stale output removed. +function vendorableIds(manifest, ids) { + const out = []; + for (const id of ids) { + const spec = requireSpec(manifest, id); + if (grammarHighlights(spec)) { + out.push(id); + } else if (spec.default) { + fail(`${id} is in the default set but its package ships no queries/highlights.scm`); + } else { + fs.rmSync(path.join(vendorOut, 'grammars', id), { recursive: true, force: true }); + warn(`${id}: package ships no highlights.scm; skipping (non-default, not compilable).`); + } + } + return out; +} + +function registryPresent() { + return fs.existsSync(path.join(vendorOut, 'generated/generated_registry.cpp')); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + const requested = args.only ?? Object.keys(manifest.grammars); + const ids = vendorableIds(manifest, requested); + const stampFile = path.join(vendorOut, 'grammars', '.stamp'); + // Only a full-set run may trust or write the stamp; a partial --only run must not. + const stampKey = args.only ? null : grammarStampKey(manifest, ids); + + // Runtime: fetched from the pinned tarball (gitignored, idempotent via stamp). + await ensureRuntime(manifest, args); + + // Grammars: restored from the grammar devDependencies. Skip when the vendored + // set already matches the pinned versions so a repeat `prepare` does not + // rewrite 34 MB on every run. + let grammarsRebuilt = false; + if (!args.force && stampKey && everyGrammarPresent(ids) && readStamp(stampFile) === stampKey) { + log('grammar sources already up to date; skipping.'); + } else { + for (const id of ids) vendorGrammar(id, requireSpec(manifest, id)); + if (stampKey) fs.writeFileSync(stampFile, stampKey + '\n'); + grammarsRebuilt = true; + log('grammar sources ready.'); + } + + // Default registry: regenerated from the vendored grammars. A partial --only + // run must not touch it (the default set may not all be vendored). Otherwise + // refresh it whenever the grammars changed or it is missing. + if (!args.only && (args.force || grammarsRebuilt || !registryPresent())) { + regenerateDefaultRegistry(manifest); + } + + log('done.'); +} + +main().catch((err) => fail(err && err.stack ? err.stack : String(err))); diff --git a/yarn.lock b/yarn.lock index bdfed3ea3..1ee946633 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7173,6 +7173,38 @@ __metadata: languageName: node linkType: hard +"@tree-sitter-grammars/tree-sitter-markdown@npm:0.3.2": + version: 0.3.2 + resolution: "@tree-sitter-grammars/tree-sitter-markdown@npm:0.3.2" + dependencies: + node-addon-api: "npm:^8.1.0" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.1" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree_sitter: + optional: true + checksum: 10c0/2f327c8839bdc0745f4df9e12e159293abe616976c7447328ec9f50b71419e24df0ccc1a88ff03fcafebbfc87a71b2cdeeb169208369a78231fcf29d8deb6b4b + languageName: node + linkType: hard + +"@tree-sitter-grammars/tree-sitter-yaml@npm:0.7.1": + version: 0.7.1 + resolution: "@tree-sitter-grammars/tree-sitter-yaml@npm:0.7.1" + dependencies: + node-addon-api: "npm:^8.3.1" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.4" + peerDependencies: + tree-sitter: ^0.22.4 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/29191c5f7acd6d0ba9f2f6aa2a3b83b6c9f60d70286399be37ec3f2eed78152e3777632855dfcb9e988d934bae10817bb4c4ba38ad0786f2303fbb4806f7bcae + languageName: node + linkType: hard + "@turbo/darwin-64@npm:2.9.16": version: 2.9.16 resolution: "@turbo/darwin-64@npm:2.9.16" @@ -14619,6 +14651,15 @@ __metadata: languageName: node linkType: hard +"node-addon-api@npm:^8.0.0, node-addon-api@npm:^8.1.0, node-addon-api@npm:^8.2.1, node-addon-api@npm:^8.2.2, node-addon-api@npm:^8.3.0, node-addon-api@npm:^8.3.1": + version: 8.9.1 + resolution: "node-addon-api@npm:8.9.1" + dependencies: + node-gyp: "npm:latest" + checksum: 10c0/189160497de690479030b7b2f5641c38ffab8601c2ada6fcb8fe3718baf14af51d651528a49ec58de5dd602ff7858459d13facb71914669106f2c6cc6d9d50a5 + languageName: node + linkType: hard + "node-fetch-native@npm:^1.6.6": version: 1.6.7 resolution: "node-fetch-native@npm:1.6.7" @@ -14647,6 +14688,17 @@ __metadata: languageName: node linkType: hard +"node-gyp-build@npm:^4.8.0, node-gyp-build@npm:^4.8.1, node-gyp-build@npm:^4.8.2, node-gyp-build@npm:^4.8.4": + version: 4.8.4 + resolution: "node-gyp-build@npm:4.8.4" + bin: + node-gyp-build: bin.js + node-gyp-build-optional: optional.js + node-gyp-build-test: build-test.js + checksum: 10c0/444e189907ece2081fe60e75368784f7782cfddb554b60123743dfb89509df89f1f29c03bbfa16b3a3e0be3f48799a4783f487da6203245fa5bed239ba7407e1 + languageName: node + linkType: hard + "node-gyp@npm:latest": version: 11.5.0 resolution: "node-gyp@npm:11.5.0" @@ -15990,6 +16042,45 @@ __metadata: release-it: "npm:^19.2.4" turbo: "npm:^2.8.21" typescript: "npm:^6.0.2" + dependenciesMeta: + "@tree-sitter-grammars/tree-sitter-markdown": + built: false + "@tree-sitter-grammars/tree-sitter-yaml": + built: false + tree-sitter-bash: + built: false + tree-sitter-c: + built: false + tree-sitter-c-sharp: + built: false + tree-sitter-cli: + built: false + tree-sitter-cpp: + built: false + tree-sitter-css: + built: false + tree-sitter-go: + built: false + tree-sitter-html: + built: false + tree-sitter-java: + built: false + tree-sitter-javascript: + built: false + tree-sitter-json: + built: false + tree-sitter-php: + built: false + tree-sitter-python: + built: false + tree-sitter-ruby: + built: false + tree-sitter-rust: + built: false + tree-sitter-swift: + built: false + tree-sitter-typescript: + built: false languageName: unknown linkType: soft @@ -16017,6 +16108,8 @@ __metadata: dependencies: "@expo/config-plugins": "npm:^55.0.6" "@react-native/babel-preset": "npm:0.85.0" + "@tree-sitter-grammars/tree-sitter-markdown": "npm:0.3.2" + "@tree-sitter-grammars/tree-sitter-yaml": "npm:0.7.1" "@types/node": "npm:^22.0.0" "@types/react": "npm:^19.2.0" clang-format: "npm:^1.8.0" @@ -16024,6 +16117,22 @@ __metadata: react: "npm:19.2.3" react-native: "npm:0.85.0" react-native-builder-bob: "npm:^0.41.0" + tree-sitter-bash: "npm:0.23.3" + tree-sitter-c: "npm:0.23.5" + tree-sitter-c-sharp: "npm:0.23.1" + tree-sitter-cpp: "npm:0.23.4" + tree-sitter-css: "npm:0.23.2" + tree-sitter-go: "npm:0.23.4" + tree-sitter-html: "npm:0.23.2" + tree-sitter-java: "npm:0.23.5" + tree-sitter-javascript: "npm:0.23.1" + tree-sitter-json: "npm:0.24.8" + tree-sitter-php: "npm:0.24.2" + tree-sitter-python: "npm:0.23.6" + tree-sitter-ruby: "npm:0.23.1" + tree-sitter-rust: "npm:0.23.2" + tree-sitter-swift: "npm:0.7.0" + tree-sitter-typescript: "npm:0.23.2" typescript: "npm:^6.0.2" peerDependencies: "@expo/config-plugins": ">=50.0.0" @@ -17969,6 +18078,291 @@ __metadata: languageName: node linkType: hard +"tree-sitter-bash@npm:0.23.3": + version: 0.23.3 + resolution: "tree-sitter-bash@npm:0.23.3" + dependencies: + node-addon-api: "npm:^8.2.1" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/b491001852b6211a5d4c0fafe7bacff0ce9f13ca39927ba6816435e706bb83f555ac34828733513b21adeb73ec63e9753ae01a7615cecb97a16cab73982073ac + languageName: node + linkType: hard + +"tree-sitter-c-sharp@npm:0.23.1": + version: 0.23.1 + resolution: "tree-sitter-c-sharp@npm:0.23.1" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/77192470aae8883585e75071ca098f7b2bab14608a81ee94319865583b8774dc70539c50bb19be947e332d23fec94f328bfab098dd129535f926c914a1f05f90 + languageName: node + linkType: hard + +"tree-sitter-c@npm:0.23.5": + version: 0.23.5 + resolution: "tree-sitter-c@npm:0.23.5" + dependencies: + node-addon-api: "npm:^8.3.0" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.4" + peerDependencies: + tree-sitter: ^0.22.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/b744cad04b862af24c40fa618f36124091bffc8d2ac6ebe627632348e255c75df680780f80f27a4ce0f2a4d54ed5223e46e0dbef9b4583bdb7e834a28b17c814 + languageName: node + linkType: hard + +"tree-sitter-c@npm:^0.23.1": + version: 0.23.6 + resolution: "tree-sitter-c@npm:0.23.6" + dependencies: + node-addon-api: "npm:^8.3.0" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.4" + peerDependencies: + tree-sitter: ^0.22.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/ac6d4000a8e235443903c1d352f9866cd2c94501b9fce2f0520d15d95a0ca8f5b68f79573ab9520eb86f084cac70edbd8fa5cf4505b7c5caab6e70850e7d4169 + languageName: node + linkType: hard + +"tree-sitter-cli@npm:^0.23": + version: 0.23.2 + resolution: "tree-sitter-cli@npm:0.23.2" + bin: + tree-sitter: cli.js + checksum: 10c0/26bc30b881992efd07c6da36d07de37594f16e5388ea38d3b6e59df5b3ba62620d60e5a19a9f9f3c95d0627edd52dda1a6134328e8d01235923aebed7bb389cf + languageName: node + linkType: hard + +"tree-sitter-cpp@npm:0.23.4": + version: 0.23.4 + resolution: "tree-sitter-cpp@npm:0.23.4" + dependencies: + node-addon-api: "npm:^8.2.1" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + tree-sitter-c: "npm:^0.23.1" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/90c8c5c21630ea062ee1aafce6d5552140cc37e04d5e06f01c6176eff37cc74da021114018556660ca0c31ca768d04db4b8f3706461d0b68061c7479f7b54911 + languageName: node + linkType: hard + +"tree-sitter-css@npm:0.23.2": + version: 0.23.2 + resolution: "tree-sitter-css@npm:0.23.2" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.22.4 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/a64ce0106a297e1cedf532db7198e38bcdeffda5e3292fd0ff691ee406947757ad84d8eacdd9b1ae737aeea1cb0d6da45ba7e7d3302c105ec5c0781becde31a4 + languageName: node + linkType: hard + +"tree-sitter-go@npm:0.23.4": + version: 0.23.4 + resolution: "tree-sitter-go@npm:0.23.4" + dependencies: + node-addon-api: "npm:^8.2.1" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/a237e7405c9bbd14d1e6da15e3dba4cb2669ab90509b83375886f8849e683d4ca2648584fd66db574f5e133fa1e1eee8a9999fe3aaf00267bda3169cafee6714 + languageName: node + linkType: hard + +"tree-sitter-html@npm:0.23.2": + version: 0.23.2 + resolution: "tree-sitter-html@npm:0.23.2" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/cb5930d2353ddf3d4c831176d884bfeea6d191d1a3a916ae2be80e0ebffbb8aa55e63a9109c086aa6f2704afd203b5490ee79749a2bd93ef2d85858cd2efaa02 + languageName: node + linkType: hard + +"tree-sitter-java@npm:0.23.5": + version: 0.23.5 + resolution: "tree-sitter-java@npm:0.23.5" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/526ea766a1f6803ffa6491e10dc5e265e31a48659be82a95c080724305b142877de28f4bc79e6ae5a0aed16fafa04a9f48c6b1180c7f8b130078a6211fe870f6 + languageName: node + linkType: hard + +"tree-sitter-javascript@npm:0.23.1, tree-sitter-javascript@npm:^0.23.1": + version: 0.23.1 + resolution: "tree-sitter-javascript@npm:0.23.1" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/0b5cefcf4d072bbd5807603355cd87b349a0e2423d76ff64a39b24a150690d1f30424e537f8f6c11c2ac62613abe37d7f06f513e1dee8f07b04f714aba4cec40 + languageName: node + linkType: hard + +"tree-sitter-json@npm:0.24.8": + version: 0.24.8 + resolution: "tree-sitter-json@npm:0.24.8" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/cc528770947d4b24e2a389b8c9f9ac103841a4447278903700cbb893414d21564366a9d9c0fc1c6b7c2d4bbe311b8622e95f3b4df3886ee3c8e2e44c839261c1 + languageName: node + linkType: hard + +"tree-sitter-php@npm:0.24.2": + version: 0.24.2 + resolution: "tree-sitter-php@npm:0.24.2" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.22.4 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/f8fec1356fa33491adbefdb17301a7f059dbf621e8b5b2c06c588e8ac1feda462d61c074f2240926eb86f7425faf4fcea309f882d15b1fb13f775e0b5348b740 + languageName: node + linkType: hard + +"tree-sitter-python@npm:0.23.6": + version: 0.23.6 + resolution: "tree-sitter-python@npm:0.23.6" + dependencies: + node-addon-api: "npm:^8.3.0" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.4" + peerDependencies: + tree-sitter: ^0.22.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/8101e5c23c1536e619b47f7a9460bc212ac9e77b45200666832531bb40bfe9d989235058ba75a610bc14df63fd47f1d8d3312acb1e38932d9673fd5c2d8f9652 + languageName: node + linkType: hard + +"tree-sitter-ruby@npm:0.23.1": + version: 0.23.1 + resolution: "tree-sitter-ruby@npm:0.23.1" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + peerDependencies: + tree-sitter: ^0.21.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/b4d26e371f6543a1c978e4ae45ac0674e4610bd0e08a0b414571a96bf99e56586b547795b1e926264182e3ba5d07cbac4b6824bccd87210ddf30aa1c106a83f0 + languageName: node + linkType: hard + +"tree-sitter-rust@npm:0.23.2": + version: 0.23.2 + resolution: "tree-sitter-rust@npm:0.23.2" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.4" + peerDependencies: + tree-sitter: ^0.22.1 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/a5e8cd43e1f78b40f6aa81f56ea43991776e92ceef5157c3b2a3d4cca37c21e0ab5ed251fbd54a6f338aadea60b486cd9ad8191e249d34fde6b184c51cbd3279 + languageName: node + linkType: hard + +"tree-sitter-swift@npm:0.7.0": + version: 0.7.0 + resolution: "tree-sitter-swift@npm:0.7.0" + dependencies: + node-addon-api: "npm:^8.0.0" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.0" + tree-sitter-cli: "npm:^0.23" + which: "npm:2.0.2" + peerDependencies: + tree-sitter: ^0.22.1 + peerDependenciesMeta: + tree_sitter: + optional: true + checksum: 10c0/7f8ef124043737227ebb9667941746a405524a262f2bc90847f86b0875f46622e70b24ddcafa35760443ef3f51314633ee675ecf91c3a92ec8a65bc013db4d21 + languageName: node + linkType: hard + +"tree-sitter-typescript@npm:0.23.2": + version: 0.23.2 + resolution: "tree-sitter-typescript@npm:0.23.2" + dependencies: + node-addon-api: "npm:^8.2.2" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.2" + tree-sitter-javascript: "npm:^0.23.1" + peerDependencies: + tree-sitter: ^0.21.0 + peerDependenciesMeta: + tree-sitter: + optional: true + checksum: 10c0/bfee4884952fa9446e117ba4683348f805b6c9f1eb8013d45decc2ba6193ed0e984df3e920890a82157efa611d43bcd581613bd757b087336f44a8617ce4c7b4 + languageName: node + linkType: hard + "ts-api-utils@npm:^2.4.0": version: 2.4.0 resolution: "ts-api-utils@npm:2.4.0" @@ -18606,7 +19000,7 @@ __metadata: languageName: node linkType: hard -"which@npm:^2.0.1": +"which@npm:2.0.2, which@npm:^2.0.1": version: 2.0.2 resolution: "which@npm:2.0.2" dependencies: