feat: tressiter for colored codeblocks

.gitignore

@@ -92,6 +92,19 @@ web-build/
 **/ios/generated
 **/android/generated
 
+# Vendored tree-sitter sources (machine-generated C, ~178 MB total). Nothing here
+# is tracked: the runtime is fetched from the pinned GitHub release tarball, the
+# grammar parse tables come from the grammar devDependencies, and the default
+# registry is codegen'd from them. All three are restored by
+# vendor/vendor-grammars.mjs via the `prepare` hook (`yarn prepare`) and baked
+# into the npm tarball on prepack. See docs and vendor/grammar-versions.json.
+packages/core/cpp/highlight/vendor/tree-sitter/
+packages/core/cpp/highlight/vendor/grammars/
+packages/core/cpp/highlight/vendor/generated/
+# Registry for a custom ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES set (iOS); kept
+# out of the committed generated/ dir so a custom build never clobbers the default.
+packages/core/cpp/highlight/vendor/generated-custom/
+
 # React Native Nitro Modules
 nitrogen/
 

.yarnrc.yml

@@ -2,3 +2,24 @@ nodeLinker: node-modules
 nmHoistingLimits: workspaces
 
 yarnPath: .yarn/releases/yarn-4.11.0.cjs
+
+# The tree-sitter grammars are vendored from source (see vendor/vendor-grammars.mjs);
+# we never load their compiled Node bindings. @tree-sitter-grammars/tree-sitter-markdown
+# and tree-sitter-swift ship a typo'd peerDependenciesMeta key ("tree_sitter" instead of
+# "tree-sitter"), so Yarn treats their optional "tree-sitter" peer as required and warns.
+# Re-declare it optional. (Their native builds are disabled via dependenciesMeta in package.json.)
+packageExtensions:
+  '@tree-sitter-grammars/tree-sitter-markdown@*':
+    peerDependenciesMeta:
+      tree-sitter:
+        optional: true
+  'tree-sitter-swift@*':
+    peerDependenciesMeta:
+      tree-sitter:
+        optional: true
+
+# The only packages with builds disabled above are those vendored tree-sitter grammars,
+# so silence the per-package "build has been explicitly disabled" notice (YN0005).
+logFilters:
+  - code: YN0005
+    level: discard

apps/react-native-example/.rnstorybook/stories/components/EnrichedMarkdownText/block/CodeHighlight.stories.tsx

@@ -5,79 +5,146 @@ import { githubFlavorArgTypes } from '../shared/storybookMarkdownStyles';
 import { splitStyleControls } from '../shared/storybookStyleBuilders';
 import type { TextStory } from '../shared/storyTypes';
 
+// A per-language sample split into two labelled sections so a tester can verify
+// language coverage at a glance (see docs/TESTING_CODE_HIGHLIGHT_VENDORING.md,
+// step 4): the "Defaults" section lists every language in the default registry,
+// which highlights out of the box; the "Extended List" section lists the
+// non-default grammars, which stay plain unless a build opts them in via the
+// language-subset override (steps 4a/5a). One short block per language.
 const MARKDOWN = [
+  '## Defaults',
+  '',
   '```javascript',
-  'const greet = (name) => `hi ${name}`; // arrow fn',
-  'export default greet(42);',
+  'const sum = (a, b) => a + b; // add two numbers',
   '```',
   '',
   '```typescript',
-  'type Pair<T> = { left: T; right: T };',
-  'function swap<T>(p: Pair<T>): Pair<T> {',
-  '  return { left: p.right, right: p.left };',
-  '}',
+  'const id: number = 7; type Name = string;',
+  '```',
+  '',
+  '```tsx',
+  'const El = () => <View title="x" id={7} />;',
   '```',
   '',
   '```python',
-  'def fib(n: int) -> int:',
-  '    return n if n < 2 else fib(n - 1) + fib(n - 2)  # recursion',
+  'def greet(name): return f"hi {name}"  # f-string',
   '```',
   '',
   '```json',
-  '{ "id": 7, "tags": ["a", "b"], "active": true }',
+  '{ "id": 7, "tags": ["a"], "active": true }',
+  '```',
+  '',
+  '```yaml',
+  'name: demo',
+  'port: 8080  # comment',
+  '```',
+  '',
+  '```bash',
+  'echo "hi $USER"  # greet',
   '```',
   '',
   '```go',
-  'package main',
-  'func main() { println("hello") }',
+  'func main() { x := 3; _ = x }',
   '```',
   '',
-  '```rust',
-  'fn main() { let x: u32 = 3; println!("{x}"); }',
+  '```java',
+  'class A { int x = 3; }',
   '```',
   '',
   '```c',
-  '#include <stdio.h>',
   'int main(void) { return 0; }',
   '```',
   '',
-  '```java',
-  'record Point(int x, int y) {}',
+  '```rust',
+  'fn main() { let x: u32 = 3; }',
   '```',
   '',
-  '```bash',
-  'for f in *.ts; do echo "$f"; done',
+  '```html',
+  '<a href="/x" class="link">go</a>',
   '```',
   '',
   '```css',
-  '.title { color: #cf222e; font-weight: 600; }',
+  '.title { color: #cf222e; }',
   '```',
   '',
-  '```html',
-  '<a href="/x" class="link">go</a>',
+  '```markdown',
+  '# Heading with **bold** and _em_ [link](/x)',
   '```',
   '',
-  '```yaml',
-  'name: build',
-  'on: [push]',
+  '## Extended List',
+  '',
+  '```cpp',
+  'int main() { auto x = 3; return x; }',
+  '```',
+  '',
+  '```swift',
+  'let x: Int = 3; print(x)',
+  '```',
+  '',
+  '```php',
+  '<?php echo "hi $name"; ?>',
+  '```',
+  '',
+  '```ruby',
+  'def greet(name) = "hi #{name}"',
+  '```',
+  '',
+  '```csharp',
+  'class A { int X = 3; }',
   '```',
 ].join('\n');
 
+// Two blocks that together exercise every token type except embedded (which
+// needs language injection, unsupported by the single-grammar highlighter). The
+// Rust block covers comment, attribute (#[derive]), keyword, type, function,
+// property (fields), variable, number, string, constant, operator and
+// punctuation; the HTML block adds tag and attribute (plus doctype -> constant).
+const MARKDOWN_ALL_TOKENS = [
+  '```rust',
+  '// distance between two points',
+  '#[derive(Debug, Clone)]',
+  'struct Point {',
+  '    x: f64,',
+  '    y: f64,',
+  '}',
+  '',
+  'impl Point {',
+  '    fn dist(&self, other: &Point) -> f64 {',
+  '        let dx = self.x - other.x;',
+  '        (dx * dx).sqrt()',
+  '    }',
+  '}',
+  '',
+  'fn main() {',
+  '    const SCALE: u32 = 2;',
+  '    let p = Point { x: 1.5, y: 3.0 };',
+  '    println!("dist = {}", p.dist(&p) * SCALE as f64);',
+  '}',
+  '```',
+  '',
+  '```html',
+  '<!doctype html>',
+  '<!-- a labelled link -->',
+  '<a href="/x" class="link" data-id="7">go</a>',
+  '```',
+].join('\n');
+
+// GitHub-dark palette; the four "inherit" tokens use the code block base color.
 const BASE_TEXT_COLOR = '#f3f4f6';
 const syntaxColorDefaults = {
-  keyword: '#cf222e',
+  keyword: '#ff7b72',
   operatorColor: BASE_TEXT_COLOR,
   punctuation: BASE_TEXT_COLOR,
-  string: '#0a3069',
-  number: '#0550ae',
-  constant: '#0550ae',
-  comment: '#6e7781',
-  function: '#8250df',
-  type: '#953800',
+  string: '#a5d6ff',
+  number: '#79c0ff',
+  constant: '#79c0ff',
+  comment: '#8b949e',
+  function: '#d2a8ff',
+  type: '#ffa657',
   variable: BASE_TEXT_COLOR,
-  property: '#0550ae',
-  tag: '#116329',
-  attribute: '#0550ae',
+  property: '#79c0ff',
+  tag: '#7ee787',
+  attribute: '#79c0ff',
   embedded: BASE_TEXT_COLOR,
 };
 
@@ -122,7 +189,27 @@ export const Default: TextStory<SyntaxColorControls> = {
     return (
       <EnrichedMarkdownTextStory
         title="Code Highlight"
-        description="Per-token syntax colors via markdownStyle.codeBlock.syntaxColors. Colors render only when the optional highlighting module is compiled in; otherwise code blocks stay plain. Retheme any token with the color controls, and switch flavor to compare the block and inline renderers."
+        description="Per-token syntax colors via markdownStyle.codeBlock.syntaxColors. The Defaults section lists every language in the default registry (highlighted out of the box); the Extended List section lists non-default grammars that stay plain unless a build opts them in via the language-subset override. Colors render only when the optional highlighting module is compiled in; otherwise code blocks stay plain. Retheme any token with the color controls, and switch flavor to compare the block and inline renderers."
+        {...rest}
+        style={{ codeBlock: { syntaxColors: controls } }}
+      />
+    );
+  },
+};
+
+export const AllTokens: TextStory<SyntaxColorControls> = {
+  args: {
+    markdown: MARKDOWN_ALL_TOKENS,
+    flavor: 'github',
+    ...syntaxColorDefaults,
+  },
+  argTypes,
+  render: (args) => {
+    const { controls, rest } = splitStyleControls(args, syntaxColorDefaults);
+    return (
+      <EnrichedMarkdownTextStory
+        title="Code Highlight — All Tokens"
+        description="A Rust block plus an HTML block that together exercise every syntax token type except embedded (which needs language injection), each rethemable with its own color control below. Colors render only when the optional highlighting module is compiled in with the rust and html grammars; otherwise the code blocks stay plain."
         {...rest}
         style={{ codeBlock: { syntaxColors: controls } }}
       />

apps/react-native-example/android/gradle.properties

@@ -45,3 +45,5 @@ edgeToEdgeEnabled=false
 
 # Set to false to disable LaTeX math rendering and remove the RaTeX dependency.
 #enrichedMarkdown.enableMath=false
+
+enrichedMarkdown.codeHighlightLanguages=json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,php,ruby,c-sharp

apps/react-native-example/ios/Podfile

@@ -5,6 +5,10 @@ ENV['RCT_NEW_ARCH_ENABLED'] = '1'
 # ENV['ENRICHED_MARKDOWN_ENABLE_MATH'] = '0'
 ENV['USE_FRAMEWORKS'] = 'dynamic' if ENV['ENRICHED_MARKDOWN_ENABLE_MATH'] != '0'
 
+# Keep project default to full list of languages
+ENV['ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES'] = 'json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,php,ruby,c-sharp'
+# ENV['ENRICHED_MARKDOWN_ENABLE_CODE_HIGHLIGHT'] = '0'
+
 # Resolve react_native_pods.rb with node to allow for hoisting
 require Pod::Executable.execute_command('node', ['-p',
   'require.resolve(

apps/react-native-example/ios/Podfile.lock

@@ -2557,9 +2557,9 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native/ReactCommon/yoga"
 
 SPEC CHECKSUMS:
-  EnrichedMarkdownCore: 35aa4b07e4f23204609527c0e2440bbded65f23d
+  EnrichedMarkdownCore: 77755092110fabe3ea3a522b86b29ae98bf4d21a
   FBLazyVector: b3e7ad108f0d882e30445c5527d774e3fd432f3d
-  hermes-engine: 4a0ceceb51f483aef72bfeb288e965aff6defa54
+  hermes-engine: 1622a566d44d0dd4d377a1532e3ac82be7a0443c
   RCTDeprecation: 2a74a2c57675e64419bd89078efde81f7c1de90b
   RCTRequired: 30451112e6fef4e6f31b4e7eee0845156e35e4b0
   RCTSwiftUI: 5aaf0b07e747ba749dc6acc94d8bd41eea4b570f
@@ -2568,7 +2568,7 @@ SPEC CHECKSUMS:
   React: 2574546f2d017abd14d0c9b48cf2b6a0547c2591
   React-callinvoker: 03cd4b931d1d583d87aae99b8f7b6fe26bf571ee
   React-Core: 1c824d9c7dd8aa760b5f1b50d5a54c2a3f598f87
-  React-Core-prebuilt: 13924a267683b3d6fa4bde9c80380becf83a9c5c
+  React-Core-prebuilt: 7da85c26e5616d52f119b81ce5e3e6fc5c9bda49
   React-CoreModules: 2f9ed75bca7f6dea2b70e8a1f4a5ca9b6be52d76
   React-cxxreact: 7103d5ba69848c039e11079e74ceede05efe795e
   React-debug: e47fe49e67816470d183595746b1d0e20cd09fab
@@ -2644,6 +2644,6 @@ SPEC CHECKSUMS:
   RNWorklets: ae7f2b95d75b903387e6359583f2fd67bc9a93ff
   Yoga: c30c859b7e9b75e547b600b486fe33165f60de00
 
-PODFILE CHECKSUM: ef24009d29b47aa09e94351250ad7451d534a441
+PODFILE CHECKSUM: 1958dc707e4ee490c590b8544a91b81340ad9af8
 
 COCOAPODS: 1.16.2