Merge pull request #5 from vizhub-core/streaming-parser

Streaming parser

Author: curran

.prettierignore

@@ -0,0 +1,2 @@
+.gitignore
+.prettierignore

.prettierrc

@@ -0,0 +1 @@
+{}

package-lock.json

@@ -19,7 +19,9 @@
     "build:esm": "tsc -p tsconfig.json",
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "test": "vitest run",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build",
+    "typecheck": "tsc --noEmit",
+    "prettier": "prettier {*.*,**/*.*} --write"
   },
   "repository": {
     "type": "git",
@@ -44,6 +46,7 @@
   ],
   "devDependencies": {
     "typescript": "^5.8.2",
-    "vitest": "^3.0.9"
+    "vitest": "^3.1.1",
+    "prettier": "^3.5.3"
   }
 }

package.json

@@ -1,2 +1,6 @@
-export { parseMarkdownFiles } from './parseMarkdownFiles.js';
-export { serializeMarkdownFiles } from './serializeMarkdownFiles.js';
+export { parseMarkdownFiles } from "./parseMarkdownFiles.js";
+export { serializeMarkdownFiles } from "./serializeMarkdownFiles.js";
+export {
+  StreamingMarkdownParser,
+  StreamingParserCallbacks,
+} from "./streamingParser.js";

src/index.ts

@@ -365,7 +365,10 @@ And then we do some work on it and...
   const { files, format } = parseMarkdownFiles(markdownString);
   expect(format).toBe("Bold Format");
   expect(files).toEqual([
-    { name: "index.html", text: "<!-- New HTML content -->" },
+    {
+      name: "index.html",
+      text: "<!-- New HTML content -->",
+    },
   ]);
 });
 
@@ -537,7 +540,7 @@ test("parseMarkdownFiles throws error when an unsupported format is specified",
 \`\`\`
   `;
   expect(() =>
-    parseMarkdownFiles(markdownString, "unsupported-format")
+    parseMarkdownFiles(markdownString, "unsupported-format"),
   ).toThrow("Unsupported format: unsupported-format");
 });
 
@@ -559,7 +562,10 @@ test("parseMarkdownFiles handles duplicate file names in Bold Format when format
   const { files, format } = parseMarkdownFiles(markdownString, "bold");
   expect(format).toBe("Bold Format");
   expect(files).toEqual([
-    { name: "index.html", text: "<!-- New HTML content -->" },
+    {
+      name: "index.html",
+      text: "<!-- New HTML content -->",
+    },
   ]);
 });
 

src/parseMarkdownFiles.test.ts

@@ -47,9 +47,21 @@ export function parseMarkdownFiles(markdownString: string, format?: string) {
       format: "Standard Heading Format",
       key: "standard-heading",
     },
-    { regex: colonFormatRegex, format: "Colon Format", key: "colon" },
-    { regex: boldFormatRegex, format: "Bold Format", key: "bold" },
-    { regex: hashFormatRegex, format: "Hash Format", key: "hash" },
+    {
+      regex: colonFormatRegex,
+      format: "Colon Format",
+      key: "colon",
+    },
+    {
+      regex: boldFormatRegex,
+      format: "Bold Format",
+      key: "bold",
+    },
+    {
+      regex: hashFormatRegex,
+      format: "Hash Format",
+      key: "hash",
+    },
     {
       regex: numberedBoldFormatRegex,
       format: "Numbered Bold Format",

src/parseMarkdownFiles.ts

@@ -6,11 +6,11 @@ const language = (name: string) => {
 };
 
 export const serializeMarkdownFiles = (
-  files: Array<{ name: string; text: string }>
+  files: Array<{ name: string; text: string }>,
 ) =>
   files
     .map(({ name, text }) =>
-      [`**${name}**\n`, "```" + language(name), text, "```\n"].join("\n")
+      [`**${name}**\n`, "```" + language(name), text, "```\n"].join("\n"),
     )
     .join("\n")
     .trim();

src/serializeMarkdownFiles.ts

@@ -0,0 +1,133 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import {
+  StreamingMarkdownParser,
+  StreamingParserCallbacks,
+} from "./streamingParser";
+
+describe("StreamingMarkdownParser", () => {
+  let fileNameChanges: Array<{
+    name: string;
+    format: string;
+  }>;
+  let codeLines: string[];
+  let parser: StreamingMarkdownParser;
+
+  const callbacks: StreamingParserCallbacks = {
+    onFileNameChange: (fileName, format) => {
+      fileNameChanges.push({ name: fileName, format });
+    },
+    onCodeLine: (line) => {
+      codeLines.push(line);
+    },
+  };
+
+  beforeEach(() => {
+    fileNameChanges = [];
+    codeLines = [];
+    parser = new StreamingMarkdownParser(callbacks);
+  });
+
+  it("should process a complete markdown block in one chunk", () => {
+    const input = "**index.html**\n```\n<html>\n</html>\n```\n";
+    parser.processChunk(input);
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "index.html", format: "Bold Format" },
+    ]);
+    expect(codeLines).toEqual(["<html>", "</html>"]);
+  });
+
+  it("should handle multiple chunks with split lines", () => {
+    // Simulate splitting a header and code block across chunks
+    parser.processChunk("**inde");
+    parser.processChunk("x.html**\n```\n<ht");
+    parser.processChunk("ml>\n</html>\n```");
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "index.html", format: "Bold Format" },
+    ]);
+    expect(codeLines).toEqual(["<html>", "</html>"]);
+  });
+
+  it("should process multiple files and code blocks", () => {
+    const input =
+      "**index.html**\n```\n<html>\n</html>\n```\n" +
+      "**styles.css**\n```\nbody { color: blue; }\n```\n";
+    parser.processChunk(input);
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "index.html", format: "Bold Format" },
+      { name: "styles.css", format: "Bold Format" },
+    ]);
+    expect(codeLines).toEqual(["<html>", "</html>", "body { color: blue; }"]);
+  });
+
+  it("should handle code fence markers with language specifiers", () => {
+    const input = "**script.js**\n```js\nconsole.log('Hello');\n```\n";
+    parser.processChunk(input);
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "script.js", format: "Bold Format" },
+    ]);
+    expect(codeLines).toEqual(["console.log('Hello');"]);
+  });
+
+  it("should flush remaining partial lines on flushRemaining", () => {
+    // Provide a header line without a trailing newline
+    parser.processChunk("**partial.html**");
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "partial.html", format: "Bold Format" },
+    ]);
+  });
+
+  it("should not trigger callbacks for irrelevant lines outside code fences", () => {
+    // Provide a line that doesn't match any header or code fence
+    parser.processChunk("This is an irrelevant line\n");
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([]);
+    expect(codeLines).toEqual([]);
+  });
+
+  it("should handle nested chunks with header and code block boundaries", () => {
+    // Simulate a stream with multiple boundaries and chunk splits
+    const chunks = [
+      "**index.html**\n", // Header detected
+      "```\n<ht", // Start code fence and partial code
+      "ml>\n</ht", // Continuation of code
+      "ml>\n```\n", // End code fence
+      "Some irrelevant line\n", // Irrelevant line outside code fence
+      "**styles.css**\n", // New header
+      "```\nbody { color:", // Start second code fence with split code line
+      " blue; }\n```\n", // End code fence
+    ];
+
+    chunks.forEach((chunk) => parser.processChunk(chunk));
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "index.html", format: "Bold Format" },
+      { name: "styles.css", format: "Bold Format" },
+    ]);
+
+    expect(codeLines).toEqual(["<html>", "</html>", "body { color: blue; }"]);
+  });
+
+  it("should handle bold format with extra text", () => {
+    const input =
+      "**index.html** (some commentary)\n```\n<html>\n</html>\n```\n";
+    parser.processChunk(input);
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "index.html", format: "Bold Format" },
+    ]);
+    expect(codeLines).toEqual(["<html>", "</html>"]);
+  });
+});

src/streamingParser.test.ts

@@ -0,0 +1,102 @@
+export type StreamingParserCallbacks = {
+  /**
+   * Called when a file header is detected outside a code fence.
+   * @param fileName - The detected file name.
+   * @param format - The header format that was matched.
+   */
+  onFileNameChange: (fileName: string, format: string) => void;
+
+  /**
+   * Called for each line emitted from inside a code fence.
+   * @param line - A line of code from the code block.
+   */
+  onCodeLine: (line: string) => void;
+};
+
+export class StreamingMarkdownParser {
+  private buffer: string = "";
+  private insideCodeFence: boolean = false;
+  private currentFileName: string | null = null;
+  private detectedFormat: string = "Unknown Format";
+  private callbacks: StreamingParserCallbacks;
+
+  /**
+   * An array of regex patterns for detecting file headers.
+   * Currently only supports Bold Format, but can be extended in the future.
+   */
+  private headerPatterns: {
+    regex: RegExp;
+    format: string;
+  }[] = [
+    // Matches: **filename.js**
+    {
+      regex: /^\s*\*\*([^\n*`]+?)\*\*(?:[^\n]*)\s*$/,
+      format: "Bold Format",
+    },
+  ];
+
+  constructor(callbacks: StreamingParserCallbacks) {
+    this.callbacks = callbacks;
+  }
+
+  /**
+   * Processes an incoming chunk from the stream.
+   * Chunks are buffered until full lines (ending with '\n') are available.
+   * @param chunk - A string chunk from the stream.
+   */
+  processChunk(chunk: string) {
+    this.buffer += chunk;
+    let newlineIndex: number;
+
+    while ((newlineIndex = this.buffer.indexOf("\n")) !== -1) {
+      const line = this.buffer.slice(0, newlineIndex);
+      this.buffer = this.buffer.slice(newlineIndex + 1);
+      this.processLine(line);
+    }
+  }
+
+  /**
+   * Flushes any remaining content in the buffer.
+   * Should be called once after the stream has ended.
+   */
+  flushRemaining() {
+    if (this.buffer.length > 0) {
+      this.processLine(this.buffer);
+      this.buffer = "";
+    }
+  }
+
+  /**
+   * Processes a single line.
+   * If the line is a code fence marker (starting with "```"), it toggles the code block state.
+   * When inside a code block, every line is emitted via onCodeLine.
+   * Outside of a code block, the line is checked against header patterns,
+   * and if a match is found, onFileNameChange is invoked.
+   * @param line - A single line of text.
+   */
+  private processLine(line: string) {
+    // Check if the line is a code fence marker (could be "```" or "```lang")
+    if (line.trim().startsWith("```")) {
+      this.insideCodeFence = !this.insideCodeFence;
+      return; // The fence marker itself is not emitted as code content.
+    }
+
+    if (this.insideCodeFence) {
+      // Emit every line inside the code fence as a code line.
+      this.callbacks.onCodeLine(line);
+    } else {
+      // Outside a code fence, check for file header patterns.
+      for (const { regex, format } of this.headerPatterns) {
+        const match = regex.exec(line);
+        if (match) {
+          const fileName = match[1].trim();
+          this.currentFileName = fileName;
+          this.detectedFormat = format;
+          this.callbacks.onFileNameChange(fileName, format);
+          break; // Stop after the first matching header is found.
+        }
+      }
+      // Non-header lines outside code fences are ignored
+    }
+  }
+}