feat(cli): support mermaid diagrams within GFM (github flavored markdown)

packages/cli/README.md

@@ -29,6 +29,10 @@ npx @mermaidchart/cli --help
 `@mermaidchart/cli` allows you to easily sync local diagrams with your diagrams
 on https://mermaidchart.com.
 
+These local diagrams can either be stored in `.mmd` or `.mermaid` files, or
+they can be stored within ```` ```mermaid```` code blocks within `.md`
+[GFM](https://github.github.com/gfm/) markdown files.
+
 ### `login`
 
 Firstly, go to https://www.mermaidchart.com/app/user/settings and generate an

packages/cli/package.json

@@ -42,12 +42,14 @@
     "@tsconfig/strictest": "^2.0.2",
     "@types/iarna__toml": "^2.0.5",
     "@types/js-yaml": "^4.0.9",
+    "@types/mdast": "^4.0.3",
     "@types/node": "^18.18.11",
     "@typescript-eslint/eslint-plugin": "^6.11.0",
     "@typescript-eslint/parser": "^6.11.0",
     "eslint": "^8.54.0",
     "tsx": "^3.12.8",
     "typescript": "^5.2.2",
+    "vfile": "^6.0.1",
     "vitest": "^0.34.6"
   },
   "dependencies": {
@@ -58,6 +60,11 @@
     "@inquirer/select": "^1.3.1",
     "@mermaidchart/sdk": "workspace:^",
     "commander": "^11.1.0",
-    "js-yaml": "^4.1.0"
+    "js-yaml": "^4.1.0",
+    "remark": "^15.0.1",
+    "remark-frontmatter": "^5.0.0",
+    "remark-gfm": "^4.0.0",
+    "to-vfile": "^8.0.0",
+    "unist-util-visit": "^5.0.0"
   }
 }

packages/cli/src/commander.test.ts

@@ -11,6 +11,14 @@ import type { MCDocument, MCProject, MCUser } from '@mermaidchart/sdk/dist/types
 
 /** Config file with auth_key setup */
 const CONFIG_AUTHED = 'test/fixtures/config-authed.toml';
+/** Markdown file that has every Mermaid diagrams already linked */
+const LINKED_MARKDOWN_FILE = 'test/fixtures/linked-markdown-file.md';
+/** Markdown file that has some linked and some unlinked Mermaid diagrams */
+const PARTIALLY_LINKED_MARKDOWN_FILE = 'test/fixtures/partially-linked-markdown-file.md';
+/** Markdown file that has unlinked Mermaid diagrams */
+const UNLINKED_MARKDOWN_FILE = 'test/fixtures/unlinked-markdown-file.md';
+/** Markdown file that has non-standard Markdown features like YAML frontmatter */
+const UNUSUAL_MARKDOWN_FILE = 'test/fixtures/unusual-markdown-file.md';
 
 type Optional<T> = T | undefined;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -269,16 +277,97 @@ describe('link', () => {
       );
     });
   }
+
+  it('should link diagrams in a markdown file', async () => {
+    const unlinkedMarkdownFile = 'test/output/markdown-file.md';
+    await copyFile(UNLINKED_MARKDOWN_FILE, unlinkedMarkdownFile);
+
+    const { program } = mockedProgram();
+
+    vi.mock('@inquirer/confirm');
+    vi.mock('@inquirer/select');
+    vi.mocked(confirm).mockResolvedValue(true);
+    vi.mocked(select).mockResolvedValueOnce(mockedProjects[0].id);
+
+    vi.mocked(MermaidChart.prototype.createDocument)
+      .mockResolvedValueOnce(mockedEmptyDiagram)
+      .mockResolvedValueOnce({ ...mockedEmptyDiagram, documentID: 'second-id' });
+    await program.parseAsync(['--config', CONFIG_AUTHED, 'link', unlinkedMarkdownFile], {
+      from: 'user',
+    });
+
+    const file = await readFile(unlinkedMarkdownFile, { encoding: 'utf8' });
+
+    expect(file).toMatch(`id: ${mockedEmptyDiagram.documentID}\n`);
+    expect(file).toMatch(`id: second-id\n`);
+  });
+
+  it('should link diagrams in partially linked markdown file', async () => {
+    const partiallyLinkedMarkdownFile = 'test/output/partially-linked-markdown-file.md';
+    await copyFile(PARTIALLY_LINKED_MARKDOWN_FILE, partiallyLinkedMarkdownFile);
+
+    const { program } = mockedProgram();
+
+    vi.mock('@inquirer/confirm');
+    vi.mock('@inquirer/select');
+    vi.mocked(confirm).mockResolvedValue(true);
+    vi.mocked(select).mockResolvedValueOnce(mockedProjects[0].id);
+
+    vi.mocked(MermaidChart.prototype.createDocument).mockResolvedValueOnce({
+      ...mockedEmptyDiagram,
+      documentID: 'second-id',
+    });
+    await program.parseAsync(['--config', CONFIG_AUTHED, 'link', partiallyLinkedMarkdownFile], {
+      from: 'user',
+    });
+
+    const file = await readFile(partiallyLinkedMarkdownFile, { encoding: 'utf8' });
+
+    expect(file).toMatch(`id: second-id\n`);
+  });
+
+  it('should handle unusual markdown formatting', async () => {
+    const unusualMarkdownFile = 'test/output/unusual-markdown-file.md';
+    await copyFile(UNUSUAL_MARKDOWN_FILE, unusualMarkdownFile);
+
+    const { program } = mockedProgram();
+
+    vi.mock('@inquirer/confirm');
+    vi.mock('@inquirer/select');
+    vi.mocked(confirm).mockResolvedValue(true);
+    vi.mocked(select).mockResolvedValueOnce(mockedProjects[0].id);
+
+    vi.mocked(MermaidChart.prototype.createDocument).mockResolvedValueOnce({
+      ...mockedEmptyDiagram,
+      documentID: 'my-mocked-diagram-id',
+    });
+    await program.parseAsync(['--config', CONFIG_AUTHED, 'link', unusualMarkdownFile], {
+      from: 'user',
+    });
+
+    const file = await readFile(unusualMarkdownFile, { encoding: 'utf8' });
+
+    const idLineRegex = /^.*id: my-mocked-diagram-id\n/gm;
+
+    expect(file).toMatch(idLineRegex);
+    // other than the added `id: xxxx` field, everything else should be identical,
+    // although in practice, we'd expect some formatting changes
+    expect(file.replace(idLineRegex, '')).toStrictEqual(
+      await readFile(UNUSUAL_MARKDOWN_FILE, { encoding: 'utf8' }),
+    );
+  });
 });
 
 describe('pull', () => {
   const diagram = 'test/output/connected-diagram.mmd';
   const diagram2 = 'test/output/connected-diagram-2.mmd';
+  const linkedMarkdownFile = 'test/output/linked-markdown-file.md';
 
   beforeEach(async () => {
     await Promise.all([
       copyFile('test/fixtures/connected-diagram.mmd', diagram),
       copyFile('test/fixtures/connected-diagram.mmd', diagram2),
+      copyFile(LINKED_MARKDOWN_FILE, linkedMarkdownFile),
     ]);
   });
 
@@ -327,16 +416,41 @@ title: My cool flowchart
       expect(diagramContents).toContain("flowchart TD\n      A[I've been updated!]");
     }
   });
+
+  it('should pull documents from within markdown file', async () => {
+    const { program } = mockedProgram();
+
+    vi.mocked(MermaidChart.prototype.getDocument)
+      .mockResolvedValueOnce({
+        ...mockedEmptyDiagram,
+        code: "flowchart TD\n      A[I've been updated!]",
+      })
+      .mockResolvedValueOnce({
+        ...mockedEmptyDiagram,
+        code: 'pie\n  "Flowchart" : 2',
+      });
+
+    await program.parseAsync(['--config', CONFIG_AUTHED, 'pull', linkedMarkdownFile], {
+      from: 'user',
+    });
+
+    const file = await readFile(linkedMarkdownFile, { encoding: 'utf8' });
+
+    expect(file).toMatch("flowchart TD\n      A[I've been updated!]");
+    expect(file).toMatch('pie\n  "Flowchart" : 2');
+  });
 });
 
 describe('push', () => {
   const diagram = 'test/output/connected-diagram.mmd';
   const diagram2 = 'test/output/connected-diagram-2.mmd';
+  const linkedMarkdownFile = 'test/output/linked-markdown-file.md';
 
   beforeEach(async () => {
     await Promise.all([
       copyFile('test/fixtures/connected-diagram.mmd', diagram),
       copyFile('test/fixtures/connected-diagram.mmd', diagram2),
+      copyFile(LINKED_MARKDOWN_FILE, linkedMarkdownFile),
     ]);
   });
 
@@ -368,4 +482,16 @@ describe('push', () => {
       }),
     );
   });
+
+  it('should push documents from within markdown file', async () => {
+    const { program } = mockedProgram();
+
+    vi.mocked(MermaidChart.prototype.getDocument).mockResolvedValue(mockedEmptyDiagram);
+
+    await program.parseAsync(['--config', CONFIG_AUTHED, 'push', linkedMarkdownFile], {
+      from: 'user',
+    });
+
+    expect(vi.mocked(MermaidChart.prototype.setDocument)).toHaveBeenCalledTimes(2);
+  });
 });

packages/cli/src/commander.ts

@@ -11,7 +11,8 @@
 import input from '@inquirer/input';
 import select, { Separator } from '@inquirer/select';
 import { type Config, defaultConfigPath, readConfig, writeConfig } from './config.js';
-import { link, type LinkOptions, pull, push } from './methods.js';
+import { type Cache, link, type LinkOptions, pull, push } from './methods.js';
+import { processMarkdown } from './remark.js';
 
 /**
  * Global configuration option for the root Commander Command.
@@ -80,7 +81,7 @@
 
       const user = await client.getUser();
 
       console.log(user.emailAddress);
     });
 }
 
@@ -130,7 +131,7 @@
 
       await writeConfig(optsWithGlobals['config'], { ...config, auth_token: answer });
 
       console.log(`API token for ${user.emailAddress} saved to ${optsWithGlobals['config']}`);
     });
 }
 
@@ -142,21 +143,25 @@
       const { auth_token, ...config } = await readConfig(optsWithGlobals['config']);
 
       if (auth_token === undefined) {
         console.log(`Nothing to do, there's no auth_token in ${optsWithGlobals['config']}`);
       }
 
       await writeConfig(optsWithGlobals['config'], config);
 
       try {
         const user = await (await createClient(optsWithGlobals, config)).getUser();
         console.log(`API token for ${user.emailAddress} removed from ${optsWithGlobals['config']}`);
       } catch (error) {
         // API token might have been expired
         console.log(`API token removed from ${optsWithGlobals['config']}`);
       }
     });
 }
 
+function isMarkdownFile(path: string): path is `${string}.${'md' | 'markdown'}` {
+  return /\.(md|markdown)$/.test(path);
+}
+
 function linkCmd() {
   return createCommand('link')
     .description('Link the given Mermaid diagrams to Mermaid Chart')
@@ -200,15 +205,20 @@
         return projectId;
       };
 
-      const linkCache = {};
+      const linkCache: Cache = {};
 
       for (const path of paths) {
+        if (isMarkdownFile(path)) {
+          await processMarkdown(path, { command: 'link', client, cache: linkCache, getProjectId });
+          continue;
+        }
         const existingFile = await readFile(path, { encoding: 'utf8' });
 
         const linkedDiagram = await link(existingFile, client, {
           cache: linkCache,
           title: path,
           getProjectId,
+          ignoreAlreadyLinked: false,
         });
 
         await writeFile(path, linkedDiagram, { encoding: 'utf8' });
@@ -220,25 +230,31 @@
   return createCommand('pull')
     .description('Pulls documents from Mermaid Chart')
     .addArgument(new Argument('<path...>', 'The paths of the files to pull.'))
-    .option('--check', 'Check whether the local files would be overwrited')
+    .option('--check', 'Check whether the local files would be overwrited', false)
     .action(async (paths, options, command) => {
       const optsWithGlobals = command.optsWithGlobals<CommonOptions>();
       const client = await createClient(optsWithGlobals);
+
       await Promise.all(
         paths.map(async (path) => {
+          if (isMarkdownFile(path)) {
+            await processMarkdown(path, { command: 'pull', client, check: options['check'] });
+            return;
+          }
+
           const text = await readFile(path, { encoding: 'utf8' });
 
           const newFile = await pull(text, client, { title: path });
 
           if (text === newFile) {
             console.log(`✅ - ${path} is up to date`);
           } else {
             if (options['check']) {
               console.log(`❌ - ${path} would be updated`);
               process.exitCode = 1;
             } else {
               await writeFile(path, newFile, { encoding: 'utf8' });
               console.log(`✅ - ${path} was updated`);
             }
           }
         }),
@@ -255,6 +271,11 @@
       const client = await createClient(optsWithGlobals);
       await Promise.all(
         paths.map(async (path) => {
+          if (isMarkdownFile(path)) {
+            await processMarkdown(path, { command: 'push', client });
+            return;
+          }
+
           const text = await readFile(path, { encoding: 'utf8' });
 
           await push(text, client, { title: path });

packages/cli/src/methods.ts

@@ -39,28 +39,37 @@
 export interface LinkOptions extends CommonOptions {
   /** Function that asks the user which project id they want to upload a diagram to */
   getProjectId: (cache: LinkOptions['cache'], documentTitle: string) => Promise<string>;
-  // cache to be shared between link calls. This object may be modified between calls.
+  /** cache to be shared between link calls. This object may be modified between calls. */
   cache: Cache;
+  /** If `true`, ignore diagrams that are already linked. */
+  ignoreAlreadyLinked: boolean;
 }
 
 /**
  * Creates a new diagram on MermaidChart.com for the given local diagram.
  *
  * @returns The diagram with an added `id: xxxx` field.
  */
-export async function link(diagram: string, client: MermaidChart, options: LinkOptions) {
+export async function link(
+  diagram: string,
+  client: MermaidChart,
+  { title, getProjectId, cache, ignoreAlreadyLinked }: LinkOptions,
+) {
   const frontmatter = extractFrontMatter(diagram);
 
   if (frontmatter.metadata.id) {
-    throw new CommanderError(
-      /*exitCode=*/ 1,
-      'EALREADY_LINKED',
-      'This document already has an `id` field',
-    );
+    if (ignoreAlreadyLinked) {
+      console.log(`○ - ${title} is already linked`);
+      return diagram; // no change required
+    } else {
+      throw new CommanderError(
+        /*exitCode=*/ 1,
+        'EALREADY_LINKED',
+        'This document already has an `id` field',
+      );
+    }
   }
 
-  const { title, getProjectId, cache } = options;
-
   const projectId = await getProjectId(cache, title);
 
   const createdDocument = await client.createDocument(projectId);
@@ -127,7 +136,7 @@
   const diagramToUpload = removeFrontMatterKeys(diagram, new Set(['id']));
 
   if (existingDiagram.code === diagramToUpload) {
     console.log(`✅ - ${title} is up to date`);
   } else {
     await client.setDocument({
       projectID: existingDiagram.projectID,