fix: notions export pods

packages/common-all/src/helpers.ts

@@ -29,3 +29,47 @@ export async function asyncLoopOneAtATime<T, R = any>(
 export async function asyncLoop<T>(things: T[], cb: (t: T) => Promise<any>) {
   return Promise.all(things.map((t) => cb(t)));
 }
+
+
+/**
+ * Retry resolving promise factory n number of times before rejecting it
+ * @param retriable
+ * @param handler
+ * @param n
+ * @returns
+ */
+export async function asyncRetry<T>(retriable: () => Promise<T>, handler: (arg0: any) => any, n?: number) {
+  n = n ?? 3;
+
+  for (let i = 0; i < n; i += 1) {
+    try {
+      /* eslint-disable no-await-in-loop */
+      return await retriable();
+    } catch (error) {
+      if (i < (n - 1)) {
+        continue;
+      }
+      return handler(error);
+    }
+  }
+}
+
+/**
+ * Deferrable promise
+ */
+export class Deferred<T> {
+  promise: Promise<T>;
+  reject!: (reason?: any) => void;
+  resolve!: (value: T) => void;
+
+  constructor(value?: T | null) {
+    this.promise = new Promise((resolve, reject) => {
+      this.reject = reject
+      this.resolve = resolve
+    })
+
+    if(value) {
+      this.resolve(value);
+    }
+  }
+}

packages/engine-test-utils/src/__tests__/pods-core/NotionPod.spec.ts

@@ -19,7 +19,7 @@ describe("GIVEN a Notion export pod", () => {
           const vaultName = VaultUtils.getName(vaults[0]);
           pod.createPagesInNotion = jest.fn();
           pod.getAllNotionPages = jest.fn();
-          pod.convertMdToNotionBlock = jest.fn();
+          NotionExportPod.convertMdToNotionBlock = jest.fn();
           await pod.execute({
             engine,
             vaults,
@@ -32,7 +32,7 @@ describe("GIVEN a Notion export pod", () => {
             utilityMethods,
           });
           expect(pod.createPagesInNotion).toHaveBeenCalledTimes(1);
-          expect(pod.convertMdToNotionBlock).toHaveBeenCalledTimes(1);
+          expect(NotionExportPod.convertMdToNotionBlock).toHaveBeenCalledTimes(1);
         },
         {
           expect,

packages/engine-test-utils/src/__tests__/pods-core/podsv2.spec.ts

@@ -19,6 +19,7 @@ import {
   RunnableGoogleDocsV2PodConfig,
   RunnableNotionV2PodConfig,
   RunnableJSONV2PodConfig,
+  NotionExportPod,
 } from "@dendronhq/pods-core";
 import fs from "fs-extra";
 import _ from "lodash";
@@ -471,7 +472,7 @@ describe("GIVEN a Notion Export Pod with a particular config", () => {
             ],
             errors: [],
           };
-          pod.convertMdToNotionBlock = jest.fn();
+          NotionExportPod.convertMdToNotionBlock = jest.fn();
           pod.createPagesInNotion = jest.fn().mockResolvedValue(response);
           const result = await pod.exportNotes([props]);
           const entCreate = result.data?.created!;

packages/pods-core/package.json

@@ -56,8 +56,8 @@
     "@dendronhq/common-server": "^0.124.0",
     "@dendronhq/engine-server": "^0.124.0",
     "@dendronhq/unified": "^0.124.0",
-    "@instantish/martian": "1.0.3",
-    "@notionhq/client": "^0.1.9",
+    "@tryfabric/martian": "1.2.4",
+    "@notionhq/client": "^1.0.4",
     "@octokit/graphql": "^4.6.4",
     "@types/airtable": "^0.10.1",
     "airtable": "^0.11.1",

packages/pods-core/src/builtin/NotionPod.ts

@@ -1,10 +1,12 @@
 import {
   asyncLoop,
+  asyncRetry,
+  Deferred,
   DendronError,
   ERROR_SEVERITY,
   NoteProps,
 } from "@dendronhq/common-all";
-import { markdownToBlocks } from "@instantish/martian";
+import { markdownToBlocks } from "@tryfabric/martian";
 import type {
   Page,
   TitlePropertyValue,
@@ -53,28 +55,48 @@ export class NotionExportPod extends ExportPod<NotionExportConfig> {
    * Method to create pages in Notion
    */
   createPagesInNotion = (
-    blockPagesArray: any,
-    notion: Client
+    notes: NoteProps[],
+    notion: Client,
+    parentPageId: string,
   ): Promise<any[]> => {
-    return asyncLoop(blockPagesArray, async (block: any) => {
-      await limiter.removeTokens(1);
-      try {
-        await notion.pages.create(block);
-      } catch (error) {
-        throw new DendronError({
-          message: "Failed to export all the notes. " + JSON.stringify(error),
-          severity: ERROR_SEVERITY.MINOR,
+    const keyMap = Object.fromEntries(
+      notes.map(note => [
+        note.parent ?? "undefined", 
+        note.parent ? new Deferred<string>() : new Deferred<string>(parentPageId)
+      ])
+    );
+
+    return asyncLoop(notes, async (note: NoteProps) => {
+      const parentId = await keyMap[note.parent ?? "undefined"].promise;
+      const blockPage = NotionExportPod.convertMdToNotionBlock(note, parentId);
+
+      await asyncRetry(
+        async () => {
+          await limiter.removeTokens(1);
+
+          const page = await notion.pages.create(blockPage);
+
+          keyMap[note.id]?.resolve(page.id);
+
+          return {
+            notionId: page.id,
+            dendronId: note.id,
+          }
+        },
+        (error) => {
+          keyMap[note.id]?.reject(`Failed to export the note title:'${note.title}' - id:${note.id}.`);
+          throw new DendronError({
+            message: `Failed to export the note title:'${note.title}' - id:${note.id} - content: ${JSON.stringify(blockPage)}` + JSON.stringify(error),
+            severity: ERROR_SEVERITY.MINOR,
+          });
         });
-      }
     });
   };
 
   /**
    * Method to convert markdown to Notion Block
    */
-  convertMdToNotionBlock = (notes: NoteProps[], pageId: string) => {
-    const notionBlock = notes.map((note) => {
-      const children = markdownToBlocks(note.body);
+  public static convertMdToNotionBlock = (note: NoteProps, pageId: string): any => {
       return {
         parent: {
           page_id: pageId,
@@ -84,10 +106,8 @@ export class NotionExportPod extends ExportPod<NotionExportConfig> {
             title: [{ type: "text", text: { content: note.title } }],
           },
         },
-        children,
+        children: markdownToBlocks(note.body),
       };
-    });
-    return notionBlock;
   };
 
   /**
@@ -159,8 +179,7 @@ export class NotionExportPod extends ExportPod<NotionExportConfig> {
       return { notes: [] };
     }
     const pageId = pagesMap[selectedPage];
-    const blockPagesArray = this.convertMdToNotionBlock(notes, pageId);
-    await this.createPagesInNotion(blockPagesArray, notion);
+    await this.createPagesInNotion(notes, notion, pageId);
     return { notes };
   }
 }

packages/pods-core/src/v2/pods/export/NotionExportPodV2.ts

@@ -1,4 +1,7 @@
 import {
+  asyncLoop,
+  asyncRetry,
+  Deferred,
   DendronCompositeError,
   DendronError,
   DEngineClient,
@@ -7,14 +10,14 @@ import {
   ResponseUtil,
   RespV2,
 } from "@dendronhq/common-all";
-import { markdownToBlocks } from "@instantish/martian";
 import { JSONSchemaType } from "ajv";
 import { RateLimiter } from "limiter";
 import _ from "lodash";
 import {
   Client,
   ConfigFileUtils,
   ExportPodV2,
+  NotionExportPod,
   NotionV2PodConfig,
   RunnableNotionV2PodConfig,
 } from "../../..";
@@ -50,8 +53,7 @@ export class NotionExportPodV2 implements ExportPodV2<NotionExportReturnType> {
 
   async exportNotes(notes: NoteProps[]): Promise<NotionExportReturnType> {
     const { parentPageId } = this._config;
-    const blockPagesArray = this.convertMdToNotionBlock(notes, parentPageId);
-    const { data, errors } = await this.createPagesInNotion(blockPagesArray);
+    const { data, errors } = await this.createPagesInNotion(notes, parentPageId);
     const createdNotes = data.filter(
       (ent): ent is NotionFields => !_.isUndefined(ent)
     );
@@ -70,59 +72,48 @@ export class NotionExportPodV2 implements ExportPodV2<NotionExportReturnType> {
     }
   }
 
-  /**
-   * Method to convert markdown to Notion Block
-   */
-  convertMdToNotionBlock = (notes: NoteProps[], parentPageId: string) => {
-    const notionBlock = notes.map((note) => {
-      const children = markdownToBlocks(note.body);
-      return {
-        dendronId: note.id,
-        block: {
-          parent: {
-            page_id: parentPageId,
-          },
-          properties: {
-            title: {
-              title: [{ type: "text", text: { content: note.title } }],
-            },
-          },
-          children,
-        },
-      };
-    });
-    return notionBlock;
-  };
-
   /**
    * Method to create pages in Notion
    */
   createPagesInNotion = async (
-    blockPagesArray: any
+    notes: NoteProps[],
+    parentPageId: string,
   ): Promise<{
     data: NotionFields[];
     errors: IDendronError[];
   }> => {
     const notion = new Client({
       auth: this._config.apiKey,
     });
+
+    const keyMap = Object.fromEntries(
+      notes.map(note => [
+        note.parent ?? "undefined",
+        note.parent ? new Deferred<string>() : new Deferred<string>(parentPageId)
+      ])
+    );
+
     const errors: IDendronError[] = [];
-    const out: NotionFields[] = await Promise.all(
-      blockPagesArray.map(async (ent: any) => {
-        // @ts-ignore
-        await limiter.removeTokens(1);
-        try {
-          const response = await notion.pages.create(ent.block);
+    const out: NotionFields[] = await asyncLoop(notes, async (note: NoteProps) => {
+      const parentId = await keyMap[note.parent ?? "undefined"].promise;
+      const blockPage = NotionExportPod.convertMdToNotionBlock(note, parentId);
+
+      return asyncRetry(
+        async () => {
+          await limiter.removeTokens(1);
+          const page = await notion.pages.create(blockPage);
+          keyMap[note.id]?.resolve(page.id);
           return {
-            notionId: response.id,
-            dendronId: ent.dendronId,
-          };
-        } catch (error) {
+            notionId: page.id,
+            dendronId: note.id,
+          }
+        },
+        (error) => {
+          keyMap[note.id]?.reject(`Failed to export the note title:'${note.title}' - id:${note.id}.`);
           errors.push(error as DendronError);
-          return;
-        }
-      })
-    );
+        });
+    });
+
     return {
       data: out,
       errors,