From 562ea6276805767f84bbefe1d94da89f181674c8 Mon Sep 17 00:00:00 2001
From: Michael Wedl <michael@syslifters.com>
Date: Thu, 26 Sep 2024 08:08:24 +0200
Subject: [PATCH 1/3] Refactor collab connections and throttling

---
 .../reportcreator_api/pentests/consumers.py   |   2 +-
 frontend/src/utils/collab.ts                  | 367 +++++++++---------
 2 files changed, 177 insertions(+), 192 deletions(-)

diff --git a/api/src/reportcreator_api/pentests/consumers.py b/api/src/reportcreator_api/pentests/consumers.py
index 7b89cd595..c396b58d7 100644
--- a/api/src/reportcreator_api/pentests/consumers.py
+++ b/api/src/reportcreator_api/pentests/consumers.py
@@ -235,7 +235,7 @@ class UserNotesConsumer(NotesConsumerBase):
     async def get_related_id(self):
         user_id = self.scope['url_route']['kwargs']['pentestuser_pk']
         if user_id == 'self':
-            return self.user.id
+            return getattr(self.user, 'id', None)
         else:
             return user_id
 
diff --git a/frontend/src/utils/collab.ts b/frontend/src/utils/collab.ts
index a87db95bc..306af69bb 100644
--- a/frontend/src/utils/collab.ts
+++ b/frontend/src/utils/collab.ts
@@ -3,11 +3,10 @@ import urlJoin from "url-join";
 import { ChangeSet, EditorSelection, SelectionRange, Text } from "reportcreator-markdown/editor"
 import { CommentStatus, type Comment, type UserShortInfo } from "#imports"
 
+const WS_THROTTLE_INTERVAL = 1_000;
 const WS_RESPONSE_TIMEOUT = 7_000;
 const WS_PING_INTERVAL = 30_000;
-const WS_THROTTLE_INTERVAL_UPDATE_KEY = 1_000;
-const WS_THROTTLE_INTERVAL_UPDATE_TEXT = 1_000;
-const WS_THROTTLE_INTERVAL_AWARENESS = 1_000;
+const HTTP_THROTTLE_INTERVAL = 5_000;
 const HTTP_FALLBACK_INTERVAL = 10_000;
 
 export enum CollabEventType {
@@ -70,11 +69,15 @@ export enum CollabConnectionState {
 export type CollabConnectionInfo = {
   type: CollabConnectionType;
   connectionState: CollabConnectionState;
+  connectionConfig: {
+    throttleInterval: number;
+  },
   connectionError?: { error: any, message?: string };
   connectionAttempt?: number;
+  sendThrottled?: ReturnType<typeof throttle>;
   connect: () => Promise<void>;
   disconnect: () => Promise<void>;
-  send: (msg: CollabEvent) => void;
+  send: (events: CollabEvent[]) => void;
 };
 
 export type AwarenessInfos = {
@@ -98,6 +101,7 @@ export type CollabStoreState<T> = {
   connection?: CollabConnectionInfo;
   handleAdditionalWebSocketMessages?: (event: CollabEvent, collabState: CollabStoreState<T>) => boolean;
   perPathState: Map<string, {
+    pendingEvents: CollabEvent[];
     unconfirmedTextUpdates: TextUpdate[];
   }>;
   awareness: AwarenessInfos,
@@ -147,20 +151,9 @@ function isSubpath(subpath?: string|null, parent?: string|null) {
   return subpath === parent || subpath.startsWith(parent + (!parent.endsWith('/') ? '.' : ''));
 }
 
-async function sendPendingMessagesHttp<T>(storeState: CollabStoreState<T>, messages?: CollabEvent[]) {
+async function sendEventsHttp<T>(storeState: CollabStoreState<T>, messages?: CollabEvent[]) {
   messages = Array.from(messages || []);
 
-  for (const [p, s] of storeState.perPathState.entries()) {
-    if (s.unconfirmedTextUpdates.length > 0) {
-      messages.push({
-        type: CollabEventType.UPDATE_TEXT,
-        path: p,
-        version: storeState.version,
-        updates: s.unconfirmedTextUpdates.map(u => ({ changes: u.changes.toJSON() })),
-      });
-    }
-  }
-
   return await $fetch<{ version: number, messages: CollabEvent[], clients: CollabClientInfo[] }>(urlJoin(storeState.apiPath, '/fallback/'), { 
     method: 'POST', 
     body: {
@@ -175,22 +168,21 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
   const serverUrl = `${window.location.protocol === 'http:' ? 'ws' : 'wss'}://${window.location.host}/`;
   const wsUrl = urlJoin(serverUrl, storeState.apiPath);
   const websocket = ref<WebSocket|null>(null);
-  const perPathState = new Map<string, {
-    sendUpdateTextThrottled: ReturnType<typeof throttle>;
-    sendUpdateKeyThrottled: ReturnType<typeof throttle>;
-  }>();
-  const sendAwarenessThrottled = throttle(websocketSendAwareness, WS_THROTTLE_INTERVAL_AWARENESS, { leading: false, trailing: true });
 
   const connectionInfo = reactive<CollabConnectionInfo>({
     type: CollabConnectionType.WEBSOCKET,
     connectionState: CollabConnectionState.CLOSED,
     connectionError: undefined,
+    connectionConfig: {
+      throttleInterval: WS_THROTTLE_INTERVAL,
+    },
     connect,
     disconnect,
     send,
   });
 
-  const websocketConnectionLostTimeout = throttle(() => {
+  const websocketPingInterval = useIntervalFn(websocketSendPing, WS_PING_INTERVAL, { immediate: false });
+  const websocketResponseTimeout = throttle(() => {
     // Possible reasons: network outage, browser tab becomes inactive, server crashes
     if (websocket.value && ![WebSocket.CLOSED, WebSocket.CLOSING].includes(websocket.value?.readyState as any)) {
       websocket.value?.close(4504, 'Connection loss detection timeout');
@@ -207,7 +199,7 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
       connectionInfo.connectionState = CollabConnectionState.CONNECTING;
       websocket.value = new WebSocket(wsUrl);
       websocket.value.addEventListener('open', () => {
-        websocketConnectionLossDetection();
+        websocketPingInterval.resume();
       })
       websocket.value.addEventListener('close', (event) => {
         // Error handling
@@ -221,15 +213,12 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
           connectionInfo.connectionError = { error: event, message: event.reason };
         }
         // eslint-disable-next-line no-console
-        console.log('Websocket closed', event, connectionInfo.connectionError);
+        console.log('Websocket closed', event, toValue(connectionInfo.connectionError));
         
         // Reset data
         websocket.value = null;
-        websocketConnectionLostTimeout.cancel();
-        sendAwarenessThrottled?.cancel();
-        for (const s of perPathState.values()) {
-          s.sendUpdateTextThrottled.cancel();
-        }
+        websocketPingInterval.pause();
+        websocketResponseTimeout.cancel();
         connectionInfo.connectionState = CollabConnectionState.CLOSED;
 
         reject(connectionInfo.connectionError);
@@ -238,7 +227,7 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
         const msgData = JSON.parse(event.data) as CollabEvent;
 
         // Reset connection loss detection
-        websocketConnectionLostTimeout?.cancel();
+        websocketResponseTimeout?.cancel();
         
         // Handle message
         onReceiveMessage(msgData);
@@ -246,22 +235,7 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
         // Promise result
         if (msgData.type === CollabEventType.INIT) {
           connectionInfo.connectionState = CollabConnectionState.OPEN;
-          sendAwarenessThrottled();
           resolve();
-        } else if (msgData.type === CollabEventType.UPDATE_KEY) {
-          if (msgData.client_id !== storeState.clientID) {
-            // Clear pending events of sub-fields
-            for (const [k, v] of perPathState.entries()) {
-              if (isSubpath(k, msgData.path)) {
-                v.sendUpdateTextThrottled.cancel();
-              }
-            }
-          }
-        } else if (msgData.type === CollabEventType.CONNECT) {
-          if (msgData.client_id !== storeState.clientID) {
-            // Send awareness info to new client
-            sendAwarenessThrottled();
-          }
         }
       });
     });
@@ -272,12 +246,6 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
       return;
     }
 
-    // Send all pending messages
-    for (const s of perPathState.values()) {
-      s.sendUpdateTextThrottled.flush();
-    }
-    await nextTick();
-
     if (websocket.value?.readyState === WebSocket.OPEN) {
       await new Promise<void>((resolve) => {
         websocket.value?.addEventListener('close', () => resolve(), { once: true });
@@ -286,33 +254,8 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
     }
   }
 
-  function send(msg: CollabEvent) {
-    if (msg.type === CollabEventType.UPDATE_KEY) {
-      if (msg.update_awareness) {
-        // Awareness info is included in update_key message
-        sendAwarenessThrottled?.cancel();
-      }
-
-      // Cancel pending update_key events of child fields
-      for (const [k, v] of perPathState.entries()) {
-        if (isSubpath(k, msg.path) && k !== msg.path) {
-          v.sendUpdateKeyThrottled.cancel();
-        }
-      }
-      
-      // Throttle update_key messages
-      const s = ensurePerPathState(msg.path!);
-      s.sendUpdateKeyThrottled(msg);
-    } else if (msg.type === CollabEventType.UPDATE_TEXT) {
-      // Cancel pending awareness send: awareness info is included in the next update_text message
-      sendAwarenessThrottled.cancel();
-      // Throttle update_text messages
-      const s = ensurePerPathState(msg.path!);
-      s.sendUpdateTextThrottled();
-    } else if (msg.type === CollabEventType.AWARENESS) {
-      // Throttle awareness messages
-      sendAwarenessThrottled();
-    } else {
+  function send(events: CollabEvent[]) {
+    for (const msg of events) {
       websocketSend(msg);
     }
   }
@@ -322,74 +265,17 @@ export function connectionWebsocket<T = any>(storeState: CollabStoreState<T>, on
       return;
     }
     websocket.value?.send(JSON.stringify(msg));
-    websocketConnectionLostTimeout();
+    websocketResponseTimeout();
   }
 
-  function websocketSendAwareness() {
-    if (!storeState.permissions.write) {
-      return;
-    }
-
-    websocketSend({
-      type: CollabEventType.AWARENESS,
-      path: storeState.awareness.self.path,
+  async function websocketSendPing() {
+    websocketSend({ 
+      type: CollabEventType.PING,
       version: storeState.version,
-      selection: storeState.awareness.self.selection?.toJSON(),
+      path: null,
     });
   }
 
-  function websocketSendUpdateText(path: string) {
-    const updates = storeState.perPathState.get(path)?.unconfirmedTextUpdates.map(u => ({ changes: u.changes.toJSON() })) || [];
-    if (updates.length === 0) {
-      return;
-    }
-
-    let selection;
-    if (storeState.awareness.self.path === path) {
-      // Awareness info is included in update_text message
-      // No need to send it separately
-      selection = storeState.awareness.self.selection?.toJSON();
-      sendAwarenessThrottled?.cancel();
-    }
-
-    websocketSend({
-      type: CollabEventType.UPDATE_TEXT,
-      path,
-      version: storeState.version,
-      updates,
-      selection,
-    });
-  }
-
-  function websocketSendUpdateKey(msg: CollabEvent) {
-    websocketSend({
-      ...msg,
-      update_awareness: msg.update_awareness && msg.path === storeState.awareness.self.path,
-    });
-  }
-
-  function ensurePerPathState(path: string) {
-    if (!perPathState.has(path)) {
-      perPathState.set(path, {
-        sendUpdateTextThrottled: throttle(() => websocketSendUpdateText(path), WS_THROTTLE_INTERVAL_UPDATE_TEXT, { leading: false, trailing: true }),
-        sendUpdateKeyThrottled: throttle((msg: CollabEvent) => websocketSendUpdateKey(msg), WS_THROTTLE_INTERVAL_UPDATE_KEY, { leading: true, trailing: true }),
-      });
-    }
-    return perPathState.get(path)!;
-  }
-
-  async function websocketConnectionLossDetection() {
-    const ws = websocket.value;
-    while (connectionInfo.connectionState !== CollabConnectionState.CLOSED && websocket.value === ws) {
-      await new Promise(resolve => setTimeout(resolve, WS_PING_INTERVAL));
-      websocketSend({ 
-        type: CollabEventType.PING,
-        version: storeState.version,
-        path: null,
-      });
-    }
-  }
-
   return connectionInfo;
 }
 
@@ -397,14 +283,16 @@ export function connectionHttpFallback<T = any>(storeState: CollabStoreState<T>,
   const httpUrl = urlJoin(storeState.apiPath, '/fallback/');
   const connectionInfo = reactive<CollabConnectionInfo>({
     type: CollabConnectionType.HTTP_FALLBACK,
+    connectionConfig: {
+      throttleInterval: HTTP_THROTTLE_INTERVAL,
+    },
     connectionState: CollabConnectionState.CLOSED,
     connectionError: undefined,
     connect,
     disconnect,
     send,
   });
-  const pendingMessages = ref<CollabEvent[]>([]);
-  const sendInterval = ref();
+  const fetchInterval = useIntervalFn(() => sendAndReceiveMessages([]), HTTP_FALLBACK_INTERVAL, { immediate: false });
 
   async function connect() {
     connectionInfo.connectionState = CollabConnectionState.CONNECTING;
@@ -413,7 +301,7 @@ export function connectionHttpFallback<T = any>(storeState: CollabStoreState<T>,
       onReceiveMessage(res);
       connectionInfo.connectionState = CollabConnectionState.OPEN;
 
-      sendInterval.value = setInterval(sendPendingMessages, HTTP_FALLBACK_INTERVAL);
+      fetchInterval.resume();
     } catch (error) {
       connectionInfo.connectionError = { error };
       connectionInfo.connectionState = CollabConnectionState.CLOSED;
@@ -421,36 +309,28 @@ export function connectionHttpFallback<T = any>(storeState: CollabStoreState<T>,
     }
   }
 
-  async function disconnect(opts?: { skipSendPendingMessages?: boolean }) {
-    clearInterval(sendInterval.value);
-    sendInterval.value = undefined;
-
-    if (!opts?.skipSendPendingMessages) {
-      await sendPendingMessages();
-    }
-
+  async function disconnect() {
+    fetchInterval.pause();
     connectionInfo.connectionState = CollabConnectionState.CLOSED;
   }
 
-  function send(msg: CollabEvent) {
-    if ([CollabEventType.AWARENESS, CollabEventType.PING, CollabEventType.UPDATE_TEXT].includes(msg.type)) {
-      // Do not send awareness and ping event at all.
-      // collab.update_text is handled in sendPendingMessages
-    } else if ([CollabEventType.CREATE, CollabEventType.DELETE].includes(msg.type)) {
-      // Send immediately
-      pendingMessages.value.push(msg);
-      sendPendingMessages();
-    } else {
-      pendingMessages.value.push(msg);
+  function send(events: CollabEvent[]) {
+    // Do not send awareness and ping event at all.
+    events = events.filter(e => ![CollabEventType.AWARENESS, CollabEventType.PING].includes(e.type));
+    if (events.length === 0) {
+      return;
     }
-  }
 
-  async function sendPendingMessages() {
-    const messages = [...pendingMessages.value]
-    pendingMessages.value = [];
+    // Reset fetch interval because sendMessages also fetches server events since last update
+    fetchInterval.pause();
+    fetchInterval.resume();
+
+    sendAndReceiveMessages(events);
+  }
 
+  async function sendAndReceiveMessages(events?: CollabEvent[]) {
     try {
-      const res = await sendPendingMessagesHttp(storeState, messages);
+      const res = await sendEventsHttp(storeState, events || []);
       
       storeState.version = res.version;
       storeState.awareness.clients = res.clients;
@@ -460,7 +340,7 @@ export function connectionHttpFallback<T = any>(storeState: CollabStoreState<T>,
     } catch (error) {
       // Disconnect on error
       connectionInfo.connectionError = { error };
-      disconnect({ skipSendPendingMessages: true });
+      disconnect();
     }
   }
 
@@ -507,9 +387,17 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
       other: {},
       clients: [],
     };
+    storeState.connection?.sendThrottled?.cancel();
 
-    await connection.connect();
-    storeState.connection = connection;
+    try {
+      connection.sendThrottled = throttle(sendPendingMessages, connection.connectionConfig.throttleInterval, { leading: false, trailing: true });
+      await connection.connect();
+      storeState.connection = connection;
+    } catch (error) {
+      connection.sendThrottled?.cancel();
+      connection.sendThrottled = undefined;
+      throw error;
+    }
   }
 
   async function connect(options?: { connectionType?: CollabConnectionType }) {
@@ -524,18 +412,22 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
     } else {
       // If there are pending events from the previous connection, send them now to prevent losing events.
       // Try to sync as many changes as possible. When some events cannot be processed, we don't care since we cannot handle them anyway.
-      if (storeState.version > 0 && Array.from(storeState.perPathState.values()).some(s => s.unconfirmedTextUpdates.length > 0)) {
-        try {
-          await sendPendingMessagesHttp(storeState);
-        } catch {
-          // Ignore errors. If data is saved: good, if not: discard it (there's most likely an error or conflict in it)
+      try {
+        const pendingEvents = getPendingEvents({ clearPending: false, includeAwareness: false });
+        if (storeState.version > 0 && pendingEvents.length > 0) {
+            await sendEventsHttp(storeState, pendingEvents);
         }
+      } catch {
+        // Ignore errors. If data is saved: good, if not: discard it (there's most likely a error or conflict in it)
       }
 
       // Dummy connection info with connectionState=CONNECTING
       storeState.connection = {
         type: CollabConnectionType.WEBSOCKET,
         connectionState: CollabConnectionState.CONNECTING,
+        connectionConfig: {
+          throttleInterval: 1_000,
+        },
         connectionAttempt: 0,
         connect: () => Promise.resolve(),
         disconnect: () => Promise.resolve(),
@@ -572,6 +464,12 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
 
   async function disconnect() {
     const con = storeState.connection;
+    if (con?.sendThrottled) {
+      // Flush pending events
+      con.sendThrottled();
+      con.sendThrottled.flush();
+      con.sendThrottled = undefined;
+    }
     await con?.disconnect();
     if (storeState.connection === con) {
       storeState.connection = undefined;
@@ -595,6 +493,9 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
           path: c.path, 
           selection: undefined, 
         }]));
+
+        // Send awareness
+        sendAwarenessThrottled();
       } else if (msgData.type === CollabEventType.UPDATE_KEY) {
         if (msgData.client_id !== storeState.clientID) {
           setValue(msgData.path!, msgData.value);
@@ -623,6 +524,9 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
           // Add new client
           storeState.awareness.clients.push(msgData.client!);
         }
+
+        // Send awareness
+        sendAwarenessThrottled();
       } else if (msgData.type === CollabEventType.DISCONNECT) {
         // Remove client
         storeState.awareness.clients = storeState.awareness.clients
@@ -660,12 +564,103 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
   function ensurePerPathState(path: string) {
     if (!storeState.perPathState.has(path)) {
       storeState.perPathState.set(path, {
+        pendingEvents: [],
         unconfirmedTextUpdates: [],
       });
     }
     return storeState.perPathState.get(path)!;
   }
 
+  function getPendingEvents(options?: { clearPending?: boolean, includeAwareness?: boolean }) {
+    const events = [];
+    let sendAwarenessEvent = null as boolean|null;
+    for (const [p, s] of storeState.perPathState.entries()) {
+      // Collect non-text events
+      for (const e of s.pendingEvents) {
+        if (e.type === CollabEventType.AWARENESS) {
+          if (sendAwarenessEvent === null) {
+            sendAwarenessEvent = true;
+          }
+        } else if (e.type !== CollabEventType.UPDATE_TEXT) {
+          let update_awareness = undefined as boolean|undefined;
+          if (e.update_awareness && e.path === storeState.awareness.self.path) {
+            update_awareness = true;
+            // Do not send awareness separately
+            sendAwarenessEvent = false;
+          }
+          events.push({
+            ...e,
+            version: storeState.version,
+            update_awareness,
+          });
+        }
+      }
+      
+      // Collect text updates
+      if (s.unconfirmedTextUpdates.length > 0) {
+        let selection;
+        if (storeState.awareness.self.path === p) {
+          // Awareness info is included in update_text message
+          // No need to send it separately
+          selection = storeState.awareness.self.selection?.toJSON();
+          // Do not send awareness separately because informations are included in update_text
+          sendAwarenessEvent = false;   
+        }
+
+        events.push({
+          type: CollabEventType.UPDATE_TEXT,
+          path: p,
+          version: storeState.version,
+          updates: s.unconfirmedTextUpdates.map(u => ({ changes: u.changes.toJSON() })),
+          selection,
+        })
+      }
+
+      // Clear pending events that will be sent
+      if (options?.clearPending) {
+        s.pendingEvents = [];
+      }
+    }
+
+    // Send awareness event
+    if (options?.includeAwareness && sendAwarenessEvent) {
+      events.push({
+        type: CollabEventType.AWARENESS,
+        path: storeState.awareness.self.path,
+        version: storeState.version,
+        selection: storeState.awareness.self.selection?.toJSON(),
+      });
+    }
+
+    return events;
+  }
+
+  function sendPendingMessages() {
+    const events = getPendingEvents({ clearPending: true, includeAwareness: true });
+    storeState.connection?.send(events);
+  }
+
+  function sendEventsThrottled(...events: CollabEvent[]) {
+    for (const e of events) {
+      ensurePerPathState(e.path!).pendingEvents.push(e);
+    }
+    storeState.connection?.sendThrottled?.();
+  }
+
+  function sendEventsImmediately(...events: CollabEvent[]) {
+    sendEventsThrottled(...events);
+    storeState.connection?.sendThrottled?.flush();
+  }
+
+  function sendAwarenessThrottled() {
+    sendEventsThrottled({
+      type: CollabEventType.AWARENESS,
+      path: storeState.awareness.self.path,
+      version: storeState.version,
+      selection: storeState.awareness.self.selection?.toJSON(),
+    });
+  }
+
   function updateKey(event: any) {
     // Update local state
     const dataPath = toDataPath(event.path);
@@ -680,7 +675,7 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
     }
 
     // Propagate event to other clients
-    storeState.connection?.send({
+    sendEventsThrottled({
       type: event.type || CollabEventType.UPDATE_KEY,
       path: dataPath,
       version: storeState.version,
@@ -694,7 +689,7 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
     // Do not update local state here. Wait for server event.
     // Propagate event to other clients
     const dataPath = toDataPath(event.path);
-    storeState.connection?.send({
+    sendEventsImmediately({
       type: CollabEventType.CREATE,
       path: dataPath,
       version: storeState.version,
@@ -706,7 +701,7 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
     // Do not update local state here. Wait for server event.
     // Propagate event to other clients
     const dataPath = toDataPath(event.path);
-    storeState.connection?.send({
+    sendEventsImmediately({
       type: CollabEventType.DELETE,
       path: dataPath,
       version: storeState.version,
@@ -783,20 +778,15 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
     })));
 
     // Propagate unconfirmed events to other clients
-    storeState.connection?.send({
-      type: CollabEventType.UPDATE_TEXT,
-      path: dataPath,
-      version: storeState.version,
-      updates: perPathState.unconfirmedTextUpdates.map(u => ({ changes: u.changes.toJSON() })),
-      selection: storeState.awareness.self.selection?.toJSON(),
-    })
+    sendEventsThrottled();
   }
 
   function setValue(path: string, value: any) {
-    // Clear pending text updates, because they are overwritten by the value of collab.update_key
+    // Clear pending updates, because they are overwritten by the value of collab.update_key
     for (const [k, v] of storeState.perPathState.entries()) {
       if (isSubpath(k, path)) {
         v.unconfirmedTextUpdates = [];
+        v.pendingEvents = [];
       }
     }
 
@@ -835,12 +825,7 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
         path: dataPath,
         selection: event.selection,
       };
-      storeState.connection?.send({
-        type: CollabEventType.AWARENESS,
-        path: dataPath,
-        version: storeState.version,
-        selection: event.selection?.toJSON(),
-      });
+      sendAwarenessThrottled();
     }
   }
 

From aca12c1cc494e576bc075fafc2142647cb553b0e Mon Sep 17 00:00:00 2001
From: Michael Wedl <michael@syslifters.com>
Date: Thu, 26 Sep 2024 14:00:48 +0200
Subject: [PATCH 2/3] Begin implementing unit tests for collab.ts

---
 frontend/package-lock.json                    |   1 +
 frontend/src/utils/collab.ts                  |  11 +-
 frontend/test/collab.test.ts                  | 449 ++++++++++++++++++
 ...ons.test.js => markdownExtensions.test.ts} |   4 +-
 packages/markdown/index.js                    |   5 +
 5 files changed, 466 insertions(+), 4 deletions(-)
 create mode 100644 frontend/test/collab.test.ts
 rename frontend/test/{markdownExtensions.test.js => markdownExtensions.test.ts} (97%)

diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 01e01afce..48f186fea 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -51,6 +51,7 @@
       }
     },
     "../packages/markdown": {
+      "name": "reportcreator-markdown",
       "version": "0.1.0",
       "dependencies": {
         "@codemirror/commands": "~6.6",
diff --git a/frontend/src/utils/collab.ts b/frontend/src/utils/collab.ts
index 306af69bb..380adc9ab 100644
--- a/frontend/src/utils/collab.ts
+++ b/frontend/src/utils/collab.ts
@@ -353,6 +353,9 @@ export function connectionHttpReadonly<T = any>(storeState: CollabStoreState<T>,
     type: CollabConnectionType.HTTP_READONLY,
     connectionState: CollabConnectionState.CLOSED,
     connectionError: undefined,
+    connectionConfig: {
+      throttleInterval: HTTP_THROTTLE_INTERVAL,
+    },
     connect,
     disconnect: () => Promise.resolve(),
     send: () => {},
@@ -582,7 +585,7 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
             sendAwarenessEvent = true;
           }
         } else if (e.type !== CollabEventType.UPDATE_TEXT) {
-          let update_awareness = undefined as boolean|undefined;
+          let update_awareness = false;
           if (e.update_awareness && e.path === storeState.awareness.self.path) {
             update_awareness = true;
             // Do not send awareness separately
@@ -637,7 +640,9 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
 
   function sendPendingMessages() {
     const events = getPendingEvents({ clearPending: true, includeAwareness: true });
-    storeState.connection?.send(events);
+    if (events.length > 0) {
+      storeState.connection?.send(events);
+    }
   }
 
   function sendEventsThrottled(...events: CollabEvent[]) {
@@ -1019,8 +1024,10 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
 
   return {
     connect,
+    connectTo,
     disconnect,
     onCollabEvent,
+    onReceiveMessage,
     storeState,
     data: computed(() => storeState.data),
     readonly: computed(() => storeState.connection?.connectionState !== CollabConnectionState.OPEN || !storeState.permissions.write),
diff --git a/frontend/test/collab.test.ts b/frontend/test/collab.test.ts
new file mode 100644
index 000000000..5edcf43f0
--- /dev/null
+++ b/frontend/test/collab.test.ts
@@ -0,0 +1,449 @@
+import { describe, test, expect, beforeEach, vi } from 'vitest'
+import { setActivePinia, createPinia } from 'pinia'
+import { v4 as uuid4 } from 'uuid'
+import { cloneDeep } from 'lodash-es'
+import { CollabEventType, type CollabEvent, type User } from '#imports';
+import { ChangeSet, EditorSelection } from 'reportcreator-markdown/editor';
+
+
+async function createCollab(options?: { collabInitEvent?: Partial<CollabEvent> }) {
+  vi.useFakeTimers();
+  setActivePinia(createPinia());
+
+  const storeState = reactive(makeCollabStoreState({
+    apiPath: '/api/ws/test/',
+    initialData: {},
+  })) as CollabStoreState<any>;
+  const collab = useCollab(storeState);
+
+  const connection = {
+    type: CollabConnectionType.WEBSOCKET,
+    connectionState: CollabConnectionState.CONNECTING,
+    connectionConfig: {
+      throttleInterval: 1000,
+    },
+    connect: vi.fn(),
+    disconnect: vi.fn(),
+    send: vi.fn(),
+    receive: collab.onReceiveMessage,
+  }
+  await collab.connectTo(connection);
+
+  const collabInitEvent = {
+    type: CollabEventType.INIT,
+    version: 1,
+    client_id: 'self',
+    path: null,
+    permissions: {
+      read: true,
+      write: true,
+    },
+    data: {
+      field_text: 'ABCD',
+      field_key: 'value',
+      field_list: ['a', 'b', 'c'],
+    },
+    comments: [
+      {
+        id: uuid4(),
+        text: 'comment on field_text',
+        path: 'field_text',
+        text_range: { from: 1, to: 2 }, // 'BC'
+      },
+      {
+        id: uuid4(),
+        text: 'comment on field_list[0]',
+        path: 'field_list.[0]',
+        text_range: { from: 0, to: 1 }, // 'a'
+      }
+    ],
+    clients: [
+      {
+        client_id: 'self',
+        client_color: 'red',
+        user: { id: uuid4(), username: 'user', name: 'User Name' } as unknown as User,
+      },
+      {
+        client_id: 'other',
+        client_color: 'blue',
+        user: { id: uuid4(), username: 'other', name: 'Other User' } as unknown as User,
+      }
+    ],
+    ...(options?.collabInitEvent || {})
+  }
+  connection.receive(cloneDeep(collabInitEvent));
+
+  // Reset mocks
+  await vi.runOnlyPendingTimersAsync();
+  vi.clearAllMocks();
+
+  return {
+    collab,
+    connection: connection as typeof connection & CollabConnectionInfo,
+    collabInitEvent,
+  }
+}
+
+function createReceivedEvent(event: Partial<CollabEvent>): CollabEvent {
+  const defaultData = {
+    version: 2,
+    client_id: 'other',
+  } as Partial<CollabEvent>;
+
+  if (event.type === CollabEventType.UPDATE_KEY) {
+    Object.assign(defaultData, {
+      path: 'field_key',
+      value: 'changed value',
+    });
+  } else if (event.type === CollabEventType.UPDATE_TEXT) {
+    Object.assign(defaultData, {
+      path: 'field_text',
+      updates: [{ changes: [0, [0, 'a']]}],
+    });
+  }
+
+  return {
+    ...defaultData,
+    ...event,
+  } as CollabEvent;
+}
+
+function expectEventsSentInConnection(connection: any, events: Partial<CollabEvent>[]) {
+  vi.runOnlyPendingTimers();
+  expect(connection.send).toHaveBeenCalledTimes(1);
+  const sendArgs = connection.send.mock.calls[0]![0]! as CollabEvent[];
+  expect(sendArgs.length).toBe(events.length);
+  for (let i = 0; i < events.length; i++) {
+    expect(sendArgs[i]).toMatchObject(events[i]!);
+  }
+}
+
+
+describe('connection', () => {
+  let { collab, connection }: Awaited<ReturnType<typeof createCollab>> = {} as any;
+  beforeEach(async () => {
+    const res = await createCollab(); 
+    collab = res.collab;
+    connection = res.connection;
+  })
+
+  test('disconnect gracefully', async () => {
+    await collab.disconnect();
+    expect(connection.disconnect).toHaveBeenCalled();
+  });
+
+  test('disconnect on error', async () => {
+    collab.onReceiveMessage(createReceivedEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: 'field_text',
+      updates: [{ changes: [10, [0, 'a']]}],  // Invalid update
+    }));
+    expect(connection.disconnect).toHaveBeenCalled();
+    expect(connection.connectionError).toBeTruthy();
+  });
+
+  test('send after disconnect', async () => {
+    await collab.disconnect();
+    connection.send.mockClear();
+    collab.onCollabEvent({type: CollabEventType.UPDATE_KEY, path: collab.storeState.apiPath + 'field_key', value: 'value'});
+    await vi.runOnlyPendingTimersAsync();
+    expect(connection.send).not.toHaveBeenCalled();
+  });
+
+  test('flush send on disconnect', async () => {
+    collab.onCollabEvent({type: CollabEventType.UPDATE_KEY, path: collab.storeState.apiPath + 'field_key', value: 'value'});
+    expect(connection.send).not.toHaveBeenCalled();
+    await collab.disconnect();
+    expect(connection.send).toHaveBeenCalled();
+  });
+
+  test('send throttling', async () => {
+    collab.onCollabEvent({type: CollabEventType.UPDATE_KEY, path: collab.storeState.apiPath + 'field_key', value: 'value'});
+    expect(connection.send).not.toHaveBeenCalled();
+    await vi.runOnlyPendingTimersAsync();
+    expect(connection.send).toHaveBeenCalled();
+  });
+
+  test('send immediate', async () => {
+    collab.onCollabEvent({type: CollabEventType.CREATE, path: collab.storeState.apiPath + 'field_new', value: 'value'});
+    expect(connection.send).toHaveBeenCalled();
+  });
+
+  test('ignore events for other connections', async () => {
+    collab.onCollabEvent({type: CollabEventType.CREATE, path: 'not_for' + collab.storeState.apiPath + 'field_new', value: 'value'});
+    expect(connection.send).not.toHaveBeenCalled();
+  })
+});
+
+
+describe('send and receive', () => {
+  let { collab, connection, collabInitEvent }: Awaited<ReturnType<typeof createCollab>> = {} as any;
+  beforeEach(async () => {
+    const res = await createCollab();
+    collab = res.collab;
+    connection = res.connection;
+    collabInitEvent = res.collabInitEvent;
+  });
+
+  function expectEventsSent(...events: Partial<CollabEvent>[]) {
+    expectEventsSentInConnection(connection, events);
+  }
+
+  test('version updated', () => {
+    const newVersion = 1234;
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.UPDATE_KEY, version: newVersion}));
+    expect(collab.storeState.version).toBe(newVersion);
+  });
+
+  test('collab.update_key', () => {
+    const newValue = 'changed value';
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.UPDATE_KEY, path: 'field_key', value: newValue}));
+    expect(collab.storeState.data.field_key).toBe(newValue);
+  });
+
+  test('collab.update_key clear pending events of child fields', async () => {
+    const newValue = ['c', 'b', 'a'];
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_KEY,
+      path: collab.storeState.apiPath + 'field_list.[0]',
+      value: 'x',
+    });
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: collab.storeState.apiPath + 'field_list.[1]',
+      updates: [{ changes: ChangeSet.fromJSON([1, [0, 'y']]) }],
+    });
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.UPDATE_KEY, path: 'field_list', value: newValue}));
+    await vi.runOnlyPendingTimersAsync();
+    expect(collab.storeState.data.field_list).toEqual(newValue);
+    expect(connection.send).not.toHaveBeenCalled();
+  });
+
+  test('collab.create', () => {
+    const newValue = 'new value';
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.CREATE, path: 'field_new', value: newValue}));
+    expect(collab.storeState.data.field_new).toBe(newValue);
+  });
+
+  test('collab.delete', () => {
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.DELETE, path: 'field_text'}));
+    expect(collab.storeState.data.field_text).toBeUndefined();
+  });
+
+  test('collab.create list', () => {
+    const newValue = 'new value';
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.CREATE, path: 'field_list.[3]', value: newValue}));
+    expect(collab.storeState.data.field_list[3]).toBe(newValue);
+    expect(collab.storeState.data.field_list.length).toBe(4);
+  });
+
+  test('collab.delete list', () => {
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.DELETE, path: 'field_list.[1]'}));
+    const newList = collabInitEvent.data.field_list;
+    newList.splice(1, 1);
+    expect(collab.storeState.data.field_list.length).toBe(newList.length);
+    expect(collab.storeState.data.field_list).toEqual(newList);
+  });
+
+  test('collab.connect', () => {
+    collab.onReceiveMessage(createReceivedEvent({
+      type: CollabEventType.CONNECT,
+      client_id: 'other2',
+      client: {
+        client_id: 'other2',
+        client_color: 'green',
+        user: { id: uuid4(), username: 'other2', name: 'Other User 2' } as unknown as User,
+      },
+    }));
+    expect(collab.storeState.awareness.clients.at(-1)?.client_id).toBe('other2');
+  });
+
+  test('collab.disconnect', () => {
+    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.DISCONNECT, client_id: 'other'}));
+    expect(collab.storeState.awareness.clients.length).toBe(1);
+  })
+
+  test('collab.awareness', () => {
+    const selection = EditorSelection.create([EditorSelection.range(0, 1)]).toJSON();
+    collab.onReceiveMessage(createReceivedEvent({
+      type: CollabEventType.AWARENESS,
+      client_id: 'other',
+      path: 'field_text',
+      selection,
+    }));
+    const updatedAwareness = collab.storeState.awareness.other['other']!;
+    expect(updatedAwareness.path).toBe('field_text');
+    expect(updatedAwareness.selection?.toJSON()).toEqual(selection);
+  });
+
+  test('multiple collab.update_key: only last one sent', () => {
+    collab.onCollabEvent({type: CollabEventType.UPDATE_KEY, path: collab.storeState.apiPath + 'field_key', value: 'value1'});
+    collab.onCollabEvent({type: CollabEventType.UPDATE_KEY, path: collab.storeState.apiPath + 'field_key', value: 'value2'});
+    expectEventsSent({type: CollabEventType.UPDATE_KEY, path: 'field_key', value: 'value2'});
+  });
+
+  test('multiple collab.update_text: combined', async () => {
+    const updates1 = [{changes: ChangeSet.fromJSON([4, [0, 'E']])}];
+    const updates2 = [{changes: ChangeSet.fromJSON([5, [0, 'F']])}]
+    collab.onCollabEvent({type: CollabEventType.UPDATE_TEXT, path: collab.storeState.apiPath + 'field_text', updates: updates1});
+    collab.onCollabEvent({type: CollabEventType.UPDATE_TEXT, path: collab.storeState.apiPath + 'field_text', updates: updates2});
+    expectEventsSent({type: CollabEventType.UPDATE_TEXT, path: 'field_text', updates: updates1.concat(updates2).map(u => ({changes: u.changes.toJSON()}))});
+  });
+});
+
+
+describe('collab.awareness', () => {
+  let { collab, connection }: Awaited<ReturnType<typeof createCollab>> = {} as any;
+  beforeEach(async () => {
+    const res = await createCollab();
+    collab = res.collab;
+    connection = res.connection;
+  });
+
+  function expectEventsSent(...events: Partial<CollabEvent>[]) {
+    expectEventsSentInConnection(connection, events);
+  }
+
+  test('multiple collab.awareness events: only last one sent', () => {
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_key'});
+    const selection = EditorSelection.create([EditorSelection.cursor(1)])
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text', selection});
+    expectEventsSent({type: CollabEventType.AWARENESS, path: 'field_text', selection: selection.toJSON()});
+  });
+
+  test('collab.update_text with selection: collab.awareness not sent', () => {
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text'});
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: collab.storeState.apiPath + 'field_text',
+      updates: [{ changes: ChangeSet.fromJSON([4, [0, 'E']]) }],
+      selection: EditorSelection.create([EditorSelection.cursor(0)]),
+    });
+    const selection = EditorSelection.create([EditorSelection.range(0, 2)]);
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text', selection})
+    expectEventsSent({type: CollabEventType.UPDATE_TEXT, path: 'field_text', selection: selection.toJSON()});
+  });
+
+  test('collab.update_text with selection, then other field focussed: collab.awareness sent, update_text.selection=undefined', () => {
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: collab.storeState.apiPath + 'field_text',
+      updates: [{ changes: ChangeSet.fromJSON([4, [0, 'E']]) }],
+      selection: EditorSelection.create([EditorSelection.cursor(1)]),
+    });
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_key'});
+    expectEventsSent(
+      {type: CollabEventType.UPDATE_TEXT, path: 'field_text', selection: undefined},
+      {type: CollabEventType.AWARENESS, path: 'field_key'},
+    );
+  })
+
+  test('collab.update_key with update_awareness=True: collab.awareness not sent', () => {
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text'});
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_KEY,
+      path: collab.storeState.apiPath + 'field_key',
+      value: 'value',
+      updateAwareness: true,
+    });
+    expectEventsSent({type: CollabEventType.UPDATE_KEY, path: 'field_key', update_awareness: true});
+  });
+
+  test('collab.update_key with update_awareness=False: collab.awareness sent', () => {
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text'});
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_KEY,
+      path: collab.storeState.apiPath + 'field_key',
+      value: 'value',
+      updateAwareness: false,
+    });
+    expectEventsSent(
+      {type: CollabEventType.UPDATE_KEY, path: 'field_key', update_awareness: false },
+      {type: CollabEventType.AWARENESS, path: 'field_text'},
+    );
+  });
+
+  test('collab.update_key with update_awareness=True, other field focussed: collab.awareness sent', () => {
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_KEY,
+      path: collab.storeState.apiPath + 'field_key',
+      value: 'value',
+      updateAwareness: true,
+    });
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text'});
+    expectEventsSent(
+      {type: CollabEventType.UPDATE_KEY, path: 'field_key', update_awareness: false},
+      {type: CollabEventType.AWARENESS, path: 'field_text'},
+    );
+  });
+});
+
+
+// describe('collab.update_text', () => {
+//   let { collab }: Awaited<ReturnType<typeof createCollab>> = {} as any;
+//   beforeEach(async () => {
+//     const res = await createCollab();
+//     collab = res.collab;
+//   });
+
+  
+// });
+
+
+// TODO: collab tests
+// * [x] refactor
+//   * [x] connection
+//     * [x] connect, disconnect, connectionInfo, onReceiveMessage
+//     * [x] connectionConfig: throttle timer
+//     * [x] send(events: CollabEvent[])
+//   * [x] WSConnection
+//     * [x] send: send all events immediately, no throttle
+//   * [x] HTTPConnection
+//     * [x] timer: fetch events every 10s
+//     * [x] reset timer on send
+//   * [x] useCollab:
+//     * [x] timer with connection.connectionConfig.throttle_interval
+//     * [x] 1 timer for all fields, remove per-field timers
+//     * [x] send events to connection in batch
+//     * [x] throttle update_text, update_key, awareness
+//     * [x] do not throttle other events
+//   * [x] useCollab: moved from WSConnection
+//     * [x] before connect: stop timer, reset pending events
+//     * [x] on disconnect: stop timer, flush pending events
+//     * [x] on receive CollabEventType.INIT: sendAwarenessThrottled
+//     * [x] on receive CollabEventType.CONNECT: sendAwarenessThrottled
+// * [ ] tests
+//   * [ ] connection
+//     * [x] send not called when connectionState != connected
+//     * [ ] test connection timeout detection: no answer received from server => disconnect called
+//   * [ ] collab receive
+//     * [x] events received: version updated
+//     * [ ] test update_text received: rebase positions for text + selection + comments
+//     * [x] test update_key received: clear pending events of field + child fields
+//     * [x] test create received
+//     * [x] test delete received
+//     * [x] text receive list operations
+//   * [ ] awareness
+//     * [x] update_text event with selection: send + clear pending collab.awareness events
+//     * [x] awareness event: send + clear pending collab.awareness events
+//     * [x] send update_key with update_awareness=True: collab.awareness not sent
+//     * [x] send update_key with update_awareness=True, other field focussed:  collab.awareness sent
+//     * [x] send update_key with update_awareness=False: update_awareness sent
+//     * [x] send update_text with selection, other field focussed: update_text.selection=undefined, update_awareness sent
+//     * [ ] update selection on receive collab.update_text
+//     * [ ] update selection on send collab.update_text
+//     * [ ] rebase selection on receive collab.awareness onto unconfirmedTextUpdates
+//   * [ ] collab send
+//     * [x] multiple update_text events on same field: combine
+//     * [x] multiple update_key events on same field: only send last
+//     * [x] update_text child, then update_key parent: only send update_key
+//   * [ ] collab.update_text
+//     * [ ] test rebase on receive remote events 
+//     * [ ] test rebase on send onto unconfirmedTextUpdates
+//     * [ ] test mark text updates as confirmed on receive remote events
+//   * [ ] comments
+//     * [ ] update according to event.comments
+//     * [ ] update comments position on receive collab.update_text
+//     * [ ] update comments position on send collab.update_text
+//     * [ ] rebase comments position on receive collab.awareness onto unconfirmedTextUpdates
+
diff --git a/frontend/test/markdownExtensions.test.js b/frontend/test/markdownExtensions.test.ts
similarity index 97%
rename from frontend/test/markdownExtensions.test.js
rename to frontend/test/markdownExtensions.test.ts
index 66a2ef3cd..8a9539b48 100644
--- a/frontend/test/markdownExtensions.test.js
+++ b/frontend/test/markdownExtensions.test.ts
@@ -1,7 +1,7 @@
 import { describe, test, expect } from 'vitest'
 import { formatMarkdown, renderMarkdownToHtml } from 'reportcreator-markdown';
 
-function codeBlock(content, language = null) {
+function codeBlock(content: string, language: string|null = null) {
   return `<pre class="code-block"><code class="${language ? 'language-' + language + ' ' : ''}hljs">${
     content.split('\n')
       .map((l, idx) => `<span class="code-block-line" data-line-number="${idx + 1}">${l}</span>`)
@@ -85,7 +85,7 @@ describe('Markdown extensions', () => {
     '`{{ text }}`': '<p><code class="code-inline">&#x7B;&#x7B; text &#x7D;&#x7D;</code></p>',
     '```\n{{ text }}\n```': codeBlock('&#x7B;&#x7B; text &#x7D;&#x7D;'),
     '```\n\\# \\{\\{ text \\}\\}\n```': codeBlock('\\# \\&#x7B;\\&#x7B; text \\&#x7D;\\&#x7D;'),
-  }).map(([md, expected]) => [md, typeof expected === 'string' ? { html: expected, formatted: md } : expected])) {
+  }).map(([md, expected]) => [md, typeof expected === 'string' ? { html: expected, formatted: md } : expected]) as [string, { html: string, formatted: string }][]) {
     test(md, () => {
       const html = renderMarkdownToHtml(md).replaceAll(/ data-position=".*?"/g, '').trim()
       expect(html).toBe(expected.html);
diff --git a/packages/markdown/index.js b/packages/markdown/index.js
index 8d9c201c3..b0078b1ad 100644
--- a/packages/markdown/index.js
+++ b/packages/markdown/index.js
@@ -65,6 +65,11 @@ export function markdownParser() {
 }
 
 
+/**
+ * 
+ * @param {string} text 
+ * @returns {string}
+ */
 export function formatMarkdown(text) {
   const md = markdownParser()
     .use(remarkParse)

From 461b0f833dd3760e1c54a65fcf9df466658060c3 Mon Sep 17 00:00:00 2001
From: Michael Wedl <michael@syslifters.com>
Date: Tue, 1 Oct 2024 13:10:04 +0200
Subject: [PATCH 3/3] Implement tests for collab.ts

---
 frontend/src/stores/shareinfos.ts |   1 -
 frontend/src/utils/collab.ts      |   6 +-
 frontend/test/collab.test.ts      | 280 ++++++++++++++++++++----------
 3 files changed, 186 insertions(+), 101 deletions(-)

diff --git a/frontend/src/stores/shareinfos.ts b/frontend/src/stores/shareinfos.ts
index ca828bf53..690ef5612 100644
--- a/frontend/src/stores/shareinfos.ts
+++ b/frontend/src/stores/shareinfos.ts
@@ -19,7 +19,6 @@ export const useShareInfoStore = defineStore('shareinfo', {
   },
   actions: {
     clear() {
-      this.useNotesCollab().disconnect();
       this.data = null;
     },
     async fetchById(shareId: string) {
diff --git a/frontend/src/utils/collab.ts b/frontend/src/utils/collab.ts
index 380adc9ab..7e1d45394 100644
--- a/frontend/src/utils/collab.ts
+++ b/frontend/src/utils/collab.ts
@@ -86,7 +86,6 @@ export type AwarenessInfos = {
     selection?: EditorSelection;
   };
   other: Record<string, {
-    client_id: string;
     path: string|null;
     selection?: EditorSelection;
   }>;
@@ -538,7 +537,6 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
       } else if (msgData.type === CollabEventType.AWARENESS) {
         if (msgData.client_id !== storeState.clientID) {
           storeState.awareness.other[msgData.client_id!] = {
-            client_id: msgData.client_id!,
             path: msgData.path,
             selection: msgData.path ? parseSelection({
               selectionJson: msgData.selection, 
@@ -894,8 +892,8 @@ export function useCollab<T = any>(storeState: CollabStoreState<T>) {
       });
       
       // Update remote selections
-      for (const a of Object.values(storeState.awareness.other)) {
-        if (a.client_id === event.client_id) {
+      for (const [clientId, a] of Object.entries(storeState.awareness.other)) {
+        if (event.client_id === clientId) {
           a.path = event.path;
           if (event.selection) {
             a.selection = parseSelection({
diff --git a/frontend/test/collab.test.ts b/frontend/test/collab.test.ts
index 5edcf43f0..92b5d411a 100644
--- a/frontend/test/collab.test.ts
+++ b/frontend/test/collab.test.ts
@@ -2,7 +2,7 @@ import { describe, test, expect, beforeEach, vi } from 'vitest'
 import { setActivePinia, createPinia } from 'pinia'
 import { v4 as uuid4 } from 'uuid'
 import { cloneDeep } from 'lodash-es'
-import { CollabEventType, type CollabEvent, type User } from '#imports';
+import { CollabEventType, type CollabEvent, type Comment, type User } from '#imports';
 import { ChangeSet, EditorSelection } from 'reportcreator-markdown/editor';
 
 
@@ -29,6 +29,8 @@ async function createCollab(options?: { collabInitEvent?: Partial<CollabEvent> }
   }
   await collab.connectTo(connection);
 
+  const commentIdText = uuid4();
+  const commentIdList = uuid4();
   const collabInitEvent = {
     type: CollabEventType.INIT,
     version: 1,
@@ -42,21 +44,21 @@ async function createCollab(options?: { collabInitEvent?: Partial<CollabEvent> }
       field_text: 'ABCD',
       field_key: 'value',
       field_list: ['a', 'b', 'c'],
-    },
-    comments: [
-      {
-        id: uuid4(),
-        text: 'comment on field_text',
-        path: 'field_text',
-        text_range: { from: 1, to: 2 }, // 'BC'
+      comments: {
+        [commentIdText]: {
+          id: commentIdText,
+          text: 'comment on field_text',
+          path: 'field_text',
+          text_range: { from: 1, to: 2 }, // 'BC'
+        },
+        [commentIdList]: {
+          id: commentIdList,
+          text: 'comment on field_list[0]',
+          path: 'field_list.[0]',
+          text_range: { from: 0, to: 1 }, // 'a'
+        }
       },
-      {
-        id: uuid4(),
-        text: 'comment on field_list[0]',
-        path: 'field_list.[0]',
-        text_range: { from: 0, to: 1 }, // 'a'
-      }
-    ],
+    },
     clients: [
       {
         client_id: 'self',
@@ -73,6 +75,11 @@ async function createCollab(options?: { collabInitEvent?: Partial<CollabEvent> }
   }
   connection.receive(cloneDeep(collabInitEvent));
 
+  collab.storeState.awareness.other['other'] = {
+    path: 'field_text',
+    selection: EditorSelection.create([EditorSelection.range(1, 2)]),
+  }
+
   // Reset mocks
   await vi.runOnlyPendingTimersAsync();
   vi.clearAllMocks();
@@ -133,7 +140,7 @@ describe('connection', () => {
   });
 
   test('disconnect on error', async () => {
-    collab.onReceiveMessage(createReceivedEvent({
+    connection.receive(createReceivedEvent({
       type: CollabEventType.UPDATE_TEXT,
       path: 'field_text',
       updates: [{ changes: [10, [0, 'a']]}],  // Invalid update
@@ -191,13 +198,13 @@ describe('send and receive', () => {
 
   test('version updated', () => {
     const newVersion = 1234;
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.UPDATE_KEY, version: newVersion}));
+    connection.receive(createReceivedEvent({type: CollabEventType.UPDATE_KEY, version: newVersion}));
     expect(collab.storeState.version).toBe(newVersion);
   });
 
   test('collab.update_key', () => {
     const newValue = 'changed value';
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.UPDATE_KEY, path: 'field_key', value: newValue}));
+    connection.receive(createReceivedEvent({type: CollabEventType.UPDATE_KEY, path: 'field_key', value: newValue}));
     expect(collab.storeState.data.field_key).toBe(newValue);
   });
 
@@ -213,7 +220,7 @@ describe('send and receive', () => {
       path: collab.storeState.apiPath + 'field_list.[1]',
       updates: [{ changes: ChangeSet.fromJSON([1, [0, 'y']]) }],
     });
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.UPDATE_KEY, path: 'field_list', value: newValue}));
+    connection.receive(createReceivedEvent({type: CollabEventType.UPDATE_KEY, path: 'field_list', value: newValue}));
     await vi.runOnlyPendingTimersAsync();
     expect(collab.storeState.data.field_list).toEqual(newValue);
     expect(connection.send).not.toHaveBeenCalled();
@@ -221,24 +228,24 @@ describe('send and receive', () => {
 
   test('collab.create', () => {
     const newValue = 'new value';
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.CREATE, path: 'field_new', value: newValue}));
+    connection.receive(createReceivedEvent({type: CollabEventType.CREATE, path: 'field_new', value: newValue}));
     expect(collab.storeState.data.field_new).toBe(newValue);
   });
 
   test('collab.delete', () => {
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.DELETE, path: 'field_text'}));
+    connection.receive(createReceivedEvent({type: CollabEventType.DELETE, path: 'field_text'}));
     expect(collab.storeState.data.field_text).toBeUndefined();
   });
 
   test('collab.create list', () => {
     const newValue = 'new value';
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.CREATE, path: 'field_list.[3]', value: newValue}));
+    connection.receive(createReceivedEvent({type: CollabEventType.CREATE, path: 'field_list.[3]', value: newValue}));
     expect(collab.storeState.data.field_list[3]).toBe(newValue);
     expect(collab.storeState.data.field_list.length).toBe(4);
   });
 
   test('collab.delete list', () => {
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.DELETE, path: 'field_list.[1]'}));
+    connection.receive(createReceivedEvent({type: CollabEventType.DELETE, path: 'field_list.[1]'}));
     const newList = collabInitEvent.data.field_list;
     newList.splice(1, 1);
     expect(collab.storeState.data.field_list.length).toBe(newList.length);
@@ -246,7 +253,7 @@ describe('send and receive', () => {
   });
 
   test('collab.connect', () => {
-    collab.onReceiveMessage(createReceivedEvent({
+    connection.receive(createReceivedEvent({
       type: CollabEventType.CONNECT,
       client_id: 'other2',
       client: {
@@ -259,13 +266,13 @@ describe('send and receive', () => {
   });
 
   test('collab.disconnect', () => {
-    collab.onReceiveMessage(createReceivedEvent({type: CollabEventType.DISCONNECT, client_id: 'other'}));
+    connection.receive(createReceivedEvent({type: CollabEventType.DISCONNECT, client_id: 'other'}));
     expect(collab.storeState.awareness.clients.length).toBe(1);
   })
 
   test('collab.awareness', () => {
     const selection = EditorSelection.create([EditorSelection.range(0, 1)]).toJSON();
-    collab.onReceiveMessage(createReceivedEvent({
+    connection.receive(createReceivedEvent({
       type: CollabEventType.AWARENESS,
       client_id: 'other',
       path: 'field_text',
@@ -312,12 +319,11 @@ describe('collab.awareness', () => {
   });
 
   test('collab.update_text with selection: collab.awareness not sent', () => {
-    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text'});
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text', selection: EditorSelection.create([EditorSelection.cursor(0)])});
     collab.onCollabEvent({
       type: CollabEventType.UPDATE_TEXT,
       path: collab.storeState.apiPath + 'field_text',
       updates: [{ changes: ChangeSet.fromJSON([4, [0, 'E']]) }],
-      selection: EditorSelection.create([EditorSelection.cursor(0)]),
     });
     const selection = EditorSelection.create([EditorSelection.range(0, 2)]);
     collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text', selection})
@@ -329,8 +335,8 @@ describe('collab.awareness', () => {
       type: CollabEventType.UPDATE_TEXT,
       path: collab.storeState.apiPath + 'field_text',
       updates: [{ changes: ChangeSet.fromJSON([4, [0, 'E']]) }],
-      selection: EditorSelection.create([EditorSelection.cursor(1)]),
     });
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text', Selection: EditorSelection.create([EditorSelection.cursor(1)])});
     collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_key'});
     expectEventsSent(
       {type: CollabEventType.UPDATE_TEXT, path: 'field_text', selection: undefined},
@@ -379,71 +385,153 @@ describe('collab.awareness', () => {
 });
 
 
-// describe('collab.update_text', () => {
-//   let { collab }: Awaited<ReturnType<typeof createCollab>> = {} as any;
-//   beforeEach(async () => {
-//     const res = await createCollab();
-//     collab = res.collab;
-//   });
-
-  
-// });
-
-
-// TODO: collab tests
-// * [x] refactor
-//   * [x] connection
-//     * [x] connect, disconnect, connectionInfo, onReceiveMessage
-//     * [x] connectionConfig: throttle timer
-//     * [x] send(events: CollabEvent[])
-//   * [x] WSConnection
-//     * [x] send: send all events immediately, no throttle
-//   * [x] HTTPConnection
-//     * [x] timer: fetch events every 10s
-//     * [x] reset timer on send
-//   * [x] useCollab:
-//     * [x] timer with connection.connectionConfig.throttle_interval
-//     * [x] 1 timer for all fields, remove per-field timers
-//     * [x] send events to connection in batch
-//     * [x] throttle update_text, update_key, awareness
-//     * [x] do not throttle other events
-//   * [x] useCollab: moved from WSConnection
-//     * [x] before connect: stop timer, reset pending events
-//     * [x] on disconnect: stop timer, flush pending events
-//     * [x] on receive CollabEventType.INIT: sendAwarenessThrottled
-//     * [x] on receive CollabEventType.CONNECT: sendAwarenessThrottled
-// * [ ] tests
-//   * [ ] connection
-//     * [x] send not called when connectionState != connected
-//     * [ ] test connection timeout detection: no answer received from server => disconnect called
-//   * [ ] collab receive
-//     * [x] events received: version updated
-//     * [ ] test update_text received: rebase positions for text + selection + comments
-//     * [x] test update_key received: clear pending events of field + child fields
-//     * [x] test create received
-//     * [x] test delete received
-//     * [x] text receive list operations
-//   * [ ] awareness
-//     * [x] update_text event with selection: send + clear pending collab.awareness events
-//     * [x] awareness event: send + clear pending collab.awareness events
-//     * [x] send update_key with update_awareness=True: collab.awareness not sent
-//     * [x] send update_key with update_awareness=True, other field focussed:  collab.awareness sent
-//     * [x] send update_key with update_awareness=False: update_awareness sent
-//     * [x] send update_text with selection, other field focussed: update_text.selection=undefined, update_awareness sent
-//     * [ ] update selection on receive collab.update_text
-//     * [ ] update selection on send collab.update_text
-//     * [ ] rebase selection on receive collab.awareness onto unconfirmedTextUpdates
-//   * [ ] collab send
-//     * [x] multiple update_text events on same field: combine
-//     * [x] multiple update_key events on same field: only send last
-//     * [x] update_text child, then update_key parent: only send update_key
-//   * [ ] collab.update_text
-//     * [ ] test rebase on receive remote events 
-//     * [ ] test rebase on send onto unconfirmedTextUpdates
-//     * [ ] test mark text updates as confirmed on receive remote events
-//   * [ ] comments
-//     * [ ] update according to event.comments
-//     * [ ] update comments position on receive collab.update_text
-//     * [ ] update comments position on send collab.update_text
-//     * [ ] rebase comments position on receive collab.awareness onto unconfirmedTextUpdates
+describe('collab.update_text', () => {
+  let { collab, connection, collabInitEvent }: Awaited<ReturnType<typeof createCollab>> = {} as any;
+  beforeEach(async () => {
+    const res = await createCollab();
+    collab = res.collab;
+    connection = res.connection;
+    collabInitEvent = res.collabInitEvent;
+  });
+
+  test('send', () => {
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: collab.storeState.apiPath + 'field_text',
+      updates: [{ changes: ChangeSet.fromJSON([4, [0, 'E']]) }],
+    });
+    const selection = EditorSelection.create([EditorSelection.range(0, 1)]);
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: collab.storeState.apiPath + 'field_text',
+      updates: [{ changes: ChangeSet.fromJSON([0, [0, '0'], 5]) }],
+    });
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text', selection});
+
+    // Text updated
+    expect(collab.storeState.data.field_text).toBe('0ABCDE');
+
+    // Awareness of self updated
+    expect(collab.storeState.awareness.self).toEqual({
+      path: 'field_text',
+      selection,
+    });
+
+    // Selection position of other users updated
+    expect(collab.storeState.awareness.other['other']).toEqual({
+      path: 'field_text',
+      selection: EditorSelection.create([EditorSelection.range(2, 3)]),
+    });
+
+    // Comment positions updated
+    const commentId = Object.values(collabInitEvent.data.comments as Record<string, Partial<Comment>>).find(c => c.path === 'field_text')!.id!;
+    expect(collab.storeState.data.comments[commentId].text_range).toEqual(EditorSelection.range(2, 3));
+  });
+
+  test('receive: no unconfirmedTextUpdates', () => {
+    collab.storeState.awareness.self = {
+      path: 'field_text',
+      selection: EditorSelection.create([EditorSelection.range(1, 2)]),
+    }
+
+    const selection = EditorSelection.create([EditorSelection.range(0, 1)]);
+    const commentId = Object.values(collabInitEvent.data.comments as Record<string, Partial<Comment>>).find(c => c.path === 'field_text')!.id;
+    connection.receive(createReceivedEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      client_id: 'other',
+      path: 'field_text',
+      updates: [{ changes: [0, [0, '0'], 4] }],
+      selection: selection.toJSON(),
+      comments: [{ id: commentId, path: 'field_text', text_range: { from: 2, to: 3 }}],
+    }));
+
+    // Text updated
+    expect(collab.storeState.data.field_text).toBe('0ABCD');
+
+    // Awareness of other user updated
+    expect(collab.storeState.awareness.other['other']).toEqual({
+      path: 'field_text',
+      selection,
+    });
+
+    // Selection position of self updated
+    expect(collab.storeState.awareness.self).toEqual({
+      path: 'field_text',
+      selection: EditorSelection.create([EditorSelection.range(2, 3)])
+    });
+
+    // Comment positions updated
+    const commentActual = Object.values(collab.storeState.data.comments as Record<string, Partial<Comment>>).find(c => c.id === commentId)!;
+    expect(commentActual.path).toBe('field_text');
+    expect(commentActual.text_range).toEqual(EditorSelection.range(2, 3));
+  });
+
+  test('receive: rebase onto unconfirmedTextUpdates', () => {
+    const selection = EditorSelection.create([EditorSelection.range(0, 1)]);
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: collab.storeState.apiPath + 'field_text',
+      updates: [{ changes: ChangeSet.fromJSON([0, [0, '0'], 4]) }],
+    });
+    collab.onCollabEvent({type: CollabEventType.AWARENESS, path: collab.storeState.apiPath + 'field_text', selection});
+    const commentId = Object.values(collabInitEvent.data.comments as Record<string, Partial<Comment>>).find(c => c.path === 'field_text')!.id!;
+    connection.receive(createReceivedEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      client_id: 'other',
+      path: 'field_text',
+      updates: [{ changes: [4, [0, 'E']] }],
+      selection: EditorSelection.create([EditorSelection.range(1, 2)]).toJSON(),
+    }));
+
+    // Text updated
+    expect(collab.storeState.data.field_text).toBe('0ABCDE');
+
+    // Awareness of self updated
+    expect(collab.storeState.awareness.self).toEqual({
+      path: 'field_text',
+      selection,
+    });
+
+    // Selection position of other users updated: rebased onto unconfirmedTextUpdates
+    expect(collab.storeState.awareness.other['other']).toEqual({
+      path: 'field_text',
+      selection: EditorSelection.create([EditorSelection.range(2, 3)]),
+    });
+
+    // Comment positions updated: rebased onto unconfirmedTextUpdates
+    const commentActual = collab.storeState.data.comments[commentId];
+    expect(commentActual.path).toBe('field_text');
+    expect(commentActual.text_range).toEqual(EditorSelection.range(2, 3));
+
+    // Rebase collab.awareness
+    connection.receive({
+      type: CollabEventType.AWARENESS,
+      client_id: 'other',
+      path: 'field_text',
+      selection: EditorSelection.create([EditorSelection.cursor(1)]).toJSON(),
+    });
+    expect(collab.storeState.awareness.other['other']).toEqual({
+      path: 'field_text',
+      selection: EditorSelection.create([EditorSelection.cursor(2)]),
+    });
+  });
+
+  test('receive: confirm unconfirmedTextUpdates', () => {
+    const updates = [{ changes: ChangeSet.fromJSON([4, [0, 'E']]) }];
+    collab.onCollabEvent({
+      type: CollabEventType.UPDATE_TEXT,
+      path: collab.storeState.apiPath + 'field_text',
+      updates,
+    });
+    expect(collab.storeState.perPathState.get('field_text')!.unconfirmedTextUpdates).toEqual(updates);
+
+    connection.receive({
+      type: CollabEventType.UPDATE_TEXT,
+      client_id: 'self',
+      path: 'field_text',
+      updates: updates.map(u => ({ changes: u.changes.toJSON() })),
+    });
+    expect(collab.storeState.perPathState.get('field_text')!.unconfirmedTextUpdates).toEqual([]);
+  });
+});