Provider-level offline support (#354)

This builds on top of #352 to add offline support to the Y-Sweet
provider. It's also exposed through the React bindings by passing
`enableOfflineSupport={false,true}` to `YDocProvider`.

The approach is similar to `y-indexeddb`, where updates are stored
incrementally and then compacted when the set of updates gets large.

Browsers generally store indexeddb in plain text on disk. We encrypt the
data using `AES-GCM`, and store the key in a cookie. Since cookies often
store credentials, they are often treated more sensitively than local
storage (Chrome encrypts them on disk, and stores the decryption key in
the OS's keychain; Firefox and Safari still use plaintext.)

Each document gets its own "database" in indexeddb, but the key is
shared across the origin.

Since multiple tabs can have the same document open, they attempt to
coordinate:
- Before writing to the database, we attempt to fetch any updates we
haven't seen and apply them
- When a client updates, it broadcasts a message telling other clients
to update

examples/nextjs/src/app/(demos)/blocknote/page.tsx

@@ -5,7 +5,7 @@ import { BlockNote } from './BlockNote'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <BlockNote />
     </YDocProvider>
   )

examples/nextjs/src/app/(demos)/code-editor/page.tsx

@@ -5,7 +5,7 @@ import { CodeEditor } from './CodeEditor'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <CodeEditor />
     </YDocProvider>
   )

examples/nextjs/src/app/(demos)/color-grid/page.tsx

@@ -6,7 +6,7 @@ import StateIndicator from '@/components/StateIndicator'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <div className="p-4 lg:p-8">
         <StateIndicator />
         <ColorGrid />

examples/nextjs/src/app/(demos)/presence/page.tsx

@@ -5,7 +5,7 @@ import { Presence } from './Presence'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <Presence />
     </YDocProvider>
   )

examples/nextjs/src/app/(demos)/slate/page.tsx

@@ -5,7 +5,7 @@ import { SlateEditor } from './SlateEditor'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <SlateEditor />
     </YDocProvider>
   )

examples/nextjs/src/app/(demos)/text-editor/page.tsx

@@ -5,7 +5,7 @@ import { TextEditor } from './TextEditor'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <TextEditor />
     </YDocProvider>
   )

examples/nextjs/src/app/(demos)/to-do-list/page.tsx

@@ -5,7 +5,7 @@ import { ToDoList } from './ToDoList'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <ToDoList />
     </YDocProvider>
   )

examples/nextjs/src/app/(demos)/tree-crdt/page.tsx

@@ -5,7 +5,7 @@ import { TreeView } from './TreeView'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <TreeView />
     </YDocProvider>
   )

examples/nextjs/src/app/(demos)/voxels/page.tsx

@@ -5,7 +5,7 @@ import { VoxelEditor } from './VoxelEditor'
 export default function Home({ searchParams }: { searchParams: { doc: string } }) {
   const docId = searchParams.doc ?? randomId()
   return (
-    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth">
+    <YDocProvider docId={docId} setQueryParam="doc" authEndpoint="/api/auth" offlineSupport={true}>
       <VoxelEditor />
     </YDocProvider>
   )

js-pkg/client/src/encryption.ts

@@ -0,0 +1,48 @@
+const ALGORITHM: AesKeyGenParams = {
+  name: 'AES-GCM',
+  length: 256,
+}
+const KEY_USAGE: KeyUsage[] = ['encrypt', 'decrypt']
+
+const NONCE_LENGTH = 12
+
+function arrayBufferToBase64(buffer: ArrayBuffer): string {
+  return btoa(String.fromCharCode(...new Uint8Array(buffer)))
+}
+
+function base64ToArrayBuffer(base64: string): ArrayBuffer {
+  const binaryString = atob(base64)
+  const bytes = Uint8Array.from(binaryString, (c) => c.charCodeAt(0))
+  return bytes.buffer
+}
+
+export async function importKey(key: string): Promise<CryptoKey> {
+  const rawKey = base64ToArrayBuffer(key)
+  return await crypto.subtle.importKey('raw', rawKey, ALGORITHM, true, KEY_USAGE)
+}
+
+export async function exportKey(key: CryptoKey): Promise<string> {
+  const rawKey = await crypto.subtle.exportKey('raw', key)
+  return arrayBufferToBase64(rawKey)
+}
+
+export function generateEncryptionKey(): Promise<CryptoKey> {
+  return crypto.subtle.generateKey(ALGORITHM, true, KEY_USAGE)
+}
+
+export async function encryptData(data: Uint8Array, key: CryptoKey): Promise<Uint8Array> {
+  const iv = crypto.getRandomValues(new Uint8Array(NONCE_LENGTH))
+  const encrypted = await crypto.subtle.encrypt({ name: ALGORITHM.name, iv }, key, data)
+
+  const result = new Uint8Array(iv.length + encrypted.byteLength)
+  result.set(iv)
+  result.set(new Uint8Array(encrypted), iv.length)
+  return result
+}
+
+export async function decryptData(encryptedData: Uint8Array, key: CryptoKey): Promise<Uint8Array> {
+  const iv = encryptedData.slice(0, NONCE_LENGTH)
+  const data = encryptedData.slice(NONCE_LENGTH)
+  const decrypted = await crypto.subtle.decrypt({ name: ALGORITHM.name, iv }, key, data)
+  return new Uint8Array(decrypted)
+}

js-pkg/client/src/indexeddb.ts

@@ -0,0 +1,206 @@
+import { Doc } from 'yjs'
+import * as Y from 'yjs'
+import { getOrCreateKey } from './keystore'
+import { decryptData, encryptData } from './encryption'
+
+const DB_PREFIX = 'y-sweet-'
+const OBJECT_STORE_NAME = 'updates'
+
+/**
+ * Maximum number of independent updates to store in IndexedDB.
+ * If this is exceeded, all of the updates are compacted into one.
+ */
+const MAX_UPDATES_IN_STORE = 50
+
+/** Pair of key and value, used both for the encrypted entry and decrypted entry. */
+interface BytesWithKey {
+  key: number
+  value: Uint8Array
+}
+
+function openIndexedDB(name: string): Promise<IDBDatabase> {
+  return new Promise<IDBDatabase>((resolve, reject) => {
+    const request = indexedDB.open(name, 4)
+    request.onupgradeneeded = () => {
+      const db = request.result
+      db.createObjectStore(OBJECT_STORE_NAME, { keyPath: 'key' })
+    }
+    request.onsuccess = () => resolve(request.result)
+    request.onerror = reject
+  })
+}
+
+export async function createIndexedDBProvider(doc: Doc, docId: string): Promise<IndexedDBProvider> {
+  const db = await openIndexedDB(DB_PREFIX + docId)
+  const encryptionKey = await getOrCreateKey()
+
+  let provider = new IndexedDBProvider(doc, docId, db, encryptionKey)
+  return provider
+}
+
+export class IndexedDBProvider {
+  lastUpdateKey: number = -1
+  broadcastChannel: BroadcastChannel
+
+  constructor(
+    private doc: Doc,
+    docId: string,
+    private db: IDBDatabase,
+    private encryptionKey: CryptoKey,
+  ) {
+    this.handleUpdate = this.handleUpdate.bind(this)
+    doc.on('update', this.handleUpdate)
+
+    this.broadcastChannel = new BroadcastChannel(`y-sweet-${docId}`)
+
+    this.broadcastChannel.onmessage = (event) => {
+      if (event.data > this.lastUpdateKey) {
+        this.loadFromDb()
+      }
+    }
+
+    this.loadFromDb()
+  }
+
+  private updateKey() {
+    this.lastUpdateKey += 1
+    return this.lastUpdateKey
+  }
+
+  async loadFromDb() {
+    let range = IDBKeyRange.lowerBound(this.lastUpdateKey, true)
+    const updates = await this.getAllValues(range)
+
+    this.doc.transact(() => {
+      for (const update of updates) {
+        Y.applyUpdate(this.doc, update.value)
+
+        this.lastUpdateKey = update.key
+      }
+    }, this)
+  }
+
+  destroy() {
+    this.doc.off('update', this.handleUpdate)
+    this.broadcastChannel.close()
+  }
+
+  async handleUpdate(update: Uint8Array, origin: any) {
+    if (origin === this) {
+      return
+    }
+
+    // We attempt to write in a loop. If we are preempted by another writer, we load the latest
+    // updates and try again.
+    while (true) {
+      await this.loadFromDb()
+      let key = this.updateKey()
+      let newCount = await this.insertValue(key, update)
+
+      if (newCount === null) {
+        // Another writer wrote before we could; reload and try again.
+        continue
+      }
+
+      if (newCount > MAX_UPDATES_IN_STORE) {
+        key = this.updateKey()
+        if (!(await this.saveWholeState(key))) {
+          // Another writer wrote before we could; reload and try again.
+          continue
+        }
+      }
+
+      break
+    }
+
+    this.broadcastChannel.postMessage(this.lastUpdateKey)
+  }
+
+  async getAllValues(range?: IDBKeyRange): Promise<Array<BytesWithKey>> {
+    let transaction = this.db.transaction(OBJECT_STORE_NAME)
+    let objectStore = transaction.objectStore(OBJECT_STORE_NAME)
+    const request = objectStore.getAll(range)
+
+    let result = await new Promise<Array<BytesWithKey>>((resolve, reject) => {
+      request.onsuccess = () => {
+        resolve(request.result)
+      }
+      request.onerror = reject
+    })
+
+    return await Promise.all(
+      result.map(async (data) => {
+        let value = await decryptData(data.value, this.encryptionKey)
+
+        return {
+          key: data.key,
+          value,
+        }
+      }),
+    )
+  }
+
+  async saveWholeState(key: number): Promise<boolean> {
+    const update = Y.encodeStateAsUpdate(this.doc)
+    const encryptedUpdate = await encryptData(update, this.encryptionKey)
+    let transaction = this.db.transaction(OBJECT_STORE_NAME, 'readwrite')
+    let objectStore = transaction.objectStore(OBJECT_STORE_NAME)
+
+    if (await this.hasValue(objectStore, key)) {
+      return false
+    }
+
+    let range = IDBKeyRange.upperBound(key, false)
+    objectStore.delete(range)
+
+    objectStore.add({
+      key,
+      value: encryptedUpdate,
+    })
+
+    return true
+  }
+
+  async hasValue(objectStore: IDBObjectStore, key: number): Promise<boolean> {
+    const request = objectStore.get(key)
+    await new Promise<void>((resolve, reject) => {
+      request.onsuccess = () => resolve()
+      request.onerror = reject
+    })
+
+    return request.result !== undefined
+  }
+
+  /**
+   * Insert a value into IndexedDB. Return the new count of updates in the store if the value was inserted,
+   * or null if the desired key already exists.
+   **/
+  async insertValue(key: number, value: Uint8Array): Promise<number | null> {
+    const encryptedValue = await encryptData(value, this.encryptionKey)
+    let objectStore = this.db
+      .transaction(OBJECT_STORE_NAME, 'readwrite')
+      .objectStore(OBJECT_STORE_NAME)
+
+    if (await this.hasValue(objectStore, key)) {
+      return null
+    }
+
+    const request = objectStore.put({
+      key,
+      value: encryptedValue,
+    })
+
+    await new Promise<void>((resolve, reject) => {
+      request.onsuccess = () => resolve()
+      request.onerror = reject
+    })
+
+    let countRequest = objectStore.count()
+    let count = await new Promise<number>((resolve, reject) => {
+      countRequest.onsuccess = () => resolve(countRequest.result)
+      countRequest.onerror = reject
+    })
+
+    return count
+  }
+}

js-pkg/client/src/keystore.ts

@@ -0,0 +1,29 @@
+import { exportKey, generateEncryptionKey, importKey } from './encryption'
+
+const COOKIE_NAME = 'YSWEET_OFFLINE_KEY'
+
+function getCookie(name: string): string | null {
+  const value = `; ${document.cookie}`
+  const parts = value.split(`; ${name}=`)
+  if (parts.length === 2) return parts.pop()?.split(';').shift() || null
+  return null
+}
+
+function setCookie(name: string, value: string) {
+  document.cookie = `${name}=${value};path=/;expires=Fri, 31 Dec 9999 23:59:59 GMT;secure`
+}
+
+export async function getOrCreateKey(): Promise<CryptoKey> {
+  // Check for existing key in cookie
+  const cookieKey = getCookie(COOKIE_NAME)
+  if (cookieKey) {
+    // Use existing key from cookie
+    return await importKey(cookieKey)
+  } else {
+    // Generate new key and store in cookie
+    const rawKey = await generateEncryptionKey()
+    const key = await exportKey(rawKey)
+    setCookie(COOKIE_NAME, key)
+    return rawKey
+  }
+}