diff --git a/examples/document/.gitignore b/examples/document/.gitignore new file mode 100644 index 000000000..8fce60300 --- /dev/null +++ b/examples/document/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/examples/document/README.md b/examples/document/README.md new file mode 100644 index 000000000..a8e358c81 --- /dev/null +++ b/examples/document/README.md @@ -0,0 +1,54 @@ +# Document Editor Example + +[Github](https://github.com/canvasxyz/canvas/tree/main/examples/document) + +This example app demonstrates collaborative document editing using the CRDT provided by [Yjs](https://github.com/yjs/yjs). It allows users to collaboratively edit a single text document with persistence over libp2p. + +```ts +export const models = { + documents: { + id: "primary", + content: "yjs-doc", + }, +} + +export const actions = { + async applyDeltaToDoc(db, index, text) { + await db.ytext.applyDelta("documents", "0", index, text) + }, +} +``` + +## Server + +Run `npm run dev:server` to start a temporary in-memory server, or +`npm run start:server` to persist data to a `.cache` directory. + +To deploy the replication server: + +``` +$ cd server +$ fly deploy +``` + +If you are forking this example, you should change: + +- the Fly app name +- the `ANNOUNCE` environment variable to match your Fly app name + +## Running the Docker container locally + +Mount a volume to `/data`. Set the `PORT`, `LISTEN`, `ANNOUNCE`, and +`BOOTSTRAP_LIST` environment variables if appropriate. + +## Deploying to Railway + +Create a Railway space based on the root of this Github workspace (e.g. canvasxyz/canvas). + +- Custom build command: `npm run build && VITE_CANVAS_WS_URL=wss://chat-example.canvas.xyz npm run build --workspace=@canvas-js/example-chat` +- Custom start command: `./install-prod.sh && canvas run /tmp/canvas-example-chat --port 8080 --static examples/chat/dist --topic chat-example.canvas.xyz --init examples/chat/src/contract.ts` +- Watch paths: `/examples/chat/**` +- Public networking: + - Add a service domain for port 8080. + - Add a service domain for port 4444. +- Watch path: `/examples/chat/**`. (Only build when chat code is updated, or a chat package is updated.) diff --git a/examples/document/index.html b/examples/document/index.html new file mode 100644 index 000000000..0aa3300d3 --- /dev/null +++ b/examples/document/index.html @@ -0,0 +1,31 @@ + + + + + + + + Canvas Document Editor + + + + + + +
+ + + diff --git a/examples/document/package.json b/examples/document/package.json new file mode 100644 index 000000000..1769190b2 --- /dev/null +++ b/examples/document/package.json @@ -0,0 +1,51 @@ +{ + "private": true, + "name": "@canvas-js/document", + "version": "0.14.0-next.1", + "type": "module", + "scripts": { + "build": "npx vite build", + "dev": "npx vite", + "dev:server": "canvas run --port 8080 --static dist --network-explorer --topic document-example.canvas.xyz src/contract.ts" + }, + "dependencies": { + "@canvas-js/chain-atp": "0.14.0-next.1", + "@canvas-js/chain-cosmos": "0.14.0-next.1", + "@canvas-js/chain-ethereum": "0.14.0-next.1", + "@canvas-js/chain-ethereum-viem": "0.14.0-next.1", + "@canvas-js/chain-solana": "0.14.0-next.1", + "@canvas-js/chain-substrate": "0.14.0-next.1", + "@canvas-js/cli": "0.14.0-next.1", + "@canvas-js/core": "0.14.0-next.1", + "@canvas-js/gossiplog": "0.14.0-next.1", + "@canvas-js/hooks": "0.14.0-next.1", + "@canvas-js/interfaces": "0.14.0-next.1", + "@canvas-js/modeldb-sqlite-wasm": "0.14.0-next.1", + "@farcaster/auth-kit": "^0.6.0", + "@farcaster/frame-sdk": "^0.0.26", + "@libp2p/interface": "^2.7.0", + "@multiformats/multiaddr": "^12.3.5", + "@noble/hashes": "^1.7.1", + "@types/react": "^18.3.9", + "@types/react-dom": "^18.3.0", + "buffer": "^6.0.3", + "comlink": "^4.4.1", + "ethers": "^6.13.5", + "idb": "^8.0.2", + "multiformats": "^13.3.2", + "near-api-js": "^2.1.4", + "process": "^0.11.10", + "quill": "^2.0.3", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "vite-plugin-wasm": "^3.3.0" + }, + "devDependencies": { + "autoprefixer": "^10.4.19", + "postcss": "^8.4.38", + "tailwindcss": "^3.4.4", + "typescript": "~5.6.0", + "vite": "^5.4.8", + "vite-plugin-node-polyfills": "^0.22.0" + } +} diff --git a/examples/document/postcss.config.js b/examples/document/postcss.config.js new file mode 100644 index 000000000..1a5262473 --- /dev/null +++ b/examples/document/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/examples/document/src/App.tsx b/examples/document/src/App.tsx new file mode 100644 index 000000000..15e78d1d7 --- /dev/null +++ b/examples/document/src/App.tsx @@ -0,0 +1,99 @@ +import React, { useRef, useState } from "react" + +import type { SessionSigner } from "@canvas-js/interfaces" +import { Eip712Signer, SIWESigner, SIWFSigner } from "@canvas-js/chain-ethereum" +import { ATPSigner } from "@canvas-js/chain-atp" +import { CosmosSigner } from "@canvas-js/chain-cosmos" +import { SolanaSigner } from "@canvas-js/chain-solana" +import { SubstrateSigner } from "@canvas-js/chain-substrate" + +import { useCanvas } from "@canvas-js/hooks" + +import { AuthKitProvider } from "@farcaster/auth-kit" +import { JsonRpcProvider } from "ethers" + +import { AppContext } from "./AppContext.js" +import { ControlPanel } from "./ControlPanel.js" +import { SessionStatus } from "./SessionStatus.js" +import { ConnectEIP712Burner } from "./connect/ConnectEIP712Burner.js" +import { ConnectionStatus } from "./ConnectionStatus.js" +import { LogStatus } from "./LogStatus.js" +import * as contract from "./contract.js" +import { Editor } from "./Editor.js" +import { useQuill } from "./useQuill.js" + +export const topic = "document-example.canvas.xyz" + +const wsURL = import.meta.env.VITE_CANVAS_WS_URL ?? null +console.log("websocket API URL:", wsURL) + +const config = { + // For a production app, replace this with an Optimism Mainnet + // RPC URL from a provider like Alchemy or Infura. + relay: "https://relay.farcaster.xyz", + rpcUrl: "https://mainnet.optimism.io", + domain: "document-example.canvas.xyz", + siweUri: "https://document-example.canvas.xyz", + provider: new JsonRpcProvider(undefined, 10), +} + +export const App: React.FC<{}> = ({}) => { + const [sessionSigner, setSessionSigner] = useState(null) + const [address, setAddress] = useState(null) + + const topicRef = useRef(topic) + + const { app, ws } = useCanvas(wsURL, { + topic: topicRef.current, + contract, + signers: [ + new SIWESigner(), + new Eip712Signer(), + new SIWFSigner(), + new ATPSigner(), + new CosmosSigner(), + new SubstrateSigner({}), + new SolanaSigner(), + ], + }) + + const quillRef = useQuill({ modelName: "documents", modelKey: "0", app }) + + return ( + + + {app && ws ? ( +
+
+
+ {}} + onTextChange={async (delta, oldContents, source) => { + if (source === "user") { + await app.actions.applyDeltaToDoc(JSON.stringify(delta)) + } + }} + /> +
+
+ + + + + +
+
+
+ ) : ( +
Connecting to {wsURL}...
+ )} + + + ) +} diff --git a/examples/document/src/AppContext.ts b/examples/document/src/AppContext.ts new file mode 100644 index 000000000..2e4ec4586 --- /dev/null +++ b/examples/document/src/AppContext.ts @@ -0,0 +1,28 @@ +import { createContext } from "react" + +import type { SessionSigner } from "@canvas-js/interfaces" +import { Canvas } from "@canvas-js/core" + +export type AppContext = { + app: Canvas | null + + address: string | null + setAddress: (address: string | null) => void + + sessionSigner: SessionSigner | null + setSessionSigner: (signer: SessionSigner | null) => void +} + +export const AppContext = createContext({ + app: null, + + address: null, + setAddress: (address: string | null) => { + throw new Error("AppContext.Provider not found") + }, + + sessionSigner: null, + setSessionSigner: (signer) => { + throw new Error("AppContext.Provider not found") + }, +}) diff --git a/examples/document/src/ConnectionStatus.tsx b/examples/document/src/ConnectionStatus.tsx new file mode 100644 index 000000000..08aba6381 --- /dev/null +++ b/examples/document/src/ConnectionStatus.tsx @@ -0,0 +1,90 @@ +import React, { useContext, useEffect, useState } from "react" + +import type { Canvas, NetworkClient } from "@canvas-js/core" + +import { AppContext } from "./AppContext.js" + +export interface ConnectionStatusProps { + topic: string + ws: NetworkClient +} + +export const ConnectionStatus: React.FC = ({ topic, ws }) => { + const { app } = useContext(AppContext) + + const [, setTick] = useState(0) + useEffect(() => { + const timer = setInterval(() => { + setTick(t => t + 1) + }, 1000) + return () => clearInterval(timer) + }, []) + + if (app === null) { + return null + } + + return ( +
+
+ Topic +
+
+ {topic} +
+ +
+
+ Connection +
+
+ {import.meta.env.VITE_CANVAS_WS_URL} + ({ws.isConnected() ? 'Connected' : 'Disconnected'}) +
+
+ ) +} + +interface ConnectionListProps { + app: Canvas +} + +// const ConnectionList: React.FC = ({ app }) => { +// const [peers, setPeers] = useState([]) + +// useEffect(() => { +// if (app === null) { +// return +// } + +// const handleConnectionOpen = ({ detail: { peer } }: CustomEvent<{ peer: string }>) => +// void setPeers((peers) => [...peers, peer]) + +// const handleConnectionClose = ({ detail: { peer } }: CustomEvent<{ peer: string }>) => +// void setPeers((peers) => peers.filter((id) => id !== peer)) + +// app.messageLog.addEventListener("connect", handleConnectionOpen) +// app.messageLog.addEventListener("disconnect", handleConnectionClose) + +// return () => { +// app.messageLog.removeEventListener("connect", handleConnectionOpen) +// app.messageLog.removeEventListener("disconnect", handleConnectionClose) +// } +// }, [app]) + +// if (peers.length === 0) { +// return
No connections
+// } else { +// return ( +//
    +// {peers.map((peer) => { +// return ( +//
  • +// {peer} +//
    • +// ) +// })} +//
+// ) +// } +// } diff --git a/examples/document/src/ControlPanel.tsx b/examples/document/src/ControlPanel.tsx new file mode 100644 index 000000000..4d2d8ecc9 --- /dev/null +++ b/examples/document/src/ControlPanel.tsx @@ -0,0 +1,120 @@ +import React, { useCallback, useContext, useState } from "react" +import { deleteDB } from "idb" +import { bytesToHex, randomBytes } from "@noble/hashes/utils" + +import { AppContext } from "./AppContext.js" + +export interface ControlPanelProps {} + +// this is used for debugging in development +// when the database schema changes between page loads +// @ts-ignore +window.deleteDB = deleteDB + +export const ControlPanel: React.FC = ({}) => { + const { app, sessionSigner } = useContext(AppContext) + + const [isStarted, setIsStarted] = useState(false) + + const start = useCallback(async () => { + if (app === null) { + return + } + + // try { + // await app.libp2p.start() + // setIsStarted(true) + // } catch (err) { + // console.error(err) + // } + }, [app]) + + const stop = useCallback(async () => { + if (app === null) { + return + } + + // try { + // await app.libp2p.stop() + // setIsStarted(false) + // } catch (err) { + // console.error(err) + // } + }, [app]) + + const clear = useCallback(async () => { + if (app === null) { + return + } + + await app.stop() + + console.log("deleting database") + await deleteDB(`canvas/v1/${app.topic}`, {}) + + console.log("clearing session signer data", sessionSigner) + await sessionSigner?.clear?.(app.topic) + + window.location.reload() + }, [app, sessionSigner]) + + const spam = useCallback(async () => { + if (app === null || sessionSigner === null) { + return + } + + for (let i = 0; i < 100; i++) { + const content = bytesToHex(randomBytes(8)) + await app.as(sessionSigner).createMessage(content) + } + }, [app, sessionSigner]) + + const button = `p-2 border rounded flex` + const disabled = `bg-gray-100 text-gray-500 hover:cursor-not-allowed` + const enabledGreen = `bg-green-100 active:bg-green-300 hover:cursor-pointer hover:bg-green-200` + const enabledRed = `bg-red-100 active:bg-red-300 hover:cursor-pointer hover:bg-red-200` + const enabledYellow = `bg-yellow-100 active:bg-yellow-300 hover:cursor-pointer hover:bg-yellow-200` + + if (app === null) { + return ( +
+ {/* */} + +
+ ) + } else if (isStarted) { + return ( +
+ {/* */} + +
+ ) + } else { + return ( +
+ {/* */} + +
+ ) + } +} diff --git a/examples/document/src/Editor.tsx b/examples/document/src/Editor.tsx new file mode 100644 index 000000000..6cf9648be --- /dev/null +++ b/examples/document/src/Editor.tsx @@ -0,0 +1,61 @@ +import { Canvas } from "@canvas-js/core" +import Quill, { EmitterSource } from "quill" +import "quill/dist/quill.snow.css" +import React, { forwardRef, useEffect, useLayoutEffect, useRef } from "react" + +type EditorProps = { + readOnly: boolean + defaultValue: any + onTextChange: (delta: any, oldDelta: any, source: EmitterSource) => void + onSelectionChange: (delta: any, oldDelta: any, source: EmitterSource) => void +} + +// Editor is an uncontrolled React component +export const Editor = forwardRef(({ readOnly, defaultValue, onTextChange, onSelectionChange }: EditorProps, ref) => { + const containerRef = useRef(null) + const defaultValueRef = useRef(defaultValue) + const onTextChangeRef = useRef(onTextChange) + const onSelectionChangeRef = useRef(onSelectionChange) + + useLayoutEffect(() => { + onTextChangeRef.current = onTextChange + onSelectionChangeRef.current = onSelectionChange + }) + + useEffect(() => { + // @ts-ignore + ref.current?.enable(!readOnly) + }, [ref, readOnly]) + + useEffect(() => { + const container = containerRef.current + if (!container || !ref) return + const editorContainer = container.appendChild(container.ownerDocument.createElement("div")) + const quill = new Quill(editorContainer, { + theme: "snow", + }) + + // @ts-ignore + ref.current = quill + + if (defaultValueRef.current) { + quill.setContents(defaultValueRef.current) + } + + quill.on(Quill.events.TEXT_CHANGE, (...args) => { + onTextChangeRef.current?.(...args) + }) + + quill.on(Quill.events.SELECTION_CHANGE, (...args) => { + onSelectionChangeRef.current?.(...args) + }) + + return () => { + // @ts-ignore + ref.current = undefined + container.innerHTML = "" + } + }, [ref]) + + return
+}) diff --git a/examples/document/src/LogStatus.tsx b/examples/document/src/LogStatus.tsx new file mode 100644 index 000000000..811350d24 --- /dev/null +++ b/examples/document/src/LogStatus.tsx @@ -0,0 +1,67 @@ +import React, { useContext, useEffect, useState } from "react" +import { bytesToHex } from "@noble/hashes/utils" + +import type { CanvasEvents } from "@canvas-js/core" +import { MessageId } from "@canvas-js/gossiplog" + +import { AppContext } from "./AppContext.js" + +export interface LogStatusProps {} + +export const LogStatus: React.FC = ({}) => { + const { app } = useContext(AppContext) + + const [root, setRoot] = useState(null) + const [heads, setHeads] = useState(null) + useEffect(() => { + if (app === null) { + return + } + + app.messageLog.tree.read((txn) => txn.getRoot()).then((root) => setRoot(`${root.level}:${bytesToHex(root.hash)}`)) + app.db + .getAll<{ id: string }>("$heads") + .then((records) => setHeads(records.map((record) => MessageId.encode(record.id)))) + + const handleCommit = ({ detail: { root, heads } }: CanvasEvents["commit"]) => { + const rootValue = `${root.level}:${bytesToHex(root.hash)}` + setRoot(rootValue) + setHeads(heads.map(MessageId.encode)) + } + + app.addEventListener("commit", handleCommit) + return () => app.removeEventListener("commit", handleCommit) + }, [app]) + + if (app === null) { + return null + } + + return ( +
+
+ Merkle root +
+
+ {root !== null ? {root} : none} +
+
+ Message heads +
+
+ {heads !== null ? ( +
    + {heads.map((head) => ( +
  • + {head.id} + (clock: {head.clock}) +
    • + ))} +
+ ) : ( + none + )} +
+
+ ) +} diff --git a/examples/document/src/SessionStatus.tsx b/examples/document/src/SessionStatus.tsx new file mode 100644 index 000000000..189484c19 --- /dev/null +++ b/examples/document/src/SessionStatus.tsx @@ -0,0 +1,85 @@ +import React, { useContext, useMemo } from "react" +import { useLiveQuery } from "@canvas-js/hooks" +import { DeriveModelTypes } from "@canvas-js/modeldb" + +import { AppContext } from "./AppContext.js" +import { AddressView } from "./components/AddressView.js" + +const sessionSchema = { + $sessions: { + message_id: "primary", + did: "string", + public_key: "string", + address: "string", + expiration: "integer?", + // $indexes: [["did"], ["public_key"]], + }, +} as const + +export interface SessionStatusProps {} + +export const SessionStatus: React.FC = ({}) => { + const { address } = useContext(AppContext) + if (address === null) { + return null + } + + return ( +
+
+ Address +
+
+ +
+
+
+ Sessions +
+ +
+ ) +} + +interface SessionListProps { + address: string +} + +const SessionList: React.FC = ({ address }) => { + const { app } = useContext(AppContext) + + const timestamp = useMemo(() => Date.now(), []) + + const results = useLiveQuery(app, "$sessions", { + where: { address, expiration: { gt: timestamp } }, + }) + + if (results === null) { + return null + } else if (results.length === 0) { + return
No sessions
+ } else { + return ( +
    + {results.map((session) => { + return ( +
  • +
    + {session.public_key} +
    + {session.expiration && session.expiration < Number.MAX_SAFE_INTEGER ? ( +
    + Expires {new Date(session.expiration).toLocaleString()} +
    + ) : ( +
    + No expiration +
    + )} +
    • + ) + })} +
+ ) + } +} diff --git a/examples/document/src/components/AddressView.tsx b/examples/document/src/components/AddressView.tsx new file mode 100644 index 000000000..e051a4a91 --- /dev/null +++ b/examples/document/src/components/AddressView.tsx @@ -0,0 +1,16 @@ +import React from "react" + +export interface AddressViewProps { + className?: string + address: string +} + +export const AddressView: React.FC = (props) => { + const className = props.className ?? "text-sm" + return ( + + {/* {props.address.slice(0, 6)}…{props.address.slice(-4)} */} + {props.address} + + ) +} diff --git a/examples/document/src/components/MultiaddrView.tsx b/examples/document/src/components/MultiaddrView.tsx new file mode 100644 index 000000000..30a7073ae --- /dev/null +++ b/examples/document/src/components/MultiaddrView.tsx @@ -0,0 +1,22 @@ +import React from "react" + +import type { PeerId } from "@libp2p/interface" +import type { Multiaddr } from "@multiformats/multiaddr" + +export interface MultiaddrViewProps { + addr: Multiaddr + peerId?: PeerId +} + +export const MultiaddrView: React.FC = (props) => { + let address = props.addr.toString() + if (props.peerId && address.endsWith(`/p2p/${props.peerId}`)) { + address = address.slice(0, address.lastIndexOf("/p2p/")) + } + + if (address.endsWith("/p2p-circuit/webrtc")) { + return /webrtc + } else { + return {address} + } +} diff --git a/examples/document/src/components/PeerIdView.tsx b/examples/document/src/components/PeerIdView.tsx new file mode 100644 index 000000000..1d523e773 --- /dev/null +++ b/examples/document/src/components/PeerIdView.tsx @@ -0,0 +1,19 @@ +import React from "react" + +import type { PeerId } from "@libp2p/interface" + +export interface PeerIdViewProps { + className?: string + peerId: PeerId +} + +export const PeerIdView: React.FC = (props) => { + const className = props.className ?? "text-sm" + const id = props.peerId.toString() + return ( + + {/* {id.slice(0, 12)}…{id.slice(-4)} */} + {id} + + ) +} diff --git a/examples/document/src/connect/ConnectEIP712Burner.tsx b/examples/document/src/connect/ConnectEIP712Burner.tsx new file mode 100644 index 000000000..2e5cd987f --- /dev/null +++ b/examples/document/src/connect/ConnectEIP712Burner.tsx @@ -0,0 +1,62 @@ +import React, { useCallback, useContext, useState } from "react" +import { Eip1193Provider, EventEmitterable } from "ethers" + +import { Eip712Signer } from "@canvas-js/chain-ethereum" + +import { AppContext } from "../AppContext.js" + +declare global { + // eslint-disable-next-line no-var + var ethereum: undefined | null | (Eip1193Provider & EventEmitterable<"accountsChanged" | "chainChanged">) +} + +export interface ConnectEIP712BurnerProps {} + +export const ConnectEIP712Burner: React.FC = ({}) => { + const { app, sessionSigner, setSessionSigner, address, setAddress } = useContext(AppContext) + + const [error, setError] = useState(null) + + const connect = useCallback(async () => { + if (app === null) { + setError(new Error("app not initialized")) + return + } + + const signer = new Eip712Signer() + const address = await signer.getDid() + setAddress(address) + setSessionSigner(signer) + }, [app]) + + const disconnect = useCallback(async () => { + setAddress(null) + setSessionSigner(null) + }, [sessionSigner]) + + if (error !== null) { + return ( +
+ {error.message} +
+ ) + } else if (address !== null && sessionSigner instanceof Eip712Signer) { + return ( + + ) + } else { + return ( + + ) + } +} diff --git a/examples/document/src/contract.ts b/examples/document/src/contract.ts new file mode 100644 index 000000000..5d63ed928 --- /dev/null +++ b/examples/document/src/contract.ts @@ -0,0 +1,14 @@ +import type { Actions, ModelSchema } from "@canvas-js/core" + +export const models = { + documents: { + id: "primary", + content: "yjs-doc", + }, +} satisfies ModelSchema + +export const actions = { + async applyDeltaToDoc(db, delta) { + await db.ytext.applyDelta("documents", "0", delta) + }, +} satisfies Actions diff --git a/examples/document/src/index.tsx b/examples/document/src/index.tsx new file mode 100644 index 000000000..a2a1b0c67 --- /dev/null +++ b/examples/document/src/index.tsx @@ -0,0 +1,14 @@ +import React from "react" +import ReactDOM from "react-dom/client" + +import "../styles.css" + +import { App } from "./App.js" + +const root = ReactDOM.createRoot(document.getElementById("root")!) + +root.render( + + + , +) diff --git a/examples/document/src/useQuill.ts b/examples/document/src/useQuill.ts new file mode 100644 index 000000000..7fc72a4ae --- /dev/null +++ b/examples/document/src/useQuill.ts @@ -0,0 +1,89 @@ +import { Canvas } from "@canvas-js/core" +import { AbstractModelDB } from "@canvas-js/modeldb" +import Quill from "quill" +import { useEffect, useRef } from "react" + +export const useQuill = ({ + modelName, + modelKey, + app, +}: { + modelName: string + modelKey: string + app: Canvas | undefined +}) => { + const db = app?.db + const dbRef = useRef(db ?? null) + const subscriptionRef = useRef(null) + const quillRef = useRef() + + const cursorRef = useRef(-1) + + useEffect(() => { + // Unsubscribe from the cached database handle, if necessary + if ( + !app || + db === null || + modelName === null || + db === undefined || + modelName === undefined || + modelKey === undefined + ) { + if (dbRef.current !== null) { + if (subscriptionRef.current !== null) { + dbRef.current.unsubscribe(subscriptionRef.current) + subscriptionRef.current = null + } + } + + dbRef.current = db ?? null + return + } + + if (dbRef.current === db && subscriptionRef.current !== null) { + return + } + + if (dbRef.current !== null && subscriptionRef.current !== null) { + db.unsubscribe(subscriptionRef.current) + } + + // set the initial value + const initialContents = app.getYDoc(modelName, modelKey).getText().toDelta() + quillRef.current?.updateContents(initialContents) + + // get the initial value for cursorRef + const startId = app.getYDocId(modelName, modelKey) + const query = { + where: { + model: modelName, + key: modelKey, + isAppend: false, + id: { gt: startId }, + }, + limit: 1, + orderBy: { id: "desc" as const }, + } + + const { id } = db.subscribe("$document_updates", query, (results) => { + for (const result of results) { + const resultId = result.id as number + if (cursorRef.current < resultId) { + cursorRef.current = resultId + quillRef.current?.updateContents(result.data) + } + } + }) + dbRef.current = db + + subscriptionRef.current = id + + return () => { + if (dbRef.current !== null && subscriptionRef.current !== null) dbRef.current.unsubscribe(subscriptionRef.current) + dbRef.current = null + subscriptionRef.current = null + } + }, [(db as any)?.isProxy ? null : db, modelKey, modelName]) + + return quillRef +} diff --git a/examples/document/styles.css b/examples/document/styles.css new file mode 100644 index 000000000..fd0d1983e --- /dev/null +++ b/examples/document/styles.css @@ -0,0 +1,17 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +html, +body, +#root { + margin: 0; + width: 100%; + height: 100%; +} + +main { + height: 100%; + max-width: 960px; + padding: 2em; +} diff --git a/examples/document/tailwind.config.js b/examples/document/tailwind.config.js new file mode 100644 index 000000000..719ea4b3d --- /dev/null +++ b/examples/document/tailwind.config.js @@ -0,0 +1,4 @@ +export default { + content: ["./src/**/*.{html,js,tsx}"], + plugins: [], +} diff --git a/examples/document/tsconfig.json b/examples/document/tsconfig.json new file mode 100644 index 000000000..d21902116 --- /dev/null +++ b/examples/document/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"], + "compilerOptions": { + "types": ["vite/client"], + "rootDir": "src", + "outDir": "lib", + "esModuleInterop": true, + "jsx": "react" + }, + "references": [ + { "path": "../../packages/core" }, + { "path": "../../packages/chain-ethereum" }, + { "path": "../../packages/gossiplog" }, + { "path": "../../packages/hooks" }, + { "path": "../../packages/interfaces" }, + { "path": "../../packages/modeldb" } + ] +} diff --git a/examples/document/vite.config.js b/examples/document/vite.config.js new file mode 100644 index 000000000..8cafc7b4d --- /dev/null +++ b/examples/document/vite.config.js @@ -0,0 +1,28 @@ +import { defineConfig } from "vite" +import wasm from "vite-plugin-wasm" +import { nodePolyfills } from "vite-plugin-node-polyfills" + +export default defineConfig({ + // ...other config settings + plugins: [nodePolyfills({ globals: { Buffer: true } }), wasm()], + server: { + headers: { + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Embedder-Policy": "require-corp", + }, + }, + build: { + minify: false, + target: "es2022", + }, + optimizeDeps: { + exclude: ["@sqlite.org/sqlite-wasm", "quickjs-emscripten"], + esbuildOptions: { + // Node.js global to browser globalThis + define: { + global: "globalThis", + }, + }, + }, + publicDir: 'public' +}) diff --git a/package-lock.json b/package-lock.json index 7096d18e2..c8b2696be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -153,6 +153,61 @@ "url": "https://paulmillr.com/funding/" } }, + "examples/document": { + "name": "@canvas-js/document", + "version": "0.14.0-next.1", + "dependencies": { + "@canvas-js/chain-atp": "0.14.0-next.1", + "@canvas-js/chain-cosmos": "0.14.0-next.1", + "@canvas-js/chain-ethereum": "0.14.0-next.1", + "@canvas-js/chain-ethereum-viem": "0.14.0-next.1", + "@canvas-js/chain-solana": "0.14.0-next.1", + "@canvas-js/chain-substrate": "0.14.0-next.1", + "@canvas-js/cli": "0.14.0-next.1", + "@canvas-js/core": "0.14.0-next.1", + "@canvas-js/gossiplog": "0.14.0-next.1", + "@canvas-js/hooks": "0.14.0-next.1", + "@canvas-js/interfaces": "0.14.0-next.1", + "@canvas-js/modeldb-sqlite-wasm": "0.14.0-next.1", + "@farcaster/auth-kit": "^0.6.0", + "@farcaster/frame-sdk": "^0.0.26", + "@libp2p/interface": "^2.7.0", + "@multiformats/multiaddr": "^12.3.5", + "@noble/hashes": "^1.7.1", + "@types/react": "^18.3.9", + "@types/react-dom": "^18.3.0", + "buffer": "^6.0.3", + "comlink": "^4.4.1", + "ethers": "^6.13.5", + "idb": "^8.0.2", + "multiformats": "^13.3.2", + "near-api-js": "^2.1.4", + "process": "^0.11.10", + "quill": "^2.0.3", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "vite-plugin-wasm": "^3.3.0" + }, + "devDependencies": { + "autoprefixer": "^10.4.19", + "postcss": "^8.4.38", + "tailwindcss": "^3.4.4", + "typescript": "~5.6.0", + "vite": "^5.4.8", + "vite-plugin-node-polyfills": "^0.22.0" + } + }, + "examples/document/node_modules/@noble/hashes": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.7.1.tgz", + "integrity": "sha512-B8XBPsn4vT/KJAGqDzbwztd+6Yte3P4V7iafm24bxgDe/mlRuK6xmWPuCNrKt2vDafZ8MfJLlchDG/vYafQEjQ==", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "examples/encrypted-chat": { "name": "@canvas-js/example-chat-encrypted", "version": "0.14.0-next.1", @@ -2784,6 +2839,10 @@ "resolved": "packages/core", "link": true }, + "node_modules/@canvas-js/document": { + "resolved": "examples/document", + "link": true + }, "node_modules/@canvas-js/ethereum-contracts": { "resolved": "packages/ethereum-contracts", "link": true @@ -20094,7 +20153,6 @@ }, "node_modules/fast-diff": { "version": "1.3.0", - "dev": true, "license": "Apache-2.0" }, "node_modules/fast-fifo": { @@ -22719,6 +22777,15 @@ "ws": "*" } }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/isows": { "version": "1.0.6", "funding": [ @@ -24376,6 +24443,26 @@ "node": ">= 0.8.0" } }, + "node_modules/lib0": { + "version": "0.2.99", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.99.tgz", + "integrity": "sha512-vwztYuUf1uf/1zQxfzRfO5yzfNKhTtgOByCruuiQQxWQXnPb8Itaube5ylofcV0oM0aKal9Mv+S1s1Ky0UYP1w==", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/libp2p": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/libp2p/-/libp2p-2.8.0.tgz", @@ -24591,6 +24678,11 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, "node_modules/lodash.debounce": { "version": "4.0.8", "license": "MIT", @@ -27440,6 +27532,11 @@ "dev": true, "license": "(MIT AND Zlib)" }, + "node_modules/parchment": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parchment/-/parchment-3.0.0.tgz", + "integrity": "sha512-HUrJFQ/StvgmXRcQ1ftY6VEZUq3jA2t9ncFN4F84J/vN0/FPpQF+8FKXb3l6fLces6q0uOHj6NJn+2xvZnxO6A==" + }, "node_modules/parent-module": { "version": "1.0.1", "license": "MIT", @@ -28878,6 +28975,33 @@ "@jitl/quickjs-ffi-types": "0.31.0" } }, + "node_modules/quill": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/quill/-/quill-2.0.3.tgz", + "integrity": "sha512-xEYQBqfYx/sfb33VJiKnSJp8ehloavImQ2A6564GAbqG55PGw1dAWUn1MUbQB62t0azawUS2CZZhWCjO8gRvTw==", + "dependencies": { + "eventemitter3": "^5.0.1", + "lodash-es": "^4.17.21", + "parchment": "^3.0.0", + "quill-delta": "^5.1.0" + }, + "engines": { + "npm": ">=8.2.3" + } + }, + "node_modules/quill-delta": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-5.1.0.tgz", + "integrity": "sha512-X74oCeRI4/p0ucjb5Ma8adTXd9Scumz367kkMK5V/IatcX6A0vlgLgKbzXWy5nZmCGeNJm2oQX0d2Eqj+ZIlCA==", + "dependencies": { + "fast-diff": "^1.3.0", + "lodash.clonedeep": "^4.5.0", + "lodash.isequal": "^4.5.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/race-event": { "version": "1.3.0", "license": "Apache-2.0 OR MIT" @@ -36177,6 +36301,22 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/yjs": { + "version": "13.6.23", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.23.tgz", + "integrity": "sha512-ExtnT5WIOVpkL56bhLeisG/N5c4fmzKn4k0ROVfJa5TY2QHbH7F0Wu2T5ZhR7ErsFWQEFafyrnSI8TPKVF9Few==", + "dependencies": { + "lib0": "^0.2.99" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/yn": { "version": "3.1.1", "devOptional": true, @@ -36642,7 +36782,8 @@ "quickjs-emscripten": "^0.31.0", "uint8arraylist": "^2.4.8", "uint8arrays": "^5.1.0", - "ws": "^8.18.0" + "ws": "^8.18.0", + "yjs": "^13.6.23" }, "devDependencies": { "@canvas-js/chain-cosmos": "0.14.0-next.1", diff --git a/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts b/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts index ff638b146..e8e6f76b7 100644 --- a/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts +++ b/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts @@ -128,6 +128,8 @@ export class Secp256k1DelegateSigner implements Signer(signers, messageLog, runtime) // Check to see if the $actions table is empty and populate it if necessary @@ -290,6 +292,12 @@ export class Canvas< this.log("applied action %s", signedMessage.id) + for (const additionalUpdate of this.runtime.additionalUpdates.get(signedMessage.id) || []) { + await this.messageLog.append(additionalUpdate) + } + this.runtime.additionalUpdates.delete(signedMessage.id) + this.log("applied additional updates for action %s", signedMessage.id) + return signedMessage } @@ -478,6 +486,14 @@ export class Canvas< } } + public getYDoc(model: string, key: string) { + return this.runtime.getYDoc(model, key) + } + + public getYDocId(model: string, key: string) { + return this.runtime.getYDocId(model, key) + } + public async createSnapshot(): Promise { return createSnapshot(this) } diff --git a/packages/core/src/DocumentStore.ts b/packages/core/src/DocumentStore.ts new file mode 100644 index 000000000..b78ff9c73 --- /dev/null +++ b/packages/core/src/DocumentStore.ts @@ -0,0 +1,158 @@ +import { Message, Updates } from "@canvas-js/interfaces" +import { AbstractModelDB, ModelSchema } from "@canvas-js/modeldb" +import * as Y from "yjs" +import { YjsCall } from "./ExecutionContext.js" +import { assert } from "@canvas-js/utils" + +type Delta = Y.YTextEvent["changes"]["delta"] + +function getDeltaForYText(ytext: Y.Text, fn: () => void): Delta { + let delta: Delta | null = null + + const handler = (event: Y.YTextEvent) => { + delta = event.changes.delta + } + + ytext.observe(handler) + fn() + ytext.unobserve(handler) + return delta || [] +} + +export type DocumentUpdateRecord = { + primary: string + model: string + key: string + id: number + data: Delta + diff: Uint8Array + isAppend: boolean +} + +export class DocumentStore { + private documents: Record> = {} + private documentIds: Record> = {} + + public static schema = { + $document_updates: { + primary: "primary", + model: "string", + key: "string", + id: "number", + // applyDelta, insert or delete + data: "json", + // yjs document diff + diff: "bytes", + isAppend: "boolean", + $indexes: ["id"], + }, + } satisfies ModelSchema + + #db: AbstractModelDB | null = null + + public get db() { + assert(this.#db !== null, "internal error - expected this.#db !== null") + return this.#db + } + + public set db(db: AbstractModelDB) { + this.#db = db + } + + public getYDoc(model: string, key: string) { + this.documents[model] ||= {} + this.documents[model][key] ||= new Y.Doc() + return this.documents[model][key] + } + + public async loadSavedDocuments() { + assert(this.#db !== null, "internal error - expected this.#db !== null") + // iterate over the past document operations + // and create the yjs documents + for await (const operation of this.#db.iterate("$document_updates")) { + const doc = this.getYDoc(operation.model, operation.key) + Y.applyUpdate(doc, operation.diff) + const existingId = this.getId(operation.model, operation.key) + if (operation.id > existingId) { + this.setId(operation.model, operation.key, operation.id) + } + } + } + + public getId(model: string, key: string) { + this.documentIds[model] ||= {} + return this.documentIds[model][key] ?? -1 + } + + public setId(model: string, key: string, id: number) { + this.documentIds[model] ||= {} + this.documentIds[model][key] = id + } + + public getNextId(model: string, key: string) { + return this.getId(model, key) + 1 + } + + public async applyYjsCalls(model: string, key: string, calls: YjsCall[]) { + assert(this.#db !== null, "internal error - expected this.#db !== null") + + const doc = this.getYDoc(model, key) + + // get the initial state of the document + const beforeState = Y.encodeStateAsUpdate(doc) + + const delta = getDeltaForYText(doc.getText(), () => { + for (const call of calls) { + if (call.call === "insert") { + doc.getText().insert(call.index, call.content) + } else if (call.call === "delete") { + doc.getText().delete(call.index, call.length) + } else if (call.call === "applyDelta") { + // TODO: do we actually need to call sanitize here? + doc.getText().applyDelta(call.delta.ops, { sanitize: true }) + } else { + throw new Error("unexpected call type") + } + } + }) + + // diff the document with the initial state + const afterState = Y.encodeStateAsUpdate(doc) + const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) + + await this.writeDocumentUpdate(model, key, delta || [], diff, true) + + return { model, key, diff } + } + + public async consumeUpdatesMessage(message: Message) { + assert(this.#db !== null, "internal error - expected this.#db !== null") + + for (const { model, key, diff } of message.payload.updates) { + // apply the diff to the doc + const doc = this.getYDoc(model, key) + const delta = getDeltaForYText(doc.getText(), () => { + Y.applyUpdate(doc, diff) + }) + + // save the observed update to the db + await this.writeDocumentUpdate(model, key, delta, diff, false) + } + } + + private async writeDocumentUpdate(model: string, key: string, data: Delta, diff: Uint8Array, isAppend: boolean) { + assert(this.#db !== null, "internal error - expected this.#db !== null") + const id = this.getNextId(model, key) + this.setId(model, key, id) + + await this.#db.set(`$document_updates`, { + primary: `${model}/${key}/${id}`, + model, + key, + id, + data, + diff, + isAppend, + }) + } +} diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index 7d2439d5e..289e0669d 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -1,14 +1,32 @@ import * as cbor from "@ipld/dag-cbor" import { blake3 } from "@noble/hashes/blake3" import { bytesToHex } from "@noble/hashes/utils" +import * as Y from "yjs" -import type { Action, MessageType } from "@canvas-js/interfaces" +import type { Action, MessageType, Updates } from "@canvas-js/interfaces" import { ModelValue, PropertyValue, validateModelValue, updateModelValue, mergeModelValue } from "@canvas-js/modeldb" import { AbstractGossipLog, SignedMessage, MessageId } from "@canvas-js/gossiplog" import { assert, mapValues } from "@canvas-js/utils" export const getKeyHash = (key: string) => bytesToHex(blake3(key, { dkLen: 16 })) +type YjsCallInsert = { + call: "insert" + index: number + content: string +} +type YjsCallDelete = { + call: "delete" + index: number + length: number +} +type YjsCallApplyDelta = { + call: "applyDelta" + // we don't have a declared type for Quill Deltas + delta: any +} + +export type YjsCall = YjsCallInsert | YjsCallDelete | YjsCallApplyDelta export class ExecutionContext { // // recordId -> { version, value } @@ -18,6 +36,7 @@ export class ExecutionContext { // public readonly writes: Record = {} public readonly modelEntries: Record> + public readonly yjsCalls: Record> = {} public readonly root: MessageId[] constructor( @@ -137,4 +156,24 @@ export class ExecutionContext { validateModelValue(this.db.models[model], result) this.modelEntries[model][key] = result } + + public pushYjsCall(modelName: string, id: string, call: YjsCall) { + this.yjsCalls[modelName] ||= {} + this.yjsCalls[modelName][id] ||= [] + this.yjsCalls[modelName][id].push(call) + } + + public async getYDoc(modelName: string, id: string): Promise { + const existingStateEntries = await this.db.query<{ id: string; content: Uint8Array }>(`${modelName}:state`, { + where: { id: id }, + limit: 1, + }) + if (existingStateEntries.length > 0) { + const doc = new Y.Doc() + Y.applyUpdate(doc, existingStateEntries[0].content) + return doc + } else { + return null + } + } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ec6890370..326442227 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,6 +1,7 @@ export * from "./Canvas.js" export * from "./CanvasLoadable.js" export * from "./types.js" +export * from "./DocumentStore.js" export { hashContract } from "./snapshot.js" export { Action, Session, Snapshot } from "@canvas-js/interfaces" export { NetworkClient } from "@canvas-js/gossiplog" diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index 9bfef9076..701c0f94b 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -1,15 +1,25 @@ import * as cbor from "@ipld/dag-cbor" import { logger } from "@libp2p/logger" -import type { Action, Session, Snapshot, SignerCache, Awaitable, MessageType } from "@canvas-js/interfaces" +import type { + Action, + Session, + Snapshot, + SignerCache, + Awaitable, + MessageType, + Updates, + Message, +} from "@canvas-js/interfaces" import { AbstractModelDB, Effect, ModelSchema } from "@canvas-js/modeldb" import { GossipLogConsumer, MAX_MESSAGE_ID, AbstractGossipLog, SignedMessage } from "@canvas-js/gossiplog" import { assert } from "@canvas-js/utils" +import { DocumentStore } from "../DocumentStore.js" import { ExecutionContext, getKeyHash } from "../ExecutionContext.js" -import { isAction, isSession, isSnapshot } from "../utils.js" import { Contract } from "../types.js" +import { isAction, isSession, isSnapshot, isUpdates } from "../utils.js" export type EffectRecord = { key: string; value: Uint8Array | null; branch: number; clock: number } @@ -64,12 +74,30 @@ export abstract class AbstractRuntime { } satisfies ModelSchema protected static getModelSchema(schema: ModelSchema): ModelSchema { + const outputSchema: ModelSchema = {} + for (const [modelName, modelSchema] of Object.entries(schema)) { + // @ts-ignore + if (modelSchema.content === "yjs-doc") { + if ( + Object.entries(modelSchema).length !== 2 && + // @ts-ignore + modelSchema.id !== "primary" + ) { + // not valid + throw new Error("yjs-doc tables must have two columns, one of which is 'id'") + } + } else { + outputSchema[modelName] = modelSchema + } + } + return { - ...schema, + ...outputSchema, ...AbstractRuntime.sessionsModel, ...AbstractRuntime.actionsModel, ...AbstractRuntime.effectsModel, ...AbstractRuntime.usersModel, + ...DocumentStore.schema, } } @@ -79,8 +107,11 @@ export abstract class AbstractRuntime { public abstract readonly actionNames: string[] public abstract readonly contract: string | Contract + public readonly additionalUpdates = new Map() + protected readonly log = logger("canvas:runtime") #db: AbstractModelDB | null = null + #documentStore = new DocumentStore() protected constructor() {} @@ -95,20 +126,24 @@ export abstract class AbstractRuntime { public set db(db: AbstractModelDB) { this.#db = db + this.#documentStore.db = db } public getConsumer(): GossipLogConsumer { const handleSession = this.handleSession.bind(this) const handleAction = this.handleAction.bind(this) const handleSnapshot = this.handleSnapshot.bind(this) + const handleUpdates = this.handleUpdates.bind(this) - return async function (this: AbstractGossipLog, signedMessage) { + return async function (this: AbstractGossipLog, signedMessage, isAppend: boolean) { if (isSession(signedMessage)) { return await handleSession(signedMessage) } else if (isAction(signedMessage)) { - return await handleAction(signedMessage, this) + return await handleAction(signedMessage, this, isAppend) } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) + } else if (isUpdates(signedMessage)) { + return await handleUpdates(signedMessage.message, isAppend) } else { throw new Error("invalid message payload type") } @@ -166,7 +201,11 @@ export abstract class AbstractRuntime { await this.db.apply(effects) } - private async handleAction(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { + private async handleAction( + signedMessage: SignedMessage, + messageLog: AbstractGossipLog, + isAppend: boolean, + ) { const { id, signature, message } = signedMessage const { did, name, context } = message.payload @@ -249,6 +288,39 @@ export abstract class AbstractRuntime { throw err } + if (isAppend) { + const updates = [] + for (const [model, modelCalls] of Object.entries(executionContext.yjsCalls)) { + for (const [key, calls] of Object.entries(modelCalls)) { + const update = await this.#documentStore.applyYjsCalls(model, key, calls) + updates.push(update) + } + } + if (updates.length > 0) { + this.additionalUpdates.set(id, [{ type: "updates", updates }]) + } else { + this.additionalUpdates.set(id, []) + } + } + return result } + + public getYDoc(model: string, key: string) { + return this.#documentStore.getYDoc(model, key) + } + + public getYDocId(model: string, key: string) { + return this.#documentStore.getId(model, key) + } + + public async loadSavedDocuments() { + await this.#documentStore.loadSavedDocuments() + } + + public async handleUpdates(message: Message, isAppend: boolean) { + if (!isAppend) { + return await this.#documentStore.consumeUpdatesMessage(message) + } + } } diff --git a/packages/core/src/runtime/ContractRuntime.ts b/packages/core/src/runtime/ContractRuntime.ts index b64849418..4364388d7 100644 --- a/packages/core/src/runtime/ContractRuntime.ts +++ b/packages/core/src/runtime/ContractRuntime.ts @@ -8,6 +8,7 @@ import { assert, mapValues } from "@canvas-js/utils" import { Contract } from "../types.js" import { ExecutionContext } from "../ExecutionContext.js" import { AbstractRuntime } from "./AbstractRuntime.js" +import { DocumentStore } from "../DocumentStore.js" export class ContractRuntime extends AbstractRuntime { public static async init( @@ -136,6 +137,28 @@ export class ContractRuntime extends AbstractRuntime { const key = vm.context.getString(keyHandle) this.context.deleteModelValue(model, key) }), + ytext: vm.wrapObject({ + insert: vm.context.newFunction("insert", (modelHandle, keyHandle, indexHandle, contentHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const index = vm.context.getNumber(indexHandle) + const content = vm.context.getString(contentHandle) + this.context.pushYjsCall(model, key, { call: "insert", index, content }) + }), + delete: vm.context.newFunction("delete", (modelHandle, keyHandle, indexHandle, lengthHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const index = vm.context.getNumber(indexHandle) + const length = vm.context.getNumber(lengthHandle) + this.context.pushYjsCall(model, key, { call: "delete", index, length }) + }), + applyDelta: vm.context.newFunction("applyDelta", (modelHandle, keyHandle, deltaHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const delta = vm.context.getString(deltaHandle) + this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) + }), + }), }) .consume(vm.cache) } diff --git a/packages/core/src/runtime/FunctionRuntime.ts b/packages/core/src/runtime/FunctionRuntime.ts index f2956207b..6d79cf176 100644 --- a/packages/core/src/runtime/FunctionRuntime.ts +++ b/packages/core/src/runtime/FunctionRuntime.ts @@ -174,6 +174,17 @@ export class FunctionRuntime extends AbstractRuntim this.releaseLock() } }, + ytext: { + insert: async (model: string, key: string, index: number, content: string) => { + this.context.pushYjsCall(model, key, { call: "insert", index, content }) + }, + delete: async (model: string, key: string, index: number, length: number) => { + this.context.pushYjsCall(model, key, { call: "delete", index, length }) + }, + applyDelta: async (model: string, key: string, delta: any) => { + this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) + }, + }, } } diff --git a/packages/core/src/schema.ts b/packages/core/src/schema.ts index e8605f5ba..fce4e89cd 100644 --- a/packages/core/src/schema.ts +++ b/packages/core/src/schema.ts @@ -1,7 +1,7 @@ import { fromDSL } from "@ipld/schema/from-dsl.js" import { create } from "@ipld/schema/typed.js" -import type { Action, MessageType, Session } from "@canvas-js/interfaces" +import type { Action, MessageType, Session, Snapshot, Updates } from "@canvas-js/interfaces" const schema = ` type ActionContext struct { @@ -39,10 +39,21 @@ type Snapshot struct { effects [SnapshotEffect] } +type Update struct { + model String + key String + diff Bytes +} + +type Updates struct { + updates [Update] +} + type Payload union { | Action "action" | Session "session" | Snapshot "snapshot" + | Updates "updates" } representation inline { discriminantKey "type" } @@ -51,6 +62,11 @@ type Payload union { const { toTyped } = create(fromDSL(schema), "Payload") export function validatePayload(payload: unknown): payload is MessageType { - const result = toTyped(payload) as { Action: Omit } | { Session: Omit } | undefined + const result = toTyped(payload) as + | { Action: Omit } + | { Session: Omit } + | { Snapshot: Omit } + | { Updates: Omit } + | undefined return result !== undefined } diff --git a/packages/core/src/snapshot.ts b/packages/core/src/snapshot.ts index 11388d8fb..790752170 100644 --- a/packages/core/src/snapshot.ts +++ b/packages/core/src/snapshot.ts @@ -1,6 +1,7 @@ import { sha256 } from "@noble/hashes/sha256" import { bytesToHex } from "@noble/hashes/utils" import * as cbor from "@ipld/dag-cbor" +import * as Y from "yjs" import { MIN_MESSAGE_ID } from "@canvas-js/gossiplog" import { Snapshot, SnapshotEffect } from "@canvas-js/interfaces" @@ -9,6 +10,7 @@ import type { PropertyType } from "@canvas-js/modeldb" import { Canvas } from "./Canvas.js" import { Contract } from "./types.js" import { EffectRecord } from "./runtime/AbstractRuntime.js" +import { DocumentUpdateRecord } from "./DocumentStore.js" // typeguards export const isIndexInit = (value: unknown): value is string[] => Array.isArray(value) @@ -60,6 +62,25 @@ export async function createSnapshot>(app: Canvas): Prom effectsMap.set(recordId, { key: effectKey, value, branch, clock }) } } + + // iterate over the document updates table + const documentsMap = new Map() + for await (const row of app.db.iterate("$document_updates")) { + const { primary, diff } = row + + const doc = documentsMap.get(primary) || new Y.Doc() + Y.applyUpdate(doc, diff) + documentsMap.set(primary, doc) + } + + modelData[`$document_updates`] = [] + for (const [primary, doc] of documentsMap.entries()) { + const diff = Y.encodeStateAsUpdate(doc) + const [model, key, id_] = primary.split("/") + const row = { primary, model, key, id: 0, data: doc.getText().toDelta(), diff, isAppend: false } + modelData[`$document_updates`].push(cbor.encode(row)) + } + const effects = Array.from(effectsMap.values()).map(({ key, value }: SnapshotEffect) => ({ key, value })) return { diff --git a/packages/core/src/targets/interface.ts b/packages/core/src/targets/interface.ts index 3bd0b6101..88d189bf1 100644 --- a/packages/core/src/targets/interface.ts +++ b/packages/core/src/targets/interface.ts @@ -1,6 +1,6 @@ import type pg from "pg" -import type { Action, MessageType, Session, Snapshot } from "@canvas-js/interfaces" +import type { MessageType } from "@canvas-js/interfaces" import type { AbstractGossipLog, GossipLogInit } from "@canvas-js/gossiplog" import type { Canvas } from "@canvas-js/core" import type { SqlStorage } from "@cloudflare/workers-types" diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 24d3b3acc..4fb57822f 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -17,14 +17,18 @@ export type ActionImplementation< ModelsT extends ModelSchema = ModelSchema, Args extends Array = any, Result = any, -> = (this: ActionContext>, db: ModelAPI>, ...args: Args) => Awaitable +> = ( + this: ActionContext>, + db: ModelAPI>, + ...args: Args +) => Awaitable export type Chainable> = Promise & { link: ( model: T, primaryKey: string, through?: { through: string }, - ) => Promise, + ) => Promise unlink: ( model: T, primaryKey: string, @@ -40,6 +44,16 @@ export type ModelAPI> = { update: (model: T, value: Partial) => Chainable merge: (model: T, value: Partial) => Chainable delete: (model: T, key: string) => Promise + ytext: { + insert: ( + model: T, + key: string, + index: number, + content: string, + ) => Promise + delete: (model: T, key: string, index: number, length: number) => Promise + applyDelta: (model: T, key: string, delta: string) => Promise + } } export type ActionContext> = { diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index f1ead69d3..9c12a4ea9 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -4,7 +4,7 @@ import { blake3 } from "@noble/hashes/blake3" import { utf8ToBytes } from "@noble/hashes/utils" import { base64 } from "multiformats/bases/base64" -import { Action, MessageType, Session, Snapshot } from "@canvas-js/interfaces" +import { Action, MessageType, Session, Snapshot, Updates } from "@canvas-js/interfaces" import { SignedMessage } from "@canvas-js/gossiplog" import { PrimaryKeyValue } from "@canvas-js/modeldb" @@ -17,6 +17,9 @@ export const isSession = (signedMessage: SignedMessage): signedMess export const isSnapshot = (signedMessage: SignedMessage): signedMessage is SignedMessage => signedMessage.message.payload.type === "snapshot" +export const isUpdates = (signedMessage: SignedMessage): signedMessage is SignedMessage => + signedMessage.message.payload.type === "updates" + export const topicPattern = /^[a-zA-Z0-9.-]+$/ export function getErrorMessage(err: unknown): string { diff --git a/packages/core/test/documentStore.test.ts b/packages/core/test/documentStore.test.ts new file mode 100644 index 000000000..496ee4158 --- /dev/null +++ b/packages/core/test/documentStore.test.ts @@ -0,0 +1,29 @@ +import test from "ava" + +import { DocumentStore } from "@canvas-js/core" +import { ModelDB } from "@canvas-js/modeldb-sqlite" + +test("save and load a document using the document store", async (t) => { + const db = await ModelDB.open(null, { models: DocumentStore.schema }) + const ds = new DocumentStore() + ds.db = db + + const delta = { ops: [{ insert: "hello world" }] } + await ds.applyYjsCalls("documents", "0", [{ call: "applyDelta", delta }]) + + const ds2 = new DocumentStore() + ds2.db = db + await ds2.loadSavedDocuments() + const doc = ds2.getYDoc("documents", "0") + + t.deepEqual(doc.getText().toDelta(), delta.ops) +}) + +test("get and set id", async (t) => { + const ds = new DocumentStore() + + t.is(ds.getId("documents", "0"), -1) + ds.setId("documents", "0", 42) + t.is(ds.getId("documents", "0"), 42) + t.is(ds.getNextId("documents", "0"), 43) +}) diff --git a/packages/core/test/snapshot.test.ts b/packages/core/test/snapshot.test.ts index 76149bc0d..3e1c459b3 100644 --- a/packages/core/test/snapshot.test.ts +++ b/packages/core/test/snapshot.test.ts @@ -11,6 +11,10 @@ test("snapshot persists data across apps", async (t) => { id: "primary", content: "string", }, + documents: { + id: "primary", + content: "yjs-doc", + }, }, actions: { async createPost(db, { id, content }: { id: string; content: string }) { @@ -19,6 +23,12 @@ test("snapshot persists data across apps", async (t) => { async deletePost(db, { id }: { id: string }) { await db.delete("posts", id) }, + async insertIntoDocument(db, key, index, text) { + await db.ytext.insert("documents", key, index, text) + }, + async deleteFromDocument(db, key, index, length) { + await db.ytext.delete("documents", key, index, length) + }, }, }, } @@ -35,9 +45,11 @@ test("snapshot persists data across apps", async (t) => { await app.actions.createPost({ id: "d", content: "baz" }) await app.actions.deletePost({ id: "b" }) await app.actions.deletePost({ id: "d" }) + await app.actions.insertIntoDocument("e", 0, "Hello, world") + await app.actions.deleteFromDocument("e", 5, 7) const [clock, parents] = await app.messageLog.getClock() - t.is(clock, 8) // one session, six actions + t.is(clock, 12) // one session, eight actions, two "updates" messages t.is(parents.length, 1) // snapshot and add some more actions @@ -52,13 +64,17 @@ test("snapshot persists data across apps", async (t) => { t.is(await app2.db.get("posts", "d"), null) t.is(await app2.db.get("posts", "e"), null) + const doc1 = app2.getYDoc("documents", "e") + t.is(doc1.getText().toJSON(), "Hello") + await app2.actions.createPost({ id: "a", content: "1" }) await app2.actions.createPost({ id: "b", content: "2" }) await app2.actions.createPost({ id: "e", content: "3" }) await app2.actions.createPost({ id: "f", content: "4" }) + await app2.actions.insertIntoDocument("e", 6, "?") const [clock2, parents2] = await app2.messageLog.getClock() - t.is(clock2, 7) // one snapshot, one session, four actions + t.is(clock2, 9) // one snapshot, one session, four actions t.is(parents2.length, 1) t.is((await app2.db.get("posts", "a"))?.content, "1") @@ -67,6 +83,8 @@ test("snapshot persists data across apps", async (t) => { t.is(await app2.db.get("posts", "d"), null) t.is((await app2.db.get("posts", "e"))?.content, "3") t.is((await app2.db.get("posts", "f"))?.content, "4") + const doc2 = app2.getYDoc("documents", "e") + t.is(doc2.getText().toJSON(), "Hello?") // snapshot a second time const snapshot2 = await app2.createSnapshot() @@ -79,6 +97,8 @@ test("snapshot persists data across apps", async (t) => { t.is((await app3.db.get("posts", "e"))?.content, "3") t.is((await app3.db.get("posts", "f"))?.content, "4") t.is(await app3.db.get("posts", "g"), null) + const doc3 = app3.getYDoc("documents", "e") + t.is(doc3.getText().toJSON(), "Hello?") const [clock3] = await app3.messageLog.getClock() t.is(clock3, 2) // one snapshot diff --git a/packages/core/test/yjs.test.ts b/packages/core/test/yjs.test.ts new file mode 100644 index 000000000..09b8a023b --- /dev/null +++ b/packages/core/test/yjs.test.ts @@ -0,0 +1,74 @@ +import { SIWESigner } from "@canvas-js/chain-ethereum" +import test, { ExecutionContext } from "ava" +import { Canvas } from "@canvas-js/core" + +const contract = ` +export const models = { + articles: { + id: "primary", + content: "yjs-doc" + } +}; +export const actions = { + async createNewArticle(db) { + const { id } = this + await db.ytext.insert("articles", id, 0, "") + }, + async insertIntoDoc(db, key, index, text) { + await db.ytext.insert("articles", key, index, text) + }, + async deleteFromDoc(db, key, index, length) { + await db.ytext.delete("articles", key, index, length) + } +}; +` + +const init = async (t: ExecutionContext) => { + const signer = new SIWESigner() + const app = await Canvas.initialize({ + contract, + topic: "com.example.app", + reset: true, + signers: [signer], + }) + + t.teardown(() => app.stop()) + return { app, signer } +} + +test("apply an action and read a record from the database", async (t) => { + const { app: app1 } = await init(t) + + const { id } = await app1.actions.createNewArticle() + + t.log(`applied action ${id}`) + + await app1.actions.insertIntoDoc(id, 0, "Hello, world") + t.is(app1.getYDoc("articles", id).getText().toJSON(), "Hello, world") + + // create another app + const { app: app2 } = await init(t) + + // sync the apps + await app1.messageLog.serve((s) => app2.messageLog.sync(s)) + t.is(app2.getYDoc("articles", id).getText().toJSON(), "Hello, world") + + // insert ! into app1 + await app1.actions.insertIntoDoc(id, 12, "!") + t.is(app1.getYDoc("articles", id).getText().toJSON(), "Hello, world!") + + // insert ? into app2 + await app2.actions.insertIntoDoc(id, 12, "?") + t.is(app2.getYDoc("articles", id).getText().toJSON(), "Hello, world?") + + // sync app2 -> app1 + await app2.messageLog.serve((s) => app1.messageLog.sync(s)) + const app1MergedText = app1.getYDoc("articles", id).getText().toJSON() + + // sync app1 -> app2 + await app1.messageLog.serve((s) => app2.messageLog.sync(s)) + const app2MergedText = app2.getYDoc("articles", id).getText().toJSON() + + // both apps should now have converged + t.is(app1MergedText, app2MergedText) +}) diff --git a/packages/gossiplog/src/AbstractGossipLog.ts b/packages/gossiplog/src/AbstractGossipLog.ts index 48ae7d133..f6403d7ef 100644 --- a/packages/gossiplog/src/AbstractGossipLog.ts +++ b/packages/gossiplog/src/AbstractGossipLog.ts @@ -26,6 +26,7 @@ import { gossiplogTopicPattern } from "./utils.js" export type GossipLogConsumer = ( this: AbstractGossipLog, signedMessage: SignedMessage, + isAppend: boolean, ) => Awaitable export interface GossipLogInit { @@ -155,7 +156,8 @@ export abstract class AbstractGossipLog extends const { signature, message, branch } = record const signedMessage = this.encode(signature, message, { branch }) assert(signedMessage.id === id) - await this.#apply.apply(this, [signedMessage]) + // TODO: should isAppend be true or false? + await this.#apply.apply(this, [signedMessage, true]) } }) } @@ -290,7 +292,7 @@ export abstract class AbstractGossipLog extends const signedMessage = this.encode(signature, message) this.log("appending message %s at clock %d with parents %o", signedMessage.id, clock, parents) - const applyResult = await this.apply(txn, signedMessage) + const applyResult = await this.apply(txn, signedMessage, true) root = applyResult.root heads = applyResult.heads @@ -328,7 +330,7 @@ export abstract class AbstractGossipLog extends return null } - return await this.apply(txn, signedMessage) + return await this.apply(txn, signedMessage, false) }) if (result !== null) { @@ -341,6 +343,7 @@ export abstract class AbstractGossipLog extends private async apply( txn: ReadWriteTransaction, signedMessage: SignedMessage, + isAppend: boolean, ): Promise<{ root: Node; heads: string[]; result: Result }> { const { id, signature, message, key, value } = signedMessage this.log.trace("applying %s %O", id, message) @@ -358,7 +361,7 @@ export abstract class AbstractGossipLog extends const branch = await this.getBranch(id, parentMessageRecords) signedMessage.branch = branch - const result = await this.#apply.apply(this, [signedMessage]) + const result = await this.#apply.apply(this, [signedMessage, isAppend]) const hash = toString(hashEntry(key, value), "hex") diff --git a/packages/interfaces/src/MessageType.ts b/packages/interfaces/src/MessageType.ts index 389aad8e3..3e47cf939 100644 --- a/packages/interfaces/src/MessageType.ts +++ b/packages/interfaces/src/MessageType.ts @@ -1,5 +1,6 @@ import { Action } from "./Action.js" import { Session } from "./Session.js" import { Snapshot } from "./Snapshot.js" +import { Updates } from "./Updates.js" -export type MessageType = Action | Session | Snapshot +export type MessageType = Action | Session | Snapshot | Updates diff --git a/packages/interfaces/src/SessionSigner.ts b/packages/interfaces/src/SessionSigner.ts index 3b01566ed..af90b2661 100644 --- a/packages/interfaces/src/SessionSigner.ts +++ b/packages/interfaces/src/SessionSigner.ts @@ -1,7 +1,5 @@ import type { SignatureScheme, Signer } from "./Signer.js" import type { Session } from "./Session.js" -import type { Action } from "./Action.js" -import type { Snapshot } from "./Snapshot.js" import type { Awaitable } from "./Awaitable.js" import { MessageType } from "./MessageType.js" diff --git a/packages/interfaces/src/Updates.ts b/packages/interfaces/src/Updates.ts new file mode 100644 index 000000000..c7b1ac963 --- /dev/null +++ b/packages/interfaces/src/Updates.ts @@ -0,0 +1,10 @@ +type Update = { + model: string + key: string + diff: Uint8Array +} + +export type Updates = { + type: "updates" + updates: Update[] +} diff --git a/packages/interfaces/src/index.ts b/packages/interfaces/src/index.ts index d389bbe20..eced3c72b 100644 --- a/packages/interfaces/src/index.ts +++ b/packages/interfaces/src/index.ts @@ -8,3 +8,4 @@ export * from "./Session.js" export * from "./Snapshot.js" export * from "./Awaitable.js" export * from "./MessageType.js" +export * from "./Updates.js" diff --git a/packages/modeldb/src/types.ts b/packages/modeldb/src/types.ts index 3a332e158..6cbe8ce25 100644 --- a/packages/modeldb/src/types.ts +++ b/packages/modeldb/src/types.ts @@ -8,6 +8,7 @@ export type NullablePrimitiveType = `${PrimitiveType}?` export type ReferenceType = `@${string}` export type NullableReferenceType = `@${string}?` export type RelationType = `@${string}[]` +export type YjsDocType = "yjs-doc" export type PropertyType = | PrimaryKeyType @@ -16,6 +17,7 @@ export type PropertyType = | ReferenceType | NullableReferenceType | RelationType + | YjsDocType /** property name, or property names joined by slashes */ export type IndexInit = string diff --git a/tsconfig.json b/tsconfig.json index c8ebabe5b..7657e4021 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -44,6 +44,7 @@ { "path": "./packages/utils/test" }, { "path": "./examples/chat" }, { "path": "./examples/chat-next" }, + { "path": "./examples/document" }, { "path": "./examples/encrypted-chat" }, { "path": "./examples/snake" } ]