packages/gossiplog: Set up webrtc for browser target, update associated packages (#501)

* packages/gossiplog: add browser libp2p target with webrtc

* packages/bootstrap-peer: add maxRegistrationTTL and maxDiscoverLimit config options

* packages/gossiplog: update browser libp2p config

* packages/relay-server: update relay server config

* packages/test-network: add client-webrtc

* packages/test-network: update README.md

* packages/test-network: fix d3 error finally

* packages/test-network: remove self:peer:update in client-webrtc

* packages/test-network: show non-mesh connections as dotted lines

* packages/gossiplog: add @libp2p/circuit-relay-v2

* packages/gossiplog: set reservation ttl to 60s in the browser

* packages/test-network: adjust params

* packages/test-network: add server-side eventsource, move events and reduce to shared file

* packages/test-network: upgrade versions for npm packages in dockerfiles

* packages/test-network: better state handling and refreshing across sessions

* packages/test-network: consolidate internal imports

* packages/bootstrap-peer: upgrade @canvas-js/libp2p-rendezvous, add discoverFilter

* packages/relay-server: update testnet config

* packages/gossiplog: set DHT to client mode in browser, manually connect to all discovered peers

* packages/test-network: add worker socket system

* packages/test-network: update padding

* packages/test-network: remove nodes belonging to worker when worker closes

* packages/test-network: add autospawn

* packages/test-network: move Dockerfiles into folders

---------

Co-authored-by: Joel Gustafson <[email protected]>

Author: raykyri

package-lock.json

@@ -14,10 +14,11 @@
 	},
 	"exports": {
 		".": "./lib/index.js",
-		"./libp2p": "./lib/libp2p.js"
+		"./libp2p": "./lib/libp2p.js",
+		"./api": "./lib/api.js"
 	},
 	"dependencies": {
-		"@canvas-js/libp2p-rendezvous": "^0.4.5",
+		"@canvas-js/libp2p-rendezvous": "^0.5.2",
 		"@chainsafe/libp2p-noise": "^16.1.3",
 		"@chainsafe/libp2p-yamux": "^7.0.1",
 		"@ipld/dag-cbor": "^9.2.4",

packages/bootstrap-peer/package.json

@@ -3,31 +3,44 @@ import process from "node:process"
 import { PrivateKey } from "@libp2p/interface"
 import { generateKeyPair, privateKeyFromProtobuf } from "@libp2p/crypto/keys"
 
-const { DATABASE_PATH, LIBP2P_PRIVATE_KEY, LISTEN, ANNOUNCE, MAX_CONNECTIONS } = process.env
+const {
+	DATABASE_PATH,
+	LIBP2P_PRIVATE_KEY,
+	LISTEN,
+	ANNOUNCE,
+	MAX_CONNECTIONS,
+	MAX_DISCOVER_LIMIT,
+	MAX_REGISTRATION_TTL,
+} = process.env
 
 export interface Config {
 	path: string | null
 	privateKey: PrivateKey
 	listen: string[]
 	announce: string[]
 	maxConnections: number
+
+	maxRegistrationTTL?: number
+	maxDiscoverLimit?: number
 }
 
 export async function getConfig(config: Partial<Config>): Promise<Config> {
-	const path = DATABASE_PATH ?? null
+	const path = config.path ?? DATABASE_PATH ?? null
 
 	let privateKey = config.privateKey
 	if (privateKey === undefined) {
 		privateKey = await getPrivateKey()
 	}
 
-	let { maxConnections = 1024 } = config
-	if (MAX_CONNECTIONS !== undefined) maxConnections = parseInt(MAX_CONNECTIONS)
+	let { maxConnections = 1024, maxRegistrationTTL, maxDiscoverLimit } = config
+	if (MAX_CONNECTIONS !== undefined) maxConnections ??= parseInt(MAX_CONNECTIONS)
+	if (MAX_REGISTRATION_TTL !== undefined) maxRegistrationTTL ??= parseInt(MAX_REGISTRATION_TTL)
+	if (MAX_DISCOVER_LIMIT !== undefined) maxDiscoverLimit ??= parseInt(MAX_DISCOVER_LIMIT)
 
 	const listen = config.listen ?? LISTEN?.split(",") ?? ["/ip4/127.0.0.1/tcp/8080/ws"]
 	const announce = config.announce ?? ANNOUNCE?.split(",") ?? []
 
-	return { path, privateKey, listen, announce, maxConnections }
+	return { path, privateKey, listen, announce, maxConnections, maxRegistrationTTL, maxDiscoverLimit }
 }
 
 async function getPrivateKey(): Promise<PrivateKey> {

packages/bootstrap-peer/src/config.ts

@@ -3,21 +3,25 @@ import { webSockets } from "@libp2p/websockets"
 import { yamux } from "@chainsafe/libp2p-yamux"
 import { noise } from "@chainsafe/libp2p-noise"
 import { Identify, identify } from "@libp2p/identify"
-import { PingService, ping } from "@libp2p/ping"
+import { Ping, ping } from "@libp2p/ping"
 import { prometheusMetrics } from "@libp2p/prometheus-metrics"
 import { peerIdFromPrivateKey } from "@libp2p/peer-id"
+import { Multiaddr } from "@multiformats/multiaddr"
 
 import { rendezvousServer, RendezvousServer } from "@canvas-js/libp2p-rendezvous/server"
 
 import { Config, getConfig } from "./config.js"
 
 export type ServiceMap = {
 	identify: Identify
-	ping: PingService
+	ping: Ping
 	rendezvous: RendezvousServer
 }
 
-export const maxRegistrationTTL = 2 * 60 * 60 // 2h
+const defaultMaxRegistrationTTL = 2 * 60 * 60 // 2h
+const defaultMaxDiscoverLimit = 64
+
+const isWebRTC = (addr: Multiaddr) => addr.protoNames().slice(-3).join("/") === "p2p-circuit/webrtc/p2p"
 
 export async function getLibp2p(config: Partial<Config> = {}) {
 	const { path, privateKey, listen, announce, maxConnections } = await getConfig(config)
@@ -34,14 +38,17 @@ export async function getLibp2p(config: Partial<Config> = {}) {
 		announce.map((addr) => `${addr}/p2p/${peerId}`),
 	)
 
+	const maxRegistrationTTL = config.maxRegistrationTTL ?? defaultMaxRegistrationTTL
+	const maxDiscoverLimit = config.maxDiscoverLimit ?? defaultMaxDiscoverLimit
+
 	const libp2p = await createLibp2p<ServiceMap>({
 		privateKey: privateKey,
 		start: false,
 		addresses: { listen, announce },
 		transports: [webSockets({})],
 		connectionGater: { denyDialMultiaddr: (addr) => false },
 		connectionManager: { maxConnections },
-		connectionMonitor: { enabled: false, protocolPrefix: "canvas" },
+		connectionMonitor: { enabled: true, protocolPrefix: "canvas" },
 
 		peerStore: {
 			maxAddressAge: maxRegistrationTTL * 1000,
@@ -56,7 +63,21 @@ export async function getLibp2p(config: Partial<Config> = {}) {
 		services: {
 			identify: identify({ protocolPrefix: "canvas" }),
 			ping: ping({ protocolPrefix: "canvas" }),
-			rendezvous: rendezvousServer({ path, maxRegistrationTTL }),
+			rendezvous: rendezvousServer({
+				path,
+				maxRegistrationTTL,
+				maxDiscoverLimit,
+				discoverFilter: (namespace, peerId, multiaddrs) => {
+					if (multiaddrs.some(isWebRTC)) {
+						const connections = libp2p.getConnections(peerId)
+						if (connections.length === 0) {
+							return false
+						}
+					}
+
+					return true
+				},
+			}),
 		},
 	})
 

packages/bootstrap-peer/src/libp2p.ts

@@ -59,7 +59,7 @@
 		"any-signal": "^4.1.1",
 		"chalk": "^5.4.1",
 		"cors": "^2.8.5",
-		"esbuild": "^0.24.2",
+		"esbuild": "^0.25.5",
 		"esbuild-wasm": "^0.25.1",
 		"ethers": "^6.13.5",
 		"express": "^5.1.0",

packages/core/package.json

@@ -36,6 +36,12 @@
 			"node": "./lib/targets/node/index.js",
 			"react-native": "./lib/targets/react-native/index.js",
 			"default": "./lib/targets/default/index.js"
+		},
+		"#target/libp2p": {
+			"browser": "./lib/targets/browser/libp2p.js",
+			"node": "./lib/targets/node/libp2p.js",
+			"react-native": "./lib/targets/react-native/libp2p.js",
+			"default": "./lib/targets/default/libp2p.js"
 		}
 	},
 	"scripts": {
@@ -46,7 +52,7 @@
 	},
 	"dependencies": {
 		"@canvas-js/interfaces": "0.16.0-next.3",
-		"@canvas-js/libp2p-rendezvous": "^0.4.5",
+		"@canvas-js/libp2p-rendezvous": "^0.5.2",
 		"@canvas-js/modeldb": "0.16.0-next.3",
 		"@canvas-js/modeldb-durable-objects": "0.16.0-next.3",
 		"@canvas-js/modeldb-idb": "0.16.0-next.3",
@@ -63,6 +69,8 @@
 		"@chainsafe/libp2p-yamux": "^7.0.1",
 		"@ipld/dag-cbor": "^9.2.4",
 		"@ipld/dag-json": "^10.2.5",
+		"@libp2p/bootstrap": "^11.0.39",
+		"@libp2p/circuit-relay-v2": "^3.2.15",
 		"@libp2p/crypto": "^5.1.5",
 		"@libp2p/identify": "^3.0.33",
 		"@libp2p/interface": "^2.10.3",
@@ -73,6 +81,7 @@
 		"@libp2p/peer-id": "^5.1.6",
 		"@libp2p/ping": "^2.0.33",
 		"@libp2p/prometheus-metrics": "^4.3.22",
+		"@libp2p/webrtc": "^5.2.16",
 		"@libp2p/websockets": "^9.2.14",
 		"@multiformats/multiaddr": "^12.4.1",
 		"@noble/hashes": "^1.8.0",

packages/gossiplog/package.json

@@ -15,8 +15,8 @@ import type {
 import { ed25519, prepareMessage } from "@canvas-js/signatures"
 import { assert, zip } from "@canvas-js/utils"
 
-import { NetworkConfig, ServiceMap } from "@canvas-js/gossiplog/libp2p"
 import { NetworkClient } from "@canvas-js/gossiplog/client"
+import { NetworkConfig, ServiceMap } from "@canvas-js/gossiplog/libp2p"
 import { AbortError, MessageNotFoundError, MissingParentError } from "@canvas-js/gossiplog/errors"
 import * as sync from "@canvas-js/gossiplog/sync"
 

packages/gossiplog/src/AbstractGossipLog.ts

@@ -1,2 +1,4 @@
-export * from "./libp2p.js"
+export { getLibp2p } from "#target/libp2p"
+
+export * from "./interface.js"
 export * from "./service.js"

packages/gossiplog/src/libp2p/index.ts

@@ -0,0 +1,41 @@
+import type { MultiaddrConnection, PrivateKey, PubSub } from "@libp2p/interface"
+import type { Identify } from "@libp2p/identify"
+import type { KadDHT } from "@libp2p/kad-dht"
+import type { Ping } from "@libp2p/ping"
+import type { GossipsubEvents } from "@chainsafe/libp2p-gossipsub"
+export type { GossipSub } from "@chainsafe/libp2p-gossipsub"
+import type { Multiaddr } from "@multiformats/multiaddr"
+import type { Registry } from "prom-client"
+
+import type { RendezvousClient } from "@canvas-js/libp2p-rendezvous/client"
+
+import type { GossipLogService } from "./service.js"
+
+export interface NetworkConfig {
+	/** start libp2p on initialization (default: true) */
+	start?: boolean
+	privateKey?: PrivateKey
+
+	/** array of local WebSocket multiaddrs, e.g. "/ip4/127.0.0.1/tcp/3000/ws" */
+	listen?: string[]
+
+	/** array of public WebSocket multiaddrs, e.g. "/dns4/myapp.com/tcp/443/wss" */
+	announce?: string[]
+
+	bootstrapList?: string[]
+	relayServer?: string
+	denyDialMultiaddr?(multiaddr: Multiaddr): Promise<boolean> | boolean
+	denyInboundConnection?(maConn: MultiaddrConnection): Promise<boolean> | boolean
+
+	maxConnections?: number
+	registry?: Registry
+}
+
+export type ServiceMap<Payload> = {
+	identify: Identify
+	ping: Ping
+	pubsub: PubSub<GossipsubEvents>
+	gossipLog: GossipLogService<Payload>
+	dht: KadDHT
+	rendezvous: RendezvousClient
+}

packages/gossiplog/src/libp2p/interface.ts

@@ -4,6 +4,8 @@ import { NetworkClient } from "@canvas-js/gossiplog/client"
 import { assert } from "@canvas-js/utils"
 
 import type { PlatformTarget } from "../interface.js"
+import { getLibp2p } from "./libp2p.js"
+import { getPrivateKey } from "./privateKey.js"
 
 const target: PlatformTarget = {
 	async connect(gossipLog, url, options = {}) {
@@ -22,7 +24,12 @@ const target: PlatformTarget = {
 	},
 
 	async startLibp2p(gossipLog, config) {
-		throw new Error("Cannot start libp2p in the browser")
+		let privateKey = config.privateKey
+		if (privateKey === undefined) {
+			privateKey = await getPrivateKey()
+		}
+
+		return await getLibp2p(gossipLog, { ...config, privateKey })
 	},
 }