WIP - Rework lens configuration

The node setup is far more powerful, and the new WithRuntime func will allow selecting the runtime via the CLI

.github/workflows/test-and-upload-coverage.yml

@@ -34,6 +34,7 @@ jobs:
         client-type: [go, http, cli]
         database-type: [badger-file, badger-memory]
         mutation-type: [gql, collection-named, collection-save]
+        lens-type: [wasm-time]
         detect-changes: [false]
         database-encryption: [false]
         include:
@@ -55,14 +56,21 @@ jobs:
             mutation-type: collection-save
             detect-changes: false
             database-encryption: false
-## TODO: https://github.com/sourcenetwork/defradb/issues/2080
-## Uncomment the lines below to Re-enable the windows build once this todo is resolved.
-##        - os: windows-latest
-##          client-type: go
-##          database-type: badger-memory
-##          mutation-type: collection-save
-##          detect-changes: false
-##          database-encryption: false
+          - os: windows-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            # Windows only works reliably in our CI due to https://github.com/sourcenetwork/defradb/issues/2080
+            lens-type: wazero
+            detect-changes: false
+            database-encryption: false
+          - os: ubuntu-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            lens-type: wasmer
+            detect-changes: false
+            database-encryption: false
 
     runs-on: ${{ matrix.os }}
 
@@ -80,6 +88,7 @@ jobs:
       DEFRA_BADGER_FILE: ${{ matrix.database-type == 'badger-file' }}
       DEFRA_BADGER_ENCRYPTION: ${{ matrix.database-encryption }}
       DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }}
+      DEFRA_LENS_TYPE: ${{ matrix.lens-type }}
 
     steps:
       - name: Checkout code into the directory

cli/server_dump.go

@@ -37,7 +37,7 @@ func MakeServerDumpCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP)
+			db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP, nil)
 			if err != nil {
 				return errors.Wrap("failed to initialize database", err)
 			}

cli/start.go

@@ -136,6 +136,16 @@ func MakeStartCommand() *cobra.Command {
 				opts = append(opts, node.WithACPPath(rootDir))
 			}
 
+			switch cfg.GetString("lens.runtime") {
+			case "wasm-time":
+				opts = append(opts, node.WasmTime)
+			case "wasmer":
+				opts = append(opts, node.Wasmer)
+			case "wazero":
+				opts = append(opts, node.Wazero)
+			default:
+			}
+
 			if !cfg.GetBool("keyring.disabled") {
 				kr, err := openKeyring(cmd)
 				if err != nil {

go.mod

@@ -282,8 +282,10 @@ require (
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/tendermint/tm-db v0.6.7 // indirect
 	github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8 // indirect
+	github.com/tetratelabs/wazero v1.5.0 // indirect
 	github.com/textileio/go-log/v2 v2.1.3-gke-2 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/wasmerio/wasmer-go v1.0.4 // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect

go.sum

@@ -1139,6 +1139,8 @@ github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu
 github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I=
 github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8 h1:RBkacARv7qY5laaXGlF4wFB/tk5rnthhPb8oIBGoagY=
 github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8/go.mod h1:9PdLyPiZIiW3UopXyRnPYyjUXSpiQNHRLu8fOsR3o8M=
+github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0=
+github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
 github.com/textileio/go-datastore-extensions v1.0.1 h1:qIJGqJaigQ1wD4TdwS/hf73u0HChhXvvUSJuxBEKS+c=
 github.com/textileio/go-datastore-extensions v1.0.1/go.mod h1:Pzj9FDRkb55910dr/FX8M7WywvnS26gBgEDez1ZBuLE=
 github.com/textileio/go-ds-badger3 v0.1.0 h1:q0kBuBmAcRUR3ClMSYlyw0224XeuzjjGinU53Qz1uXI=
@@ -1168,6 +1170,8 @@ github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsX
 github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
+github.com/wasmerio/wasmer-go v1.0.4 h1:MnqHoOGfiQ8MMq2RF6wyCeebKOe84G88h5yv+vmxJgs=
+github.com/wasmerio/wasmer-go v1.0.4/go.mod h1:0gzVdSfg6pysA6QVp6iVRPTagC6Wq9pOE8J86WKb2Fk=
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=

http/handler_ccip_test.go

@@ -193,7 +193,7 @@ func TestCCIPPost_WithInvalidBody(t *testing.T) {
 func setupDatabase(t *testing.T) client.DB {
 	ctx := context.Background()
 
-	cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, db.WithUpdateEvents())
+	cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 
 	_, err = cdb.AddSchema(ctx, `type User {

internal/db/config.go

@@ -11,7 +11,6 @@
 package db
 
 import (
-	"github.com/lens-vm/lens/host-go/engine/module"
 	"github.com/sourcenetwork/immutable"
 
 	"github.com/sourcenetwork/defradb/events"
@@ -40,19 +39,3 @@ func WithMaxRetries(num int) Option {
 		db.maxTxnRetries = immutable.Some(num)
 	}
 }
-
-// WithLensPoolSize sets the maximum number of cached migrations instances to preserve per schema version.
-//
-// Will default to `5` if not set.
-func WithLensPoolSize(size int) Option {
-	return func(db *db) {
-		db.lensPoolSize = immutable.Some(size)
-	}
-}
-
-// WithLensRuntime returns an option that sets the lens registry runtime.
-func WithLensRuntime(runtime module.Runtime) Option {
-	return func(db *db) {
-		db.lensRuntime = immutable.Some(runtime)
-	}
-}

internal/db/config_test.go

@@ -13,7 +13,6 @@ package db
 import (
 	"testing"
 
-	"github.com/lens-vm/lens/host-go/runtimes/wasmtime"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -29,15 +28,3 @@ func TestWithMaxRetries(t *testing.T) {
 	assert.True(t, d.maxTxnRetries.HasValue())
 	assert.Equal(t, 10, d.maxTxnRetries.Value())
 }
-
-func TestWithLensPoolSize(t *testing.T) {
-	d := &db{}
-	WithLensPoolSize(10)(d)
-	assert.Equal(t, 10, d.lensPoolSize.Value())
-}
-
-func TestWithLensRuntime(t *testing.T) {
-	d := &db{}
-	WithLensRuntime(wasmtime.New())(d)
-	assert.NotNil(t, d.lensRuntime.Value())
-}

internal/db/db.go

@@ -21,7 +21,6 @@ import (
 
 	ds "github.com/ipfs/go-datastore"
 	dsq "github.com/ipfs/go-datastore/query"
-	"github.com/lens-vm/lens/host-go/engine/module"
 
 	"github.com/sourcenetwork/corelog"
 	"github.com/sourcenetwork/immutable"
@@ -32,7 +31,6 @@ import (
 	"github.com/sourcenetwork/defradb/errors"
 	"github.com/sourcenetwork/defradb/events"
 	"github.com/sourcenetwork/defradb/internal/core"
-	"github.com/sourcenetwork/defradb/internal/lens"
 	"github.com/sourcenetwork/defradb/internal/request/graphql"
 )
 
@@ -57,10 +55,6 @@ type db struct {
 
 	parser core.Parser
 
-	// The maximum number of cached migrations instances to preserve per schema version.
-	lensPoolSize immutable.Option[int]
-	lensRuntime  immutable.Option[module.Runtime]
-
 	lensRegistry client.LensRegistry
 
 	// The maximum number of retries per transaction.
@@ -81,15 +75,17 @@ func NewDB(
 	ctx context.Context,
 	rootstore datastore.RootStore,
 	acp immutable.Option[acp.ACP],
+	lens client.LensRegistry,
 	options ...Option,
 ) (client.DB, error) {
-	return newDB(ctx, rootstore, acp, options...)
+	return newDB(ctx, rootstore, acp, lens, options...)
 }
 
 func newDB(
 	ctx context.Context,
 	rootstore datastore.RootStore,
 	acp immutable.Option[acp.ACP],
+	lens client.LensRegistry,
 	options ...Option,
 ) (*db, error) {
 	multistore := datastore.MultiStoreFrom(rootstore)
@@ -100,22 +96,19 @@ func newDB(
 	}
 
 	db := &db{
-		rootstore:  rootstore,
-		multistore: multistore,
-		acp:        acp,
-		parser:     parser,
-		options:    options,
+		rootstore:    rootstore,
+		multistore:   multistore,
+		acp:          acp,
+		lensRegistry: lens,
+		parser:       parser,
+		options:      options,
 	}
 
 	// apply options
 	for _, opt := range options {
 		opt(db)
 	}
 
-	// lens options may be set by `WithLens` funcs, and because they are funcs on db
-	// we have to mutate `db` here to set the registry.
-	db.lensRegistry = lens.NewRegistry(db, db.lensPoolSize, db.lensRuntime)
-
 	err = db.initialize(ctx)
 	if err != nil {
 		return nil, err

internal/db/db_test.go

@@ -26,7 +26,7 @@ func newMemoryDB(ctx context.Context) (*db, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newDB(ctx, rootstore, acp.NoACP)
+	return newDB(ctx, rootstore, acp.NoACP, nil)
 }
 
 func TestNewDB(t *testing.T) {
@@ -38,7 +38,7 @@ func TestNewDB(t *testing.T) {
 		return
 	}
 
-	_, err = NewDB(ctx, rootstore, acp.NoACP)
+	_, err = NewDB(ctx, rootstore, acp.NoACP, nil)
 	if err != nil {
 		t.Error(err)
 	}

internal/lens/registry.go

@@ -17,8 +17,6 @@ import (
 	"github.com/lens-vm/lens/host-go/config"
 	"github.com/lens-vm/lens/host-go/config/model"
 	"github.com/lens-vm/lens/host-go/engine/module"
-	"github.com/lens-vm/lens/host-go/runtimes/wasmtime"
-	"github.com/sourcenetwork/immutable"
 	"github.com/sourcenetwork/immutable/enumerable"
 
 	"github.com/sourcenetwork/defradb/client"
@@ -86,25 +84,18 @@ const DefaultPoolSize int = 5
 // It will be of size 5 (per schema version) if a size is not provided.
 func NewRegistry(
 	db TxnSource,
-	poolSize immutable.Option[int],
-	runtime immutable.Option[module.Runtime],
+	poolSize int,
+	runtime module.Runtime,
 ) client.LensRegistry {
 	registry := &lensRegistry{
-		poolSize:                    DefaultPoolSize,
-		runtime:                     wasmtime.New(),
+		poolSize:                    poolSize,
+		runtime:                     runtime,
 		modulesByPath:               map[string]module.Module{},
 		lensPoolsByCollectionID:     map[uint32]*lensPool{},
 		reversedPoolsByCollectionID: map[uint32]*lensPool{},
 		txnCtxs:                     map[uint64]*txnContext{},
 	}
 
-	if poolSize.HasValue() {
-		registry.poolSize = poolSize.Value()
-	}
-	if runtime.HasValue() {
-		registry.runtime = runtime.Value()
-	}
-
 	return &implicitTxnLensRegistry{
 		db:       db,
 		registry: registry,

net/node_test.go

@@ -36,15 +36,15 @@ func FixtureNewMemoryDBWithBroadcaster(t *testing.T) client.DB {
 	opts := badgerds.Options{Options: badger.DefaultOptions("").WithInMemory(true)}
 	rootstore, err := badgerds.NewDatastore("", &opts)
 	require.NoError(t, err)
-	database, err = db.NewDB(ctx, rootstore, acp.NoACP, db.WithUpdateEvents())
+	database, err = db.NewDB(ctx, rootstore, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 	return database
 }
 
 func TestNewNode_WithEnableRelay_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 	n, err := NewNode(
 		context.Background(),
@@ -59,7 +59,7 @@ func TestNewNode_WithDBClosed_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
 
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 	db.Close()
 
@@ -73,7 +73,7 @@ func TestNewNode_WithDBClosed_NoError(t *testing.T) {
 func TestNewNode_NoPubSub_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 	n, err := NewNode(
 		context.Background(),
@@ -88,7 +88,7 @@ func TestNewNode_NoPubSub_NoError(t *testing.T) {
 func TestNewNode_WithEnablePubSub_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 
 	n, err := NewNode(
@@ -106,7 +106,7 @@ func TestNewNode_WithEnablePubSub_NoError(t *testing.T) {
 func TestNodeClose_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 	n, err := NewNode(
 		context.Background(),
@@ -119,7 +119,7 @@ func TestNodeClose_NoError(t *testing.T) {
 func TestNewNode_BootstrapWithNoPeer_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 
 	n1, err := NewNode(
@@ -135,7 +135,7 @@ func TestNewNode_BootstrapWithNoPeer_NoError(t *testing.T) {
 func TestNewNode_BootstrapWithOnePeer_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 
 	n1, err := NewNode(
@@ -162,7 +162,7 @@ func TestNewNode_BootstrapWithOnePeer_NoError(t *testing.T) {
 func TestNewNode_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 
 	n1, err := NewNode(
@@ -192,7 +192,7 @@ func TestNewNode_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing
 func TestListenAddrs_WithListenAddresses_NoError(t *testing.T) {
 	ctx := context.Background()
 	store := memory.NewDatastore(ctx)
-	db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents())
+	db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 	n, err := NewNode(
 		context.Background(),