feat: Allow lens runtime selection via config (#2684)

## Relevant issue(s)

Resolves #2683

## Description

Allows lens runtime selection via config/cli param.

Also adds CI jobs to the matrix to test wasmer and wazero.

Author: AndrewSisley

.github/workflows/test-and-upload-coverage.yml

@@ -34,25 +34,29 @@ jobs:
         client-type: [go, http, cli]
         database-type: [badger-file, badger-memory]
         mutation-type: [gql, collection-named, collection-save]
+        lens-type: [wasm-time]
         detect-changes: [false]
         database-encryption: [false]
         include:
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
+            lens-type: wasm-time
             detect-changes: true
             database-encryption: false
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
+            lens-type: wasm-time
             detect-changes: false
             database-encryption: true
           - os: macos-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
+            lens-type: wasm-time
             detect-changes: false
             database-encryption: false
 ## TODO: https://github.com/sourcenetwork/defradb/issues/2080
@@ -63,6 +67,20 @@ jobs:
 ##          mutation-type: collection-save
 ##          detect-changes: false
 ##          database-encryption: false
+          - os: ubuntu-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            lens-type: wazero
+            detect-changes: false
+            database-encryption: false
+          - os: ubuntu-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            lens-type: wasmer
+            detect-changes: false
+            database-encryption: false
 
     runs-on: ${{ matrix.os }}
 
@@ -80,6 +98,7 @@ jobs:
       DEFRA_BADGER_FILE: ${{ matrix.database-type == 'badger-file' }}
       DEFRA_BADGER_ENCRYPTION: ${{ matrix.database-encryption }}
       DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }}
+      DEFRA_LENS_TYPE: ${{ matrix.lens-type }}
 
     steps:
       - name: Checkout code into the directory

cli/server_dump.go

@@ -37,7 +37,7 @@ func MakeServerDumpCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP)
+			db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP, nil)
 			if err != nil {
 				return errors.Wrap("failed to initialize database", err)
 			}

cli/start.go

@@ -126,6 +126,7 @@ func MakeStartCommand() *cobra.Command {
 				http.WithAllowedOrigins(cfg.GetStringSlice("api.allowed-origins")...),
 				http.WithTLSCertPath(cfg.GetString("api.pubKeyPath")),
 				http.WithTLSKeyPath(cfg.GetString("api.privKeyPath")),
+				node.WithLensRuntime(node.LensRuntimeType(cfg.GetString("lens.runtime"))),
 			}
 
 			if cfg.GetString("datastore.store") != configStoreMemory {

client/lens.go

@@ -15,6 +15,8 @@ import (
 
 	"github.com/lens-vm/lens/host-go/config/model"
 	"github.com/sourcenetwork/immutable/enumerable"
+
+	"github.com/sourcenetwork/defradb/datastore"
 )
 
 // LensConfig represents the configuration of a Lens migration in Defra.
@@ -38,9 +40,18 @@ type LensConfig struct {
 	model.Lens
 }
 
+// TxnSource represents an object capable of constructing the transactions that
+// implicit-transaction registries need internally.
+type TxnSource interface {
+	NewTxn(context.Context, bool) (datastore.Txn, error)
+}
+
 // LensRegistry exposes several useful thread-safe migration related functions which may
 // be used to manage migrations.
 type LensRegistry interface {
+	// Init initializes the registry with the provided transaction source.
+	Init(TxnSource)
+
 	// SetMigration caches the migration for the given collection ID. It does not persist the migration in long
 	// term storage, for that one should call [Store.SetMigration(ctx, cfg)].
 	//

docs/config.md

@@ -111,3 +111,12 @@ Keyring backend to use. Defaults to `file`.
 
 - `file` Stores keys in encrypted files
 - `system` Stores keys in the OS managed keyring
+
+## `lens.runtime`
+
+The LensVM wasm runtime to run lens modules in.
+
+Possible values:
+- `wasm-time` (default): https://github.com/bytecodealliance/wasmtime-go
+- `wasmer` (windows not supported): https://github.com/wasmerio/wasmer-go
+- `wazero`: https://github.com/tetratelabs/wazero

go.mod

@@ -282,8 +282,10 @@ require (
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/tendermint/tm-db v0.6.7 // indirect
 	github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8 // indirect
+	github.com/tetratelabs/wazero v1.5.0 // indirect
 	github.com/textileio/go-log/v2 v2.1.3-gke-2 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/wasmerio/wasmer-go v1.0.4 // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect

go.sum

@@ -1139,6 +1139,8 @@ github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu
 github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I=
 github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8 h1:RBkacARv7qY5laaXGlF4wFB/tk5rnthhPb8oIBGoagY=
 github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8/go.mod h1:9PdLyPiZIiW3UopXyRnPYyjUXSpiQNHRLu8fOsR3o8M=
+github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0=
+github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
 github.com/textileio/go-datastore-extensions v1.0.1 h1:qIJGqJaigQ1wD4TdwS/hf73u0HChhXvvUSJuxBEKS+c=
 github.com/textileio/go-datastore-extensions v1.0.1/go.mod h1:Pzj9FDRkb55910dr/FX8M7WywvnS26gBgEDez1ZBuLE=
 github.com/textileio/go-ds-badger3 v0.1.0 h1:q0kBuBmAcRUR3ClMSYlyw0224XeuzjjGinU53Qz1uXI=
@@ -1168,6 +1170,8 @@ github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsX
 github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
+github.com/wasmerio/wasmer-go v1.0.4 h1:MnqHoOGfiQ8MMq2RF6wyCeebKOe84G88h5yv+vmxJgs=
+github.com/wasmerio/wasmer-go v1.0.4/go.mod h1:0gzVdSfg6pysA6QVp6iVRPTagC6Wq9pOE8J86WKb2Fk=
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=

http/client_lens.go

@@ -35,6 +35,8 @@ type setMigrationRequest struct {
 	Config       model.Lens
 }
 
+func (w *LensRegistry) Init(txnSource client.TxnSource) {}
+
 func (c *LensRegistry) SetMigration(ctx context.Context, collectionID uint32, config model.Lens) error {
 	methodURL := c.http.baseURL.JoinPath("lens", "registry")
 

http/handler_ccip_test.go

@@ -193,7 +193,7 @@ func TestCCIPPost_WithInvalidBody(t *testing.T) {
 func setupDatabase(t *testing.T) client.DB {
 	ctx := context.Background()
 
-	cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, db.WithUpdateEvents())
+	cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, nil, db.WithUpdateEvents())
 	require.NoError(t, err)
 
 	_, err = cdb.AddSchema(ctx, `type User {

internal/db/config.go

@@ -11,7 +11,6 @@
 package db
 
 import (
-	"github.com/lens-vm/lens/host-go/engine/module"
 	"github.com/sourcenetwork/immutable"
 
 	"github.com/sourcenetwork/defradb/events"
@@ -40,19 +39,3 @@ func WithMaxRetries(num int) Option {
 		db.maxTxnRetries = immutable.Some(num)
 	}
 }
-
-// WithLensPoolSize sets the maximum number of cached migrations instances to preserve per schema version.
-//
-// Will default to `5` if not set.
-func WithLensPoolSize(size int) Option {
-	return func(db *db) {
-		db.lensPoolSize = immutable.Some(size)
-	}
-}
-
-// WithLensRuntime returns an option that sets the lens registry runtime.
-func WithLensRuntime(runtime module.Runtime) Option {
-	return func(db *db) {
-		db.lensRuntime = immutable.Some(runtime)
-	}
-}