From 99118cc952d417fa3ed1b50caff039ac5ee12ea3 Mon Sep 17 00:00:00 2001 From: Louis Thibault Date: Fri, 2 Aug 2024 23:47:57 -0400 Subject: [PATCH 1/6] Stub: mount IPFS unixfs to wasm guest's filesystem. --- guest/fs.go | 23 +++++++++++++++++++++++ guest/guest.go | 2 +- guest/guest_test.go | 15 +++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 guest/fs.go create mode 100644 guest/guest_test.go diff --git a/guest/fs.go b/guest/fs.go new file mode 100644 index 0000000..41eaa84 --- /dev/null +++ b/guest/fs.go @@ -0,0 +1,23 @@ +package guest + +import ( + "errors" + "io/fs" +) + +var _ fs.FS = (*FS)(nil) + +type FS struct{} + +// Open opens the named file. +// +// When Open returns an error, it should be of type *PathError +// with the Op field set to "open", the Path field set to name, +// and the Err field describing the problem. +// +// Open should reject attempts to open names that do not satisfy +// ValidPath(name), returning a *PathError with Err set to +// ErrInvalid or ErrNotExist. +func (FS) Open(name string) (fs.File, error) { + return nil, errors.New("FS.Open::NOT IMPLEMENTED") +} diff --git a/guest/guest.go b/guest/guest.go index 9444692..1cd5746 100644 --- a/guest/guest.go +++ b/guest/guest.go @@ -35,7 +35,7 @@ func (c Config) Instanatiate(ctx context.Context, r wazero.Runtime) (api.Module, // WithArgs(). // WithEnv(). WithRandSource(rand.Reader). - // WithFS(). + WithFS(FS{ /* TODO: pass IPFS here and mount a UnixFS */ }). // WithFSConfig(). // WithStartFunctions(). // remove _start so that we can call it later WithStdin(c.Sys.Stdin()). diff --git a/guest/guest_test.go b/guest/guest_test.go new file mode 100644 index 0000000..3936555 --- /dev/null +++ b/guest/guest_test.go @@ -0,0 +1,15 @@ +package guest_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/wetware/go/guest" +) + +func TestFS(t *testing.T) { + t.Parallel() + + _, err := guest.FS{}.Open("") + require.EqualError(t, err, "FS.Open::NOT IMPLEMENTED") +} From d4394ace902c14fe799a453b8cca4bdba00b1b6b Mon Sep 17 00:00:00 2001 From: Louis Thibault Date: Sat, 3 Aug 2024 00:05:38 -0400 Subject: [PATCH 2/6] Add fs.File implementation. --- guest/fs.go | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/guest/fs.go b/guest/fs.go index 41eaa84..52f991f 100644 --- a/guest/fs.go +++ b/guest/fs.go @@ -1,13 +1,20 @@ package guest import ( + "context" "errors" "io/fs" + + "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/path" + iface "github.com/ipfs/kubo/core/coreiface" ) var _ fs.FS = (*FS)(nil) -type FS struct{} +type FS struct { + IPFS iface.CoreAPI +} // Open opens the named file. // @@ -18,6 +25,32 @@ type FS struct{} // Open should reject attempts to open names that do not satisfy // ValidPath(name), returning a *PathError with Err set to // ErrInvalid or ErrNotExist. -func (FS) Open(name string) (fs.File, error) { - return nil, errors.New("FS.Open::NOT IMPLEMENTED") +func (fs FS) Open(name string) (fs.File, error) { + p, err := path.NewPath(name) + if err != nil { + return nil, err + } + + n, err := fs.IPFS.Unixfs().Get(context.TODO(), p) + return fsNode{Node: n}, err +} + +// fsNode provides access to a single file. The fs.File interface is the minimum +// implementation required of the file. Directory files should also implement [ReadDirFile]. +// A file may implement io.ReaderAt or io.Seeker as optimizations. + +type fsNode struct { + files.Node +} + +func (n fsNode) Stat() (fs.FileInfo, error) { + return nil, errors.New("fsNode.Stat::NOT IMPLEMENTED") +} + +func (n fsNode) Read([]byte) (int, error) { + return 0, errors.New("fsNode.Read::NOT IMPLEMENTED") +} + +func (n fsNode) Close() error { + return n.Node.Close() } From c0dddc3ed041ec592dc085828f41191f006248bf Mon Sep 17 00:00:00 2001 From: Louis Thibault Date: Sat, 3 Aug 2024 01:18:48 -0400 Subject: [PATCH 3/6] Clean up guest package. --- guest/fs.go | 65 +++++++++++++++++++---------- guest/{guest_test.go => fs_test.go} | 5 ++- guest/guest.go | 2 +- 3 files changed, 48 insertions(+), 24 deletions(-) rename guest/{guest_test.go => fs_test.go} (64%) diff --git a/guest/fs.go b/guest/fs.go index 52f991f..071e27b 100644 --- a/guest/fs.go +++ b/guest/fs.go @@ -2,18 +2,27 @@ package guest import ( "context" - "errors" "io/fs" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" iface "github.com/ipfs/kubo/core/coreiface" + "github.com/pkg/errors" ) var _ fs.FS = (*FS)(nil) +// An FS provides access to a hierarchical file system. +// +// The FS interface is the minimum implementation required of the file system. +// A file system may implement additional interfaces, +// such as [ReadFileFS], to provide additional or optimized functionality. +// +// [testing/fstest.TestFS] may be used to test implementations of an FS for +// correctness. type FS struct { - IPFS iface.CoreAPI + UNIX iface.UnixfsAPI + Root path.Path } // Open opens the named file. @@ -23,34 +32,48 @@ type FS struct { // and the Err field describing the problem. // // Open should reject attempts to open names that do not satisfy -// ValidPath(name), returning a *PathError with Err set to -// ErrInvalid or ErrNotExist. -func (fs FS) Open(name string) (fs.File, error) { - p, err := path.NewPath(name) +// fs.ValidPath(name), returning a *fs.PathError with Err set to +// fs.ErrInvalid or fs.ErrNotExist. +func (f FS) Open(name string) (fs.File, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: errors.New("invalid path"), + } + } + + root, err := f.UNIX.Get(context.TODO(), f.Root) if err != nil { return nil, err } - n, err := fs.IPFS.Unixfs().Get(context.TODO(), p) - return fsNode{Node: n}, err -} + switch node := root.(type) { + case files.File: + return fileNode{File: node}, nil -// fsNode provides access to a single file. The fs.File interface is the minimum -// implementation required of the file. Directory files should also implement [ReadDirFile]. -// A file may implement io.ReaderAt or io.Seeker as optimizations. + case files.Directory: + defer node.Close() -type fsNode struct { - files.Node -} + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: errors.New("is a directory"), + } -func (n fsNode) Stat() (fs.FileInfo, error) { - return nil, errors.New("fsNode.Stat::NOT IMPLEMENTED") + default: + panic(node) // unhandled type + } } -func (n fsNode) Read([]byte) (int, error) { - return 0, errors.New("fsNode.Read::NOT IMPLEMENTED") +// fileNode provides access to a single file. The fs.File interface is the minimum +// implementation required of the file. Directory files should also implement [ReadDirFile]. +// A file may implement io.ReaderAt or io.Seeker as optimizations. + +type fileNode struct { + files.File } -func (n fsNode) Close() error { - return n.Node.Close() +func (n fileNode) Stat() (fs.FileInfo, error) { + return nil, errors.New("fileNode.Stat::NOT IMPLEMENTED") } diff --git a/guest/guest_test.go b/guest/fs_test.go similarity index 64% rename from guest/guest_test.go rename to guest/fs_test.go index 3936555..915eb27 100644 --- a/guest/guest_test.go +++ b/guest/fs_test.go @@ -2,6 +2,7 @@ package guest_test import ( "testing" + "testing/fstest" "github.com/stretchr/testify/require" "github.com/wetware/go/guest" @@ -10,6 +11,6 @@ import ( func TestFS(t *testing.T) { t.Parallel() - _, err := guest.FS{}.Open("") - require.EqualError(t, err, "FS.Open::NOT IMPLEMENTED") + err := fstest.TestFS(guest.FS{}) + require.NoError(t, err) } diff --git a/guest/guest.go b/guest/guest.go index 1cd5746..4d43944 100644 --- a/guest/guest.go +++ b/guest/guest.go @@ -35,7 +35,7 @@ func (c Config) Instanatiate(ctx context.Context, r wazero.Runtime) (api.Module, // WithArgs(). // WithEnv(). WithRandSource(rand.Reader). - WithFS(FS{ /* TODO: pass IPFS here and mount a UnixFS */ }). + WithFS(FS{UNIX: c.IPFS.Unixfs(), Root: c.Root}). // WithFSConfig(). // WithStartFunctions(). // remove _start so that we can call it later WithStdin(c.Sys.Stdin()). From ef7d0d6fd5aefafd4422a3d82c9820c3c21e48c2 Mon Sep 17 00:00:00 2001 From: Louis Thibault Date: Sat, 3 Aug 2024 21:03:23 -0400 Subject: [PATCH 4/6] Small refactor. Add stub for directory case. --- guest/fs.go | 26 ++++++++++++++------------ guest/guest.go | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/guest/fs.go b/guest/fs.go index 071e27b..94a0598 100644 --- a/guest/fs.go +++ b/guest/fs.go @@ -5,7 +5,6 @@ import ( "io/fs" "github.com/ipfs/boxo/files" - "github.com/ipfs/boxo/path" iface "github.com/ipfs/kubo/core/coreiface" "github.com/pkg/errors" ) @@ -21,8 +20,7 @@ var _ fs.FS = (*FS)(nil) // [testing/fstest.TestFS] may be used to test implementations of an FS for // correctness. type FS struct { - UNIX iface.UnixfsAPI - Root path.Path + IPFS iface.CoreAPI } // Open opens the named file. @@ -37,32 +35,36 @@ type FS struct { func (f FS) Open(name string) (fs.File, error) { if !fs.ValidPath(name) { return nil, &fs.PathError{ - Op: "open", + Op: "FS.Open", Path: name, Err: errors.New("invalid path"), } } - root, err := f.UNIX.Get(context.TODO(), f.Root) + path, err := f.IPFS.Name().Resolve(context.TODO(), name) if err != nil { return nil, err } - switch node := root.(type) { + node, err := f.IPFS.Unixfs().Get(context.TODO(), path) + if err != nil { + return nil, err + } + + switch n := node.(type) { case files.File: - return fileNode{File: node}, nil + return fileNode{File: n}, nil case files.Directory: - defer node.Close() - + defer n.Close() return nil, &fs.PathError{ - Op: "open", + Op: "FS.Open", Path: name, - Err: errors.New("is a directory"), + Err: errors.New("node is a directory"), } default: - panic(node) // unhandled type + panic(n) // unhandled type } } diff --git a/guest/guest.go b/guest/guest.go index 4d43944..bccf463 100644 --- a/guest/guest.go +++ b/guest/guest.go @@ -35,7 +35,7 @@ func (c Config) Instanatiate(ctx context.Context, r wazero.Runtime) (api.Module, // WithArgs(). // WithEnv(). WithRandSource(rand.Reader). - WithFS(FS{UNIX: c.IPFS.Unixfs(), Root: c.Root}). + WithFS(FS{IPFS: c.IPFS}). // WithFSConfig(). // WithStartFunctions(). // remove _start so that we can call it later WithStdin(c.Sys.Stdin()). From 90e90200a63978d75b1c1c91d90c9289ebf84c32 Mon Sep 17 00:00:00 2001 From: Louis Thibault Date: Sat, 3 Aug 2024 21:04:22 -0400 Subject: [PATCH 5/6] Fix top-level ww package. Compiles & runs. --- go.mod | 2 +- ww.go | 128 +++++++++++++++++++++++++++++---------------------------- 2 files changed, 66 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index 36ab0c2..a5fb503 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/libp2p/go-libp2p-kad-dht v0.25.2 github.com/lmittmann/tint v1.0.4 github.com/multiformats/go-multiaddr v0.12.3 + github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 github.com/tetratelabs/wazero v1.7.1 github.com/thejerf/suture/v4 v4.0.5 @@ -113,7 +114,6 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_golang v1.19.0 // indirect diff --git a/ww.go b/ww.go index 1cdfa1b..651be34 100644 --- a/ww.go +++ b/ww.go @@ -2,20 +2,22 @@ package ww import ( "context" + "crypto/rand" + "errors" "fmt" - "log/slog" + "io" + "os" + "reflect" - "capnproto.org/go/capnp/v3/rpc" + "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" iface "github.com/ipfs/kubo/core/coreiface" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" "github.com/tetratelabs/wazero" - wasi "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" "github.com/thejerf/suture/v4" "github.com/wetware/go/guest" - "github.com/wetware/go/system" - "github.com/wetware/go/vat" ) const Proto = "/ww/0.0.0" @@ -45,81 +47,81 @@ type Cluster struct { } func (c Cluster) String() string { - peer := c.Host.ID() - return fmt.Sprintf("Cluster{peer=%s}", peer) + return c.Config.NS } // Serve the cluster's root process func (c Cluster) Serve(ctx context.Context) error { - if c.Router == nil { - slog.WarnContext(ctx, "started with null router", - "ns", c.NS) - return nil - } - - if err := c.Router.Bootstrap(ctx); err != nil { + root, err := c.IPFS.Name().Resolve(ctx, c.NS) + if err != nil { return err } - root, err := c.IPFS.Name().Resolve(ctx, c.NS) + node, err := c.IPFS.Unixfs().Get(ctx, root) if err != nil { return err } + defer node.Close() - return c.ServeVat(ctx, root) -} - -func (c Cluster) ServeVat(ctx context.Context, root path.Path) error { r := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig(). - WithMemoryLimitPages(1024). // 64MB - WithCloseOnContextDone(true). - WithDebugInfoEnabled(c.Debug)) + WithCloseOnContextDone(true)) defer r.Close(ctx) - cl, err := wasi.Instantiate(ctx, r) - if err != nil { - return err - } - defer cl.Close(ctx) + switch n := node.(type) { + case files.File: + // assume it's a WASM file; run it + body := io.LimitReader(n, 2<<32) + b, err := io.ReadAll(body) + if err != nil { + return err + } - sys, err := system.Builder{ - // Host: c.Host, - // IPFS: c.IPFS, - }.Instantiate(ctx, r) - if err != nil { - return err - } - defer sys.Close(ctx) + r := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig(). + WithCloseOnContextDone(true)) + defer r.Close(ctx) - mod, err := guest.Config{ - IPFS: c.IPFS, - Root: root, - Sys: sys, - }.Instanatiate(ctx, r) - if err != nil { - return err - } - defer mod.Close(ctx) - - // Obtain the system client. This gives us an API to our root - // process. - client := sys.Boot(ctx, mod) - defer client.Release() - - net := vat.Config{ - Host: c.Host, - Proto: vat.ProtoFromModule(mod), - }.Build(ctx) - defer net.Release() - - for { - if conn, err := net.Accept(ctx, &rpc.Options{ - BootstrapClient: client.AddRef(), - Network: net, - }); err == nil { - go net.ServeConn(ctx, conn) - } else { + cl, err := wasi_snapshot_preview1.Instantiate(ctx, r) + if err != nil { + return err + } + defer cl.Close(ctx) + + cm, err := r.CompileModule(ctx, b) + if err != nil { return err } + defer cm.Close(ctx) + + mod, err := r.InstantiateModule(ctx, cm, wazero.NewModuleConfig(). + // WithArgs(). + // WithEnv(). + // WithNanosleep(). + // WithNanotime(). + // WithOsyield(). + // WithSysNanosleep(). + // WithSysNanotime(). + // WithSysWalltime(). + // WithWalltime(). + WithStartFunctions(). + WithFS(guest.FS{IPFS: c.IPFS}). + WithRandSource(rand.Reader). + WithStdin(os.Stdin). + WithStderr(os.Stderr). + WithStdout(os.Stdout). + WithName(c.NS)) + if err != nil { + return err + } + defer mod.Close(ctx) + + _, err = mod.ExportedFunction("_start").Call(ctx) + return err + + case files.Directory: + // TODO: look for a main.wasm and execute it + return errors.New("Cluster.Serve::TODO:implement directory handler") + + default: + return fmt.Errorf("unhandled type: %s", reflect.TypeOf(n)) } } From 922d6bc535c48a46041a769a19f7cddb3d50e497 Mon Sep 17 00:00:00 2001 From: Louis Thibault Date: Sat, 3 Aug 2024 21:17:18 -0400 Subject: [PATCH 6/6] Move fs code to system/ package. --- guest/guest.go | 65 ------------------------------------ {guest => system}/fs.go | 24 ++++++++----- {guest => system}/fs_test.go | 6 ++-- ww.go | 4 +-- 4 files changed, 20 insertions(+), 79 deletions(-) delete mode 100644 guest/guest.go rename {guest => system}/fs.go (85%) rename {guest => system}/fs_test.go (63%) diff --git a/guest/guest.go b/guest/guest.go deleted file mode 100644 index bccf463..0000000 --- a/guest/guest.go +++ /dev/null @@ -1,65 +0,0 @@ -package guest - -import ( - "context" - "crypto/rand" - "io" - "os" - - "github.com/ipfs/boxo/files" - "github.com/ipfs/boxo/path" - iface "github.com/ipfs/kubo/core/coreiface" - "github.com/tetratelabs/wazero" - "github.com/tetratelabs/wazero/api" -) - -type Config struct { - IPFS iface.CoreAPI - Root path.Path - Sys interface { - Stdin() io.Reader - } -} - -// Compile and instantiate the guest module from the namespace path. -// Note that CompiledModule is produced in an intermediate step, and -// that it is not closed until r is closed. -func (c Config) Instanatiate(ctx context.Context, r wazero.Runtime) (api.Module, error) { - cm, err := c.CompileModule(ctx, r) - if err != nil { - return nil, err - } - - return r.InstantiateModule(ctx, cm, wazero.NewModuleConfig(). - // WithName(). - // WithArgs(). - // WithEnv(). - WithRandSource(rand.Reader). - WithFS(FS{IPFS: c.IPFS}). - // WithFSConfig(). - // WithStartFunctions(). // remove _start so that we can call it later - WithStdin(c.Sys.Stdin()). - WithStdout(os.Stdout). // FIXME - WithStderr(os.Stderr). // FIXME - WithSysNanotime()) -} - -func (c Config) CompileModule(ctx context.Context, r wazero.Runtime) (wazero.CompiledModule, error) { - bytecode, err := c.LoadByteCode(ctx) - if err != nil { - return nil, err - } - - return r.CompileModule(ctx, bytecode) -} - -func (c Config) LoadByteCode(ctx context.Context) ([]byte, error) { - n, err := c.IPFS.Unixfs().Get(ctx, c.Root) - if err != nil { - return nil, err - } - defer n.Close() - - // FIXME: address the obvious DoS vector - return io.ReadAll(n.(files.File)) -} diff --git a/guest/fs.go b/system/fs.go similarity index 85% rename from guest/fs.go rename to system/fs.go index 94a0598..6226b06 100644 --- a/guest/fs.go +++ b/system/fs.go @@ -1,10 +1,11 @@ -package guest +package system import ( "context" "io/fs" "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/path" iface "github.com/ipfs/kubo/core/coreiface" "github.com/pkg/errors" ) @@ -20,7 +21,8 @@ var _ fs.FS = (*FS)(nil) // [testing/fstest.TestFS] may be used to test implementations of an FS for // correctness. type FS struct { - IPFS iface.CoreAPI + API iface.UnixfsAPI + Path path.Path } // Open opens the named file. @@ -33,20 +35,24 @@ type FS struct { // fs.ValidPath(name), returning a *fs.PathError with Err set to // fs.ErrInvalid or fs.ErrNotExist. func (f FS) Open(name string) (fs.File, error) { - if !fs.ValidPath(name) { + p, err := path.Join(f.Path, name) + if err != nil { return nil, &fs.PathError{ - Op: "FS.Open", + Op: "path.Join", Path: name, - Err: errors.New("invalid path"), + Err: err, } } - path, err := f.IPFS.Name().Resolve(context.TODO(), name) - if err != nil { - return nil, err + if !fs.ValidPath(p.String()) { + return nil, &fs.PathError{ + Op: "fs.ValidPath", + Path: name, + Err: errors.New("invalid path"), + } } - node, err := f.IPFS.Unixfs().Get(context.TODO(), path) + node, err := f.API.Get(context.TODO(), p) if err != nil { return nil, err } diff --git a/guest/fs_test.go b/system/fs_test.go similarity index 63% rename from guest/fs_test.go rename to system/fs_test.go index 915eb27..da78a41 100644 --- a/guest/fs_test.go +++ b/system/fs_test.go @@ -1,16 +1,16 @@ -package guest_test +package system_test import ( "testing" "testing/fstest" "github.com/stretchr/testify/require" - "github.com/wetware/go/guest" + "github.com/wetware/go/system" ) func TestFS(t *testing.T) { t.Parallel() - err := fstest.TestFS(guest.FS{}) + err := fstest.TestFS(system.FS{}) require.NoError(t, err) } diff --git a/ww.go b/ww.go index 651be34..34397a0 100644 --- a/ww.go +++ b/ww.go @@ -17,7 +17,7 @@ import ( "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" "github.com/thejerf/suture/v4" - "github.com/wetware/go/guest" + "github.com/wetware/go/system" ) const Proto = "/ww/0.0.0" @@ -103,7 +103,7 @@ func (c Cluster) Serve(ctx context.Context) error { // WithSysWalltime(). // WithWalltime(). WithStartFunctions(). - WithFS(guest.FS{IPFS: c.IPFS}). + WithFS(system.FS{API: c.IPFS.Unixfs()}). WithRandSource(rand.Reader). WithStdin(os.Stdin). WithStderr(os.Stderr).