feat(forth3): return interpreter actions from builtins

This branch changes the `forth3` API to prepare for Forth-driven IO. In
particular, builtin words and VM execution steps are changed from
returning a `Ok(())` branch when the builtin/execution step succeeds to
returning a new `InterpretAction` type in the `Ok` branch, which
determines what action the host should perform next.

`InterpretAction` is an enum which currently has three variants:

- `InterpretAction::Done`, which has the same semantics that returning
  `Ok(())` currently has: if returned from a builtin, it indicates that
  the VM should pop the current callstack frame and keep executing, and
  if returned by the VM to the host, it indicates that the VM's
  callstack is empty.
- `InterpretAction::Continue`, which replaces the
  `Error::PendingCallAgain` variant (it seemed more morally correct for
  this to be an `Ok` return rather than an `Err`), and indicates that
  the VM should continue executing what's currently on the stack when
  returned from a builtin
- `InterpretAction::AcceptInput`, which indicates that the VM needs to
  wait for the host to fill its input buffer.

In the future, `InterpretAction::AcceptInput` will be used to provide an
improved abstraction for communicating to the host that the VM is
requesting IO. We may also add a variant indicating that the VM needs
the host to drain its output buffer. When we add implementations of
Forth words such as `quit`, which fill the input buffer, they will
return the `AcceptInput` variant, allowing us to request IO from
builtins provided by `forth3` without requiring them to actually *do*
the IO. This lets us avoid making the entire VM generic over some `Read`
and `Write` traits (or `AsyncRead`/`AsyncWrite` traits), which don't
exist in no-std, by temporarily breaking out of a VM's run loop and
returning to the host when I/O is needed.

source/forth3/src/dictionary.rs

@@ -216,7 +216,7 @@ pub trait AsyncBuiltins<'forth, T: 'static> {
     /// VM. This allows the VM's stacks to be mutated by the async builtin function.
     ///
     /// [`Future`]: core::future::Future
-    type Future: core::future::Future<Output = Result<(), crate::Error>>;
+    type Future: core::future::Future<Output = Result<crate::vm::InterpretAction, crate::Error>>;
 
     /// A static slice of [`AsyncBuiltinEntry`]s describing the builtins
     /// provided by this implementation of `AsyncBuiltin`s.
@@ -690,7 +690,7 @@ pub mod test {
     use crate::{
         dictionary::{BuiltinEntry, DictLocation, DictionaryBump, DictionaryEntry},
         leakbox::{alloc_dict, LeakBox, LeakBoxDict},
-        Error, Forth, Word,
+        Forth, ForthResult, Word,
     };
 
     #[cfg(feature = "async")]
@@ -772,7 +772,7 @@ pub mod test {
 
     #[test]
     fn allocs_work() {
-        fn stubby(_f: &mut Forth<()>) -> Result<(), Error> {
+        fn stubby(_f: &mut Forth<()>) -> ForthResult {
             panic!("Don't ACTUALLY call me!");
         }
 
@@ -789,7 +789,7 @@ pub mod test {
 
     #[test]
     fn fork_onto_works() {
-        fn stubby(_f: &mut Forth<()>) -> Result<(), Error> {
+        fn stubby(_f: &mut Forth<()>) -> ForthResult {
             panic!("Don't ACTUALLY call me!");
         }
 

source/forth3/src/lib.rs

@@ -26,7 +26,7 @@ use dictionary::AsyncBuiltinEntry;
 
 #[cfg(feature = "async")]
 pub use crate::vm::AsyncForth;
-pub use crate::vm::Forth;
+pub use crate::vm::{Forth, ForthResult, InterpretAction};
 use crate::{
     dictionary::{BumpError, DictionaryEntry},
     output::OutputError,
@@ -80,10 +80,6 @@ pub enum Error {
     DivideByZero,
     AddrOfMissingName,
     AddrOfNotAWord,
-
-    // Not *really* an error - but signals that a function should be called
-    // again. At the moment, only used for internal interpreter functions.
-    PendingCallAgain,
 }
 
 impl From<StackError> for Error {
@@ -204,7 +200,7 @@ impl<T: 'static> CallContext<T> {
 ///
 /// It takes the current "full context" (e.g. `Fif`), as well as the CFA pointer
 /// to the dictionary entry.
-type WordFunc<T> = fn(&mut Forth<T>) -> Result<(), Error>;
+type WordFunc<T> = fn(&mut Forth<T>) -> ForthResult;
 
 pub enum Lookup<T: 'static> {
     Dict(DictLocation<T>),
@@ -259,8 +255,9 @@ pub mod test {
         dictionary::DictionaryEntry,
         leakbox::{LBForth, LBForthParams},
         testutil::{all_runtest, blocking_runtest_with},
+        vm,
         word::Word,
-        Error, Forth,
+        Error, Forth, ForthResult,
     };
 
     #[derive(Default)]
@@ -342,10 +339,10 @@ pub mod test {
         // assert_eq!(176, forth.dict_alloc.used());
 
         // Takes one value off the stack, and stores it in the vec
-        fn squirrel(forth: &mut Forth<TestContext>) -> Result<(), crate::Error> {
+        fn squirrel(forth: &mut Forth<TestContext>) -> Result<vm::InterpretAction, crate::Error> {
             let val = forth.data_stack.try_pop()?;
             forth.host_ctxt.contents.push(unsafe { val.data });
-            Ok(())
+            Ok(vm::InterpretAction::Done)
         }
         forth.add_builtin("squirrel", squirrel).unwrap();
 
@@ -701,7 +698,7 @@ pub mod test {
     }
 
     impl<'forth> Future for CountingFut<'forth> {
-        type Output = Result<(), Error>;
+        type Output = ForthResult;
 
         fn poll(
             mut self: core::pin::Pin<&mut Self>,
@@ -717,7 +714,7 @@ pub mod test {
                     let word = Word::data(self.ctr as i32);
                     self.forth.data_stack.push(word)?;
                     self.ctr += 1;
-                    Poll::Ready(Ok(()))
+                    Poll::Ready(Ok(vm::InterpretAction::Done))
                 }
                 Ordering::Greater => Poll::Ready(Err(Error::InternalError)),
             }

source/forth3/src/testutil/mod.rs

@@ -55,7 +55,7 @@
 
 use crate::{
     leakbox::{LBForth, LBForthParams},
-    Error, Forth,
+    vm, Error, Forth,
 };
 
 /// Run the given forth ui test against ALL enabled forth VMs
@@ -101,7 +101,7 @@ pub fn async_blockon_runtest(contents: &str) {
 
     struct TestAsyncDispatcher;
     impl<'forth> AsyncBuiltins<'forth, ()> for TestAsyncDispatcher {
-        type Future = futures::future::Ready<Result<(), Error>>;
+        type Future = futures::future::Ready<Result<vm::InterpretAction, Error>>;
         const BUILTINS: &'static [AsyncBuiltinEntry<()>] = &[];
         fn dispatch_async(&self, _id: &FaStr, _forth: &'forth mut Forth<()>) -> Self::Future {
             unreachable!("no async builtins should be called in this test")
@@ -151,12 +151,12 @@ where
     }
 }
 
-fn check_output(res: Result<(), Error>, outcome: &Outcome, output: &str) {
+fn check_output(res: Result<vm::InterpretAction, Error>, outcome: &Outcome, output: &str) {
     #[cfg(not(miri))]
     println!("< {output}");
     match (res, outcome) {
-        (Ok(()), Outcome::OkAnyOutput) => {}
-        (Ok(()), Outcome::OkWithOutput(exp)) => {
+        (Ok(_), Outcome::OkAnyOutput) => {}
+        (Ok(_), Outcome::OkWithOutput(exp)) => {
             let act_lines = output.lines().collect::<Vec<&str>>();
             assert_eq!(act_lines.len(), exp.len());
             act_lines.iter().zip(exp.iter()).for_each(|(a, e)| {

source/forth3/src/vm/async_vm.rs

@@ -180,22 +180,31 @@ where
         &mut self.vm
     }
 
-    pub async fn process_line(&mut self) -> Result<(), Error> {
+    pub async fn process_line(&mut self) -> ForthResult {
         let res = async {
             loop {
                 match self.vm.start_processing_line()? {
                     ProcessAction::Done => {
                         self.vm.output.push_str("ok.\n")?;
-                        break Ok(());
+                        break Ok(InterpretAction::Done);
                     }
                     ProcessAction::Continue => {}
-                    ProcessAction::Execute => while self.async_pig().await? != Step::Done {},
+                    ProcessAction::Execute => {
+                        // Loop until execution completes.
+                        let mut step = InterpretAction::Continue;
+                        while step == InterpretAction::Continue {
+                            step = self.async_pig().await?;
+                        }
+                        if step == InterpretAction::AcceptInput {
+                            return Ok(InterpretAction::AcceptInput);
+                        }
+                    }
                 }
             }
         }
         .await;
         match res {
-            Ok(_) => Ok(()),
+            Ok(res) => Ok(res),
             Err(e) => {
                 self.vm.data_stack.clear();
                 self.vm.return_stack.clear();
@@ -206,15 +215,15 @@ where
     }
 
     // Single step execution (async version).
-    async fn async_pig(&mut self) -> Result<Step, Error> {
+    async fn async_pig(&mut self) -> ForthResult {
         let Self {
             ref mut vm,
             ref builtins,
         } = self;
 
         let top = match vm.call_stack.try_peek() {
             Ok(t) => t,
-            Err(StackError::StackEmpty) => return Ok(Step::Done),
+            Err(StackError::StackEmpty) => return Ok(InterpretAction::Done),
             Err(e) => return Err(Error::Stack(e)),
         };
 
@@ -225,20 +234,19 @@ where
                 EntryKind::RuntimeBuiltin => (top.eh.cast::<BuiltinEntry<T>>().as_ref().func)(vm),
                 EntryKind::Dictionary => (top.eh.cast::<DictionaryEntry<T>>().as_ref().func)(vm),
                 EntryKind::AsyncBuiltin => builtins.dispatch_async(&top.eh.as_ref().name, vm).await,
-            }
+            }?
         };
 
-        match res {
-            Ok(_) => {
-                let _ = vm.call_stack.pop();
-            }
-            Err(Error::PendingCallAgain) => {
-                // ok, just don't pop
-            }
-            Err(e) => return Err(e),
+        if res != InterpretAction::Continue {
+            // current word completed, so remove it from the call stack.
+            let _ = self.vm.call_stack.pop();
+        }
+
+        if res == InterpretAction::Done {
+            return Ok(InterpretAction::Continue);
         }
 
-        Ok(Step::NotDone)
+        Ok(res)
     }
 
     pub fn release(self) -> T {