Rewritten Var/values implementation based around flexbuffers

This basically rewrites most of the values layer to be based around
values and collections held in flexbuffers. (Flexbuffers are part of
the flatbuffers project, and are a kind of zero copy dynamic
flatbuffer.)

Part of a bigger effort to do the following:

  * Rip out custom serialization formats and replace with a
    combination of flatbuffers and flexbuffers
  * Same for all uses of bincode2
  * Clean up the Var interface more broadly

Along with this change came major cleanups generally to the interface
in the values layer:

  * Make all index-based operations take an argument stating whether
    they should operate on one-indexed (like MOO) or
    zero-indexed (like most languages). This cleans up the ad-hoc way
    the conversion was happening before, and makes it easier to use
    languages other than MOO, but using the same values layer.
  * Move printing of the literal representation of a value out into
    the `compiler` crate, making it clear that the literal printing is
    of a MOO value, and leaving room for other languages to have other
    representations, or to add different pretty printing /
    decompilation options.
  * Makes the type system interface more regular in general, providing
    traits to identify operations for scalar/associative/sequence
    classes of type.

To be done still:

  * Rip out use of bincode2 and custom serialization code and replace
    with schema'd flatbuffer entities.
  * There are definitely lots of excessive copies still happening, and
    there's likely much more clever things that can be happening
    inside Var, including switching between a 'building' and 'reading'
    mode such that a given value doesn't obtain its flexbuffered
    pickled form until first written to the database. Or something.

Cargo.toml

@@ -101,6 +101,8 @@ daumtils = { git = "https://github.com/rdaum/daumtils.git", version = "0.2.0" }
 decorum = "0.3" # For ordering & comparing our floats
 encoding_rs = "0.8.34"
 enum-primitive-derive = "0.3"
+flatbuffers = "24.3.25"
+flexbuffers = "2.0.0"
 im = "15.1"
 inventory = "0.3.15"
 itertools = "0.13.0"

crates/compiler/src/ast.rs

@@ -12,10 +12,10 @@
 // this program. If not, see <https://www.gnu.org/licenses/>.
 //
 
-use moor_values::var::Symbol;
+use moor_values::Symbol;
 use std::fmt::Display;
 
-use moor_values::var::Var;
+use moor_values::Var;
 
 /// The abstract syntax tree produced by the parser and converted by codegen into opcodes.
 use crate::names::UnboundName;

crates/compiler/src/builtins.rs

@@ -14,9 +14,9 @@
 
 use bincode::{Decode, Encode};
 use lazy_static::lazy_static;
-use moor_values::var::Symbol;
-use moor_values::var::VarType;
-use moor_values::var::VarType::TYPE_MAP;
+use moor_values::Symbol;
+use moor_values::VarType;
+use moor_values::VarType::TYPE_MAP;
 /// Global registry of built-in function names.
 use std::collections::HashMap;
 use ArgCount::{Q, U};

crates/compiler/src/codegen.rs

@@ -18,8 +18,8 @@ use std::sync::Arc;
 
 use tracing::error;
 
-use moor_values::var::Var;
-use moor_values::var::Variant;
+use moor_values::Var;
+use moor_values::Variant;
 
 use crate::ast::{
     Arg, BinaryOp, CatchCodes, Expr, ScatterItem, ScatterKind, Stmt, StmtNode, UnaryOp,

crates/compiler/src/codegen_tests.rs

@@ -21,9 +21,9 @@ mod tests {
     use crate::opcode::{ScatterArgs, ScatterLabel};
     use crate::CompileOptions;
     use moor_values::model::CompileError;
-    use moor_values::var::Error::{E_INVARG, E_INVIND, E_PERM, E_PROPNF, E_RANGE};
-    use moor_values::var::Objid;
-    use moor_values::var::Symbol;
+    use moor_values::Error::{E_INVARG, E_INVIND, E_PERM, E_PROPNF, E_RANGE};
+    use moor_values::Objid;
+    use moor_values::Symbol;
     use moor_values::SYSTEM_OBJECT;
 
     #[test]

crates/compiler/src/decompile.rs

@@ -12,8 +12,8 @@
 // this program. If not, see <https://www.gnu.org/licenses/>.
 //
 
-use moor_values::var::{v_err, v_int, v_none, v_objid, Var};
-use moor_values::var::{v_float, Variant};
+use moor_values::{v_err, v_int, v_none, v_objid, Var};
+use moor_values::{v_float, Variant};
 use std::collections::{HashMap, VecDeque};
 
 use crate::ast::{

crates/compiler/src/lib.rs

@@ -37,7 +37,7 @@ pub use crate::names::{Name, UnboundNames};
 pub use crate::opcode::{Op, ScatterLabel};
 pub use crate::parse::CompileOptions;
 pub use crate::program::{Program, EMPTY_PROGRAM};
-pub use crate::unparse::unparse;
+pub use crate::unparse::{to_literal, unparse};
 
 #[macro_use]
 extern crate pest_derive;

crates/compiler/src/names.rs

@@ -15,7 +15,7 @@
 use crate::GlobalName;
 use bincode::{Decode, Encode};
 use moor_values::model::CompileError;
-use moor_values::var::Symbol;
+use moor_values::Symbol;
 use std::collections::HashMap;
 use strum::IntoEnumIterator;
 

crates/compiler/src/opcode.rs

@@ -16,8 +16,8 @@ use crate::builtins::BuiltinId;
 use crate::labels::{Label, Offset};
 use crate::names::Name;
 use bincode::{Decode, Encode};
-use moor_values::var::Error;
-use moor_values::var::Objid;
+use moor_values::Error;
+use moor_values::Objid;
 
 #[derive(Clone, Debug, PartialEq, PartialOrd, Encode, Decode)]
 pub enum Op {

crates/compiler/src/parse.rs

@@ -19,18 +19,18 @@ use std::collections::HashMap;
 use std::rc::Rc;
 use std::str::FromStr;
 
-use moor_values::var::{v_none, Symbol};
 use moor_values::SYSTEM_OBJECT;
+use moor_values::{v_none, Symbol};
 use pest::pratt_parser::{Assoc, Op, PrattParser};
 pub use pest::Parser as PestParser;
 use tracing::{instrument, warn};
 
-use moor_values::var::Error::{
+use moor_values::Error::{
     E_ARGS, E_DIV, E_FLOAT, E_INVARG, E_INVIND, E_MAXREC, E_NACC, E_NONE, E_PERM, E_PROPNF,
     E_QUOTA, E_RANGE, E_RECMOVE, E_TYPE, E_VARNF, E_VERBNF,
 };
-use moor_values::var::Objid;
-use moor_values::var::{v_err, v_float, v_int, v_objid, v_str, v_string};
+use moor_values::Objid;
+use moor_values::{v_err, v_float, v_int, v_objid, v_str, v_string};
 
 use crate::ast::Arg::{Normal, Splice};
 use crate::ast::StmtNode::Scope;
@@ -1189,9 +1189,9 @@ pub fn unquote_str(s: &str) -> Result<String, CompileError> {
 
 #[cfg(test)]
 mod tests {
-    use moor_values::var::Error::{E_INVARG, E_PROPNF, E_VARNF};
-    use moor_values::var::{v_err, v_float, v_int, v_obj, v_str};
-    use moor_values::var::{v_none, Symbol};
+    use moor_values::Error::{E_INVARG, E_PROPNF, E_VARNF};
+    use moor_values::{v_err, v_float, v_int, v_obj, v_str};
+    use moor_values::{v_none, Symbol};
 
     use crate::ast::Arg::{Normal, Splice};
     use crate::ast::Expr::{Call, Id, Prop, Value, Verb};

crates/compiler/src/program.rs

@@ -15,10 +15,11 @@
 use crate::labels::{JumpLabel, Label};
 use crate::names::{Name, Names};
 use crate::opcode::Op;
+use crate::unparse::to_literal;
 use bincode::{Decode, Encode};
 use bytes::Bytes;
 use lazy_static::lazy_static;
-use moor_values::var::Var;
+use moor_values::Var;
 use moor_values::{AsByteBuffer, CountingWriter, DecodingError, EncodingError, BINCODE_CONFIG};
 use std::fmt::{Display, Formatter};
 use std::sync::Arc;
@@ -84,7 +85,7 @@ impl Display for Program {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         // Write literals indexed by their offset #
         for (i, l) in self.literals.iter().enumerate() {
-            writeln!(f, "L{}: {}", i, l.to_literal())?;
+            writeln!(f, "L{}: {}", i, to_literal(l))?;
         }
 
         // Write jump labels indexed by their offset & showing position & optional name