Merge branch 'php_features' of github.com:urbit-pilled/gritql into ph…

…p_features

.github/workflows/code-quality.yaml

@@ -2,7 +2,7 @@ name: code quality
 on:
   push:
     branches: [main]
-  pull_request_target:
+  pull_request:
     branches: [main]
 concurrency:
   group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.sha) || github.ref }}

.github/workflows/main.yaml

@@ -3,7 +3,7 @@ name: main
 on:
   push:
     branches: [main]
-  pull_request_target:
+  pull_request:
     branches: [main]
 
 jobs:
@@ -99,3 +99,42 @@ jobs:
         working-directory: ./crates/wasm-bindings
         run: |
           wasm-pack build --target web
+  test-stdlib:
+    name: Test the standard library
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+    runs-on:
+      - nscloud-ubuntu-22.04-amd64-4x16
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: clone code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2023-11-16
+          override: true
+      - name: build
+        run: |
+          cargo build --release
+      - name: Checkout js repo
+        uses: actions/checkout@v4
+        with:
+          repository: getgrit/stdlib
+          path: stdlib
+      - name: install-ruff
+        run: |
+          pip install ruff
+      - name: Setup Go environment
+        uses: actions/[email protected]
+        with:
+          go-version: '^1.22.0'
+      - name: test stdlib
+        working-directory: ./stdlib
+        run: |
+          ../target/release/marzano patterns test --exclude ai

.github/workflows/pr-lint.yaml

@@ -1,7 +1,7 @@
 name: pr-lint
 
 on:
-  pull_request_target:
+  pull_request:
     types:
       - opened
       - edited

CONTRIBUTING.md

@@ -9,6 +9,37 @@ We welcome contributions in the form of pull requests and issues.
 
 Note that this codebase isn't yet extensively documented. If you get stuck, please ask for help [on Discord](https://docs.grit.io/discord).
 
+## Development Setup
+
+A high-level overview of tools you need to have installed:
+
+* Rust toolchain: for compiling the codebase. You'll need [`rustc`](https://rustup.rs/) v1.74 or newer.
+  * To create WASM builds, run `rustup target install wasm32-unknown-unknown`.
+* C/C++ compiler. macOS: [Xcode Command Line Tools](https://download.developer.apple.com/Developer_Tools/Command_Line_Tools_for_Xcode_15.3/Command_Line_Tools_for_Xcode_15.3.dmg) via `xcode-select --install`, Linux: [gcc](https://learnubuntu.com/install-gcc/), Windows: [Microsoft Visual C++](https://visualstudio.microsoft.com/vs/features/cplusplus/).
+* Emscripten: a C/C++ compiler toolchain for WASM. Install v3.1.56 with [`emsdk`](https://emscripten.org/docs/getting_started/downloads.html).
+* Node.js runtime: `node`, `npm`, `npx` are used to generate parsers from `grammar.js` files. You'll need [`node`](https://nodejs.org/en/download) v18.5.0 or newer.
+* Yarn package manager. You'll need [`yarn`](https://classic.yarnpkg.com/en/docs/install) (classic). Install v1.22.19 with `npm install --global yarn`.
+* Tree-Sitter CLI: provides [`tree-sitter`](https://github.com/tree-sitter/tree-sitter/tree/master/cli) binary for testing grammars. Install v0.22.2 with `npm install --global tree-sitter-cli`.
+* Terraform CLI. Install [`terraform`](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) with `brew tap hashicorp/tap && brew install hashicorp/tap/terraform`. 
+
+## Building the Code
+
+Use `git` to clone this repository into a location of your choice. 
+```bash
+git clone https://github.com/getgrit/gritql.git
+```
+
+Change into the cloned repository and make sure all submodules are correctly set up, including any nested submodules:
+```bash
+cd gritql
+git submodule update --init --recursive
+```
+
+Before making any changes to the code, make sure you can run the tests and everything is initially passing:
+```bash
+cargo test --workspace
+```
+
 ## Feature Flags
 
 We use [feature flags](https://doc.rust-lang.org/cargo/reference/features.html) to control which parts of the codebase are compiled.
@@ -69,12 +100,3 @@ These steps are done in our cloud environment and are not necessary for contribu
 - There are also `exhaustive` runtime checks that error if a switch case doesn’t handle a language, like `makeSingleLineComment`. Search for `exhaustive(lang` and fill those out too.
 - Regenerate both DB/prisma types to add it to the DB schema and GraphQL types.
 - Add the language to `language-selector.tsx`. Pick an icon from [https://react-icons.github.io](https://react-icons.github.io/), usually from the Simple Icons category.
-
-## Development Tools
-
-Make sure you have the following tools installed to guarantee everything works:
-
-- Rust Toolchain
-  - In order to create WASM builds, you should run `rustup target install wasm32-unknown-unknown`
-- `terraform` CLI. See https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli
-- `npx`. Install with Node.js: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm

crates/core/src/binding.rs

@@ -36,6 +36,10 @@ impl Constant {
             Constant::Undefined => false,
         }
     }
+
+    pub(crate) fn is_undefined(&self) -> bool {
+        matches!(self, Self::Undefined)
+    }
 }
 
 impl Display for Constant {
@@ -80,16 +84,16 @@ pub enum Binding<'a> {
 impl PartialEq for Binding<'_> {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
-            (Binding::Empty(_, _, _), Binding::Empty(_, _, _)) => true,
-            (Binding::Node(src1, n1), Binding::Node(src2, n2)) => {
+            (Self::Empty(_, _, _), Self::Empty(_, _, _)) => true,
+            (Self::Node(src1, n1), Self::Node(src2, n2)) => {
                 n1.utf8_text(src1.as_bytes()) == n2.utf8_text(src2.as_bytes())
             }
-            (Binding::String(src1, r1), Binding::String(src2, r2)) => {
+            (Self::String(src1, r1), Self::String(src2, r2)) => {
                 src1[r1.start_byte as usize..r1.end_byte as usize]
                     == src2[r2.start_byte as usize..r2.end_byte as usize]
             }
-            (Binding::List(_, n1, f1), Binding::List(_, n2, f2)) => n1 == n2 && f1 == f2,
-            (Binding::ConstantRef(c1), Binding::ConstantRef(c2)) => c1 == c2,
+            (Self::List(_, n1, f1), Self::List(_, n2, f2)) => n1 == n2 && f1 == f2,
+            (Self::ConstantRef(c1), Self::ConstantRef(c2)) => c1 == c2,
             _ => false,
         }
     }
@@ -307,10 +311,22 @@ pub(crate) fn linearize_binding<'a>(
 }
 
 impl<'a> Binding<'a> {
+    pub(crate) fn from_constant(constant: &'a Constant) -> Self {
+        Self::ConstantRef(constant)
+    }
+
     pub(crate) fn from_node(node: NodeWithSource<'a>) -> Self {
         Self::Node(node.source, node.node)
     }
 
+    pub(crate) fn from_path(path: &'a Path) -> Self {
+        Self::FileName(path)
+    }
+
+    pub(crate) fn from_range(range: Range, source: &'a str) -> Self {
+        Self::String(source, range)
+    }
+
     /// Returns the only node bound by this binding.
     ///
     /// This includes list bindings that only match a single child.
@@ -339,8 +355,8 @@ impl<'a> Binding<'a> {
 
     pub fn get_sexp(&self) -> Option<String> {
         match self {
-            Binding::Node(_, node) => Some(node.to_sexp().to_string()),
-            Binding::List(_, parent_node, field_id) => {
+            Self::Node(_, node) => Some(node.to_sexp().to_string()),
+            Self::List(_, parent_node, field_id) => {
                 let mut cursor = parent_node.walk();
                 let mut children = parent_node.children_by_field_id(*field_id, &mut cursor);
                 let mut result = String::new();
@@ -353,20 +369,17 @@ impl<'a> Binding<'a> {
                 }
                 Some(result)
             }
-            Binding::String(..)
-            | Binding::FileName(_)
-            | Binding::Empty(..)
-            | Binding::ConstantRef(_) => None,
+            Self::String(..) | Self::FileName(_) | Self::Empty(..) | Self::ConstantRef(_) => None,
         }
     }
 
     // todo implement for empty and empty list
     pub fn position(&self) -> Option<Range> {
         match self {
-            Binding::Empty(_, _, _) => None,
-            Binding::Node(_, node) => Some(Range::from(node.range())),
-            Binding::String(_, range) => Some(range.to_owned()),
-            Binding::List(_, parent_node, field_id) => {
+            Self::Empty(_, _, _) => None,
+            Self::Node(_, node) => Some(Range::from(node.range())),
+            Self::String(_, range) => Some(range.to_owned()),
+            Self::List(_, parent_node, field_id) => {
                 let mut cursor = parent_node.walk();
                 let mut children = parent_node.children_by_field_id(*field_id, &mut cursor);
 
@@ -408,8 +421,8 @@ impl<'a> Binding<'a> {
                     }
                 }
             }
-            Binding::FileName(_) => None,
-            Binding::ConstantRef(_) => None,
+            Self::FileName(_) => None,
+            Self::ConstantRef(_) => None,
         }
     }
 
@@ -423,8 +436,8 @@ impl<'a> Binding<'a> {
         logs: &mut AnalysisLogs,
     ) -> Result<Cow<'a, str>> {
         let res: Result<Cow<'a, str>> = match self {
-            Binding::Empty(_, _, _) => Ok(Cow::Borrowed("")),
-            Binding::Node(source, node) => {
+            Self::Empty(_, _, _) => Ok(Cow::Borrowed("")),
+            Self::Node(source, node) => {
                 let range = CodeRange::from_node(source, node);
                 linearize_binding(
                     language,
@@ -440,11 +453,11 @@ impl<'a> Binding<'a> {
             }
             // can't linearize until we update source to point to the entire file
             // otherwise file file pointers won't match
-            Binding::String(s, r) => Ok(Cow::Owned(
+            Self::String(s, r) => Ok(Cow::Owned(
                 s[r.start_byte as usize..r.end_byte as usize].into(),
             )),
-            Binding::FileName(s) => Ok(Cow::Owned(s.to_string_lossy().into())),
-            Binding::List(source, _parent_node, _field_id) => {
+            Self::FileName(s) => Ok(Cow::Owned(s.to_string_lossy().into())),
+            Self::List(source, _parent_node, _field_id) => {
                 if let Some(pos) = self.position() {
                     let range = CodeRange::new(pos.start_byte, pos.end_byte, source);
                     linearize_binding(
@@ -462,37 +475,37 @@ impl<'a> Binding<'a> {
                     Ok("".into())
                 }
             }
-            Binding::ConstantRef(c) => Ok(Cow::Owned(c.to_string())),
+            Self::ConstantRef(c) => Ok(Cow::Owned(c.to_string())),
         };
         res
     }
 
     pub fn text(&self) -> String {
         match self {
-            Binding::Empty(_, _, _) => "".to_string(),
-            Binding::Node(source, node) => {
+            Self::Empty(_, _, _) => "".to_string(),
+            Self::Node(source, node) => {
                 NodeWithSource::new(node.clone(), source).text().to_string()
             }
-            Binding::String(s, r) => s[r.start_byte as usize..r.end_byte as usize].into(),
-            Binding::FileName(s) => s.to_string_lossy().into(),
-            Binding::List(source, _, _) => {
+            Self::String(s, r) => s[r.start_byte as usize..r.end_byte as usize].into(),
+            Self::FileName(s) => s.to_string_lossy().into(),
+            Self::List(source, _, _) => {
                 if let Some(pos) = self.position() {
                     source[pos.start_byte as usize..pos.end_byte as usize].to_string()
                 } else {
                     "".to_string()
                 }
             }
-            Binding::ConstantRef(c) => c.to_string(),
+            Self::ConstantRef(c) => c.to_string(),
         }
     }
 
     pub fn source(&self) -> Option<&'a str> {
         match self {
-            Binding::Empty(source, _, _) => Some(source),
-            Binding::Node(source, _) => Some(source),
-            Binding::String(source, _) => Some(source),
-            Binding::List(source, _, _) => Some(source),
-            Binding::FileName(..) | Binding::ConstantRef(..) => None,
+            Self::Empty(source, _, _) => Some(source),
+            Self::Node(source, _) => Some(source),
+            Self::String(source, _) => Some(source),
+            Self::List(source, _, _) => Some(source),
+            Self::FileName(..) | Self::ConstantRef(..) => None,
         }
     }
 
@@ -531,7 +544,7 @@ impl<'a> Binding<'a> {
     /// Returns an iterator over the items in a list.
     ///
     /// Returns `None` if the binding is not bound to a list.
-    pub(crate) fn list_items(&self) -> Option<impl Iterator<Item = NodeWithSource<'a>>> {
+    pub(crate) fn list_items(&self) -> Option<impl Iterator<Item = NodeWithSource<'a>> + Clone> {
         match self {
             Self::List(src, node, field_id) => {
                 let field_id = *field_id;

crates/core/src/pattern/after.rs

@@ -5,12 +5,11 @@ use super::{
     variable::VariableSourceLocations,
     Node, State,
 };
-use crate::{binding::Binding, context::Context, resolve};
 use crate::{binding::Constant, errors::debug};
+use crate::{context::Context, resolve};
 use anyhow::{anyhow, bail, Result};
 use core::fmt::Debug;
 use grit_util::AstNode;
-use im::vector;
 use marzano_util::analysis_logs::AnalysisLogs;
 use std::collections::BTreeMap;
 
@@ -61,7 +60,7 @@ impl After {
         };
 
         if let Some(next) = node.next_non_trivia_node() {
-            Ok(ResolvedPattern::Binding(vector![Binding::from_node(next)]))
+            Ok(ResolvedPattern::from_node(next))
         } else {
             debug(
                 logs,

crates/core/src/pattern/ast_node.rs

@@ -7,7 +7,7 @@ use super::{
     variable::VariableSourceLocations,
     State,
 };
-use crate::{binding::Binding, context::Context, resolve};
+use crate::{context::Context, resolve};
 use anyhow::{anyhow, Result};
 use itertools::Itertools;
 use marzano_language::language::{FieldId, Language, SortId};
@@ -163,12 +163,7 @@ impl Matcher for ASTNode {
             return Ok(false);
         };
         if binding.is_list() {
-            return self.execute(
-                &ResolvedPattern::from_binding(Binding::from_node(node)),
-                init_state,
-                context,
-                logs,
-            );
+            return self.execute(&ResolvedPattern::from_node(node), init_state, context, logs);
         }
 
         let NodeWithSource { node, source } = node;
@@ -181,10 +176,9 @@ impl Matcher for ASTNode {
         if context.language().is_comment(self.sort) {
             let content = context.language().comment_text(&node, source);
             let content = resolve!(content);
-            let content = Binding::String(source, content.1);
 
             return self.args[0].2.execute(
-                &ResolvedPattern::from_binding(content),
+                &ResolvedPattern::from_range(content.1, source),
                 init_state,
                 context,
                 logs,

crates/core/src/pattern/before.rs

@@ -5,12 +5,11 @@ use super::{
     variable::VariableSourceLocations,
     Node, State,
 };
-use crate::{binding::Binding, context::Context, resolve};
 use crate::{binding::Constant, errors::debug};
+use crate::{context::Context, resolve};
 use anyhow::{anyhow, bail, Result};
 use core::fmt::Debug;
 use grit_util::AstNode;
-use im::vector;
 use marzano_util::analysis_logs::AnalysisLogs;
 use std::collections::BTreeMap;
 
@@ -61,7 +60,7 @@ impl Before {
         };
 
         if let Some(prev) = node.previous_non_trivia_node() {
-            Ok(ResolvedPattern::Binding(vector![Binding::from_node(prev)]))
+            Ok(ResolvedPattern::from_node(prev))
         } else {
             debug(
                 logs,