Merge branch 'main' into list-output

.github/workflows/code-quality.yaml

@@ -1,5 +1,9 @@
 name: code quality
-on: "push"
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
 concurrency:
   group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.sha) || github.ref }}
   cancel-in-progress: true

.github/workflows/main.yaml

@@ -99,3 +99,42 @@ jobs:
         working-directory: ./crates/wasm-bindings
         run: |
           wasm-pack build --target web
+  test-stdlib:
+    name: Test the standard library
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+    runs-on:
+      - nscloud-ubuntu-22.04-amd64-4x16
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: clone code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2023-11-16
+          override: true
+      - name: build
+        run: |
+          cargo build --release
+      - name: Checkout js repo
+        uses: actions/checkout@v4
+        with:
+          repository: getgrit/stdlib
+          path: stdlib
+      - name: install-ruff
+        run: |
+          pip install ruff
+      - name: Setup Go environment
+        uses: actions/[email protected]
+        with:
+          go-version: '^1.22.0'
+      - name: test stdlib
+        working-directory: ./stdlib
+        run: |
+          ../target/release/marzano patterns test --exclude ai

.github/workflows/pr-lint.yaml

@@ -1,7 +1,7 @@
 name: pr-lint
 
 on:
-  pull_request_target:
+  pull_request:
     types:
       - opened
       - edited

.gitmodules

@@ -62,3 +62,6 @@
 [submodule "resources/language-submodules/tree-sitter-toml"]
 	path = resources/language-submodules/tree-sitter-toml
 	url = https://github.com/ikatyang/tree-sitter-toml
+[submodule "resources/language-submodules/tree-sitter-php"]
+	path = resources/language-submodules/tree-sitter-php
+	url = https://github.com/tree-sitter/tree-sitter-php

CONTRIBUTING.md

@@ -9,6 +9,37 @@ We welcome contributions in the form of pull requests and issues.
 
 Note that this codebase isn't yet extensively documented. If you get stuck, please ask for help [on Discord](https://docs.grit.io/discord).
 
+## Development Setup
+
+A high-level overview of tools you need to have installed:
+
+* Rust toolchain: for compiling the codebase. You'll need [`rustc`](https://rustup.rs/) v1.74 or newer.
+  * To create WASM builds, run `rustup target install wasm32-unknown-unknown`.
+* C/C++ compiler. macOS: [Xcode Command Line Tools](https://download.developer.apple.com/Developer_Tools/Command_Line_Tools_for_Xcode_15.3/Command_Line_Tools_for_Xcode_15.3.dmg) via `xcode-select --install`, Linux: [gcc](https://learnubuntu.com/install-gcc/), Windows: [Microsoft Visual C++](https://visualstudio.microsoft.com/vs/features/cplusplus/).
+* Emscripten: a C/C++ compiler toolchain for WASM. Install v3.1.56 with [`emsdk`](https://emscripten.org/docs/getting_started/downloads.html).
+* Node.js runtime: `node`, `npm`, `npx` are used to generate parsers from `grammar.js` files. You'll need [`node`](https://nodejs.org/en/download) v18.5.0 or newer.
+* Yarn package manager. You'll need [`yarn`](https://classic.yarnpkg.com/en/docs/install) (classic). Install v1.22.19 with `npm install --global yarn`.
+* Tree-Sitter CLI: provides [`tree-sitter`](https://github.com/tree-sitter/tree-sitter/tree/master/cli) binary for testing grammars. Install v0.22.2 with `npm install --global tree-sitter-cli`.
+* Terraform CLI. Install [`terraform`](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) with `brew tap hashicorp/tap && brew install hashicorp/tap/terraform`. 
+
+## Building the Code
+
+Use `git` to clone this repository into a location of your choice. 
+```bash
+git clone https://github.com/getgrit/gritql.git
+```
+
+Change into the cloned repository and make sure all submodules are correctly set up, including any nested submodules:
+```bash
+cd gritql
+git submodule update --init --recursive
+```
+
+Before making any changes to the code, make sure you can run the tests and everything is initially passing:
+```bash
+cargo test --workspace
+```
+
 ## Feature Flags
 
 We use [feature flags](https://doc.rust-lang.org/cargo/reference/features.html) to control which parts of the codebase are compiled.
@@ -53,7 +84,7 @@ Here are the steps for adding a new target language:
 3. Copy the grammar file into `resources/metavariable-grammars`. This alternative grammar is used for parsing `snippets` in GritQL.
 4. Patch the metavariable grammar to include  `$.grit_metavariable` anywhere we want to substitute a metavariable. This is usually at least `$identifier` and `$literal`.
     - For a snippet to match, it also needs to be a field. Often you’ll want to wrap `$thing` like: `field('thing', choice($.grit_metavariable, $thing))`
-5. Add a new language implementation in `crates/core/src/languages`. This involves implementing the `Language` trait and adding a new `Language` enum variant.
+5. Add a new language implementation in `crates/core/languages/src`. This involves implementing the `Language` trait and adding a new `Language` enum variant.
 6. Add `snippet_context_strings` [like this](https://github.com/getgrit/gritql/blob/main/crates/language/src/sql.rs#L52) to provide context for snippets to match in.
 7. Add test cases for the language in `crates/core/src/test.rs`. This is a good time to add a few dozen test cases to ensure that the language is parsed correctly, and that the metavariable grammar is working.
 

Cargo.toml

@@ -7,13 +7,11 @@ members = [
   "crates/language",
   "crates/gritmodule",
   "crates/gritcache",
+  "crates/grit-util",
   "crates/auth",
   "crates/externals",
   "crates/wasm-bindings",
-  "crates/marzano_messenger", "crates/cli_bin",
-]
-exclude = [
-  "resources",
-  "vendor/web-tree-sitter",
-  "vendor/tree-sitter-facade",
+  "crates/marzano_messenger",
+  "crates/cli_bin",
 ]
+exclude = ["resources", "vendor/web-tree-sitter", "vendor/tree-sitter-facade"]

crates/cli/src/commands/apply_pattern.rs

@@ -113,6 +113,8 @@ pub struct ApplyPatternArgs {
     /// Interpret the request as a natural language request
     #[clap(long)]
     ai: bool,
+    #[clap(long = "language", alias="lang")]
+    pub language: Option<PatternLanguage>,
 }
 
 impl Default for ApplyPatternArgs {
@@ -130,6 +132,7 @@ impl Default for ApplyPatternArgs {
             cache: Default::default(),
             refresh_cache: Default::default(),
             ai: Default::default(),
+            language: Default::default(),
         }
     }
 }
@@ -244,7 +247,8 @@ pub(crate) async fn run_apply_pattern(
         }
 
         let pattern_libs = flushable_unwrap!(emitter, get_grit_files_from_cwd().await);
-        let (lang, pattern_body) = if pattern.ends_with(".grit") || pattern.ends_with(".md") {
+        let (mut lang, pattern_body) = if pattern.ends_with(".grit") || pattern.ends_with(".md") {
+
             match fs::read_to_string(pattern.clone()).await {
                 Ok(pb) => {
                     if pattern.ends_with(".grit") {
@@ -303,6 +307,18 @@ pub(crate) async fn run_apply_pattern(
                 }
             }
         };
+        if let Some(lang_option) = &arg.language {
+            if let Some(lang) = lang {
+                if lang != *lang_option {
+                    return Err(anyhow::anyhow!(
+                        "Language option {} does not match pattern language {}",
+                        lang_option,
+                        lang
+                    ));
+                }
+            }
+            lang = Some(*lang_option);
+        }
         let pattern_libs = flushable_unwrap!(
             emitter,
             pattern_libs.get_language_directory_or_default(lang)

crates/cli_bin/fixtures/css_in_vue/pattern.grit

@@ -0,0 +1,3 @@
+language css
+
+`display: none;` => `display: some;`

crates/cli_bin/fixtures/css_in_vue/simple.vue

@@ -0,0 +1,44 @@
+<script lang="ts">
+export default {
+    inheritAttrs: false,
+};
+</script>
+
+<script setup lang="ts">
+defineProps<{
+    itemCount?: number;
+    showingCount?: boolean;
+}>();
+</script>
+
+<template>
+    <transition name="fade">
+        <span v-if="itemCount" class="item-count">
+            {{ showingCount }}
+        </span>
+    </transition>
+</template>
+
+<style lang="scss" scoped>
+.item-count {
+    position: relative;
+    display: none;
+    margin: 0 8px;
+    color: var(--theme--foreground-subdued);
+    white-space: nowrap;
+
+    @media (min-width: 600px) {
+        display: inline;
+    }
+}
+
+.fade-enter-active,
+.fade-leave-active {
+    transition: opacity var(--medium) var(--transition);
+}
+
+.fade-enter,
+.fade-leave-to {
+    opacity: 0;
+}
+</style>