getgrit · morgante · Jul 10, 2024 · Jul 10, 2024 · Jul 10, 2024 · Jul 10, 2024
@@ -33,7 +33,7 @@
 	url = https://github.com/tree-sitter/tree-sitter-ruby
 [submodule "resources/language-submodules/tree-sitter-yaml"]
 	path = resources/language-submodules/tree-sitter-yaml
-	url = https://github.com/ikatyang/tree-sitter-yaml.git
+	url = https://github.com/tree-sitter-grammars/tree-sitter-yaml.git
 [submodule "resources/language-submodules/tree-sitter-hcl"]
 	path = resources/language-submodules/tree-sitter-hcl
 	url = https://github.com/MichaHoffmann/tree-sitter-hcl

@@ -8,29 +8,32 @@ Note that this codebase isn't yet extensively documented. If you get stuck, plea
 
 A high-level overview of tools you need to have installed:
 
-* Rust toolchain: for compiling the codebase. You'll need [`rustc`](https://rustup.rs/) v1.74 or newer.
-  * To create WASM builds, run `rustup target install wasm32-unknown-unknown`.
-* C/C++ compiler. macOS: [Xcode Command Line Tools](https://download.developer.apple.com/Developer_Tools/Command_Line_Tools_for_Xcode_15.3/Command_Line_Tools_for_Xcode_15.3.dmg) via `xcode-select --install`, Linux: [gcc](https://learnubuntu.com/install-gcc/), Windows: [Microsoft Visual C++](https://visualstudio.microsoft.com/vs/features/cplusplus/).
-* Emscripten: a C/C++ compiler toolchain for WASM. Install v3.1.56 with [`emsdk`](https://emscripten.org/docs/getting_started/downloads.html).
-* Node.js runtime: `node`, `npm`, `npx` are used to generate parsers from `grammar.js` files. You'll need [`node`](https://nodejs.org/en/download) v18.5.0 or newer.
-* Yarn package manager. You'll need [`yarn`](https://classic.yarnpkg.com/en/docs/install) (classic). Install v1.22.19 with `npm install --global yarn`.
-* Tree-Sitter CLI: provides [`tree-sitter`](https://github.com/tree-sitter/tree-sitter/tree/master/cli) binary for testing grammars. Install v0.22.2 with `npm install --global tree-sitter-cli`.
-* Terraform CLI. Install [`terraform`](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) with `brew tap hashicorp/tap && brew install hashicorp/tap/terraform`. 
+- Rust toolchain: for compiling the codebase. You'll need [`rustc`](https://rustup.rs/) v1.74 or newer.
+  - To create WASM builds, run `rustup target install wasm32-unknown-unknown`.
+- C/C++ compiler. macOS: [Xcode Command Line Tools](https://download.developer.apple.com/Developer_Tools/Command_Line_Tools_for_Xcode_15.3/Command_Line_Tools_for_Xcode_15.3.dmg) via `xcode-select --install`, Linux: [gcc](https://learnubuntu.com/install-gcc/), Windows: [Microsoft Visual C++](https://visualstudio.microsoft.com/vs/features/cplusplus/).
+- Emscripten: a C/C++ compiler toolchain for WASM. Install v3.1.56 with [`emsdk`](https://emscripten.org/docs/getting_started/downloads.html).
+- Node.js runtime: `node`, `npm`, `npx` are used to generate parsers from `grammar.js` files. You'll need [`node`](https://nodejs.org/en/download) v18.5.0 or newer.
+- Yarn package manager. You'll need [`yarn`](https://classic.yarnpkg.com/en/docs/install) (classic). Install v1.22.19 with `npm install --global yarn`.
+- Tree-Sitter CLI: provides [`tree-sitter`](https://github.com/tree-sitter/tree-sitter/tree/master/cli) binary for testing grammars. Install v0.22.2 with `npm install --global tree-sitter-cli`.
+- Terraform CLI. Install [`terraform`](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) with `brew tap hashicorp/tap && brew install hashicorp/tap/terraform`.
 
 ## Building the Code
 
-Use `git` to clone this repository into a location of your choice. 
+Use `git` to clone this repository into a location of your choice.
+
 ```bash
 git clone https://github.com/getgrit/gritql.git
 ```
 
 Change into the cloned repository and make sure all submodules are correctly set up, including any nested submodules:
+
 ```bash
 cd gritql
 git submodule update --init --recursive
 ```
 
 Before making any changes to the code, make sure you can run the tests and everything is initially passing:
+
 ```bash
 cargo test --workspace
 ```
@@ -50,11 +53,18 @@ Features that should be tested in CI should be put into the `grit_ci` feature fl
 If GritQL is failing to match a code snippet, this can typically be fixed simply by adjusting the metavariable grammar for the target language.
 
 Metavariable grammars are found under [./resources/metavariable-grammars](./resources/metavariable-grammars). Typical fixes include:
+
 - Adding a new named field for a relevant node you want to manipulate.
 - Adding a `grit_metavariable` node as a choice in the corresponding spot where you want to substitute the metavariable.
 - Check [this guide](https://github.com/tree-sitter/tree-sitter/wiki/Tips-and-Tricks-for-a-grammar-author) to debug grammars generally.
 
-After making your changes, run the [./resources/edit_grammars.mjs](./resources/edit_grammars.mjs) script to regenerate the matching grammar.
+After making your changes, run the [./resources/edit_grammars.mjs](./resources/edit_grammars.mjs) script to regenerate the matching grammar. You can specify the language you want to regenerate to speed up the process.
+
+For example:
+
+```
+node ./resources/edit_grammars.mjs yaml
+```
 
 ### Snippet contexts
 
@@ -64,7 +74,7 @@ Snippet contexts are defined by implementing the `snippet_context_strings` metho
 
 ## Adding a New Target Language
 
-Note: Grit involves *two* languages:
+Note: Grit involves _two_ languages:
 
 - GritQL is [our query language](https://docs.grit.io/language/reference) for searching and transforming codebases.
 - The “target language” is the language we are transforming (ex. Python). This document describes the process of adding new target languages to Grit.
@@ -77,8 +87,8 @@ Here are the steps for adding a new target language:
 1. Find a tree sitter grammar for the language and add it as a submodule under `resources/language-submodules`.
 2. Add a simple parse test in `crates/core/src/test.rs` to ensure that the grammar is working.
 3. Copy the grammar file into `resources/metavariable-grammars`. This alternative grammar is used for parsing `snippets` in GritQL.
-4. Patch the metavariable grammar to include  `$.grit_metavariable` anywhere we want to substitute a metavariable. This is usually at least `$identifier` and `$literal`.
-    - For a snippet to match, it also needs to be a field. Often you’ll want to wrap `$thing` like: `field('thing', choice($.grit_metavariable, $thing))`
+4. Patch the metavariable grammar to include `$.grit_metavariable` anywhere we want to substitute a metavariable. This is usually at least `$identifier` and `$literal`.
+   - For a snippet to match, it also needs to be a field. Often you’ll want to wrap `$thing` like: `field('thing', choice($.grit_metavariable, $thing))`
 5. Add a new language implementation in `crates/core/languages/src`. This involves implementing the `Language` trait and adding a new `Language` enum variant.
 6. Add `snippet_context_strings` [like this](https://github.com/getgrit/gritql/blob/main/crates/language/src/sql.rs#L52) to provide context for snippets to match in.
 7. Add test cases for the language in `crates/core/src/test.rs`. This is a good time to add a few dozen test cases to ensure that the language is parsed correctly, and that the metavariable grammar is working.
@@ -88,10 +98,10 @@ Here are the steps for adding a new target language:
 These steps are done in our cloud environment and are not necessary for contributors to do.
 
 - grep for an existing language like `Sol` for solidity, and add it to all the `Language` enums you find.
-    - Add the language to `apps/web/src/views/project/details.tsx`, so repos with this language don’t get an “unsupported language” warning. (5 minutes)
-    - LSP target languages list: https://github.com/getgrit/rewriter/pull/7734/files#diff-f9d4f097b08d33241c5c8d15a2fbde0e37086c265ce0eba8decac20d5cd989c6R23
-    - VS Code client list: https://github.com/getgrit/rewriter/blob/f992490394a4807789504f1cea6a04b934ad3b24/apps/poolish/src/lsp-client.ts
-    - VS Code command palette triggers: https://github.com/getgrit/rewriter/pull/7734/files#diff-b38f1d6304993a250903310722206e6c89c58c52c2d1bd4b6fdd8f7218810570R103
+  - Add the language to `apps/web/src/views/project/details.tsx`, so repos with this language don’t get an “unsupported language” warning. (5 minutes)
+  - LSP target languages list: https://github.com/getgrit/rewriter/pull/7734/files#diff-f9d4f097b08d33241c5c8d15a2fbde0e37086c265ce0eba8decac20d5cd989c6R23
+  - VS Code client list: https://github.com/getgrit/rewriter/blob/f992490394a4807789504f1cea6a04b934ad3b24/apps/poolish/src/lsp-client.ts
+  - VS Code command palette triggers: https://github.com/getgrit/rewriter/pull/7734/files#diff-b38f1d6304993a250903310722206e6c89c58c52c2d1bd4b6fdd8f7218810570R103
 - There are also `exhaustive` runtime checks that error if a switch case doesn’t handle a language, like `makeSingleLineComment`. Search for `exhaustive(lang` and fill those out too.
 - Regenerate both DB/prisma types to add it to the DB schema and GraphQL types.
 - Add the language to `language-selector.tsx`. Pick an icon from [https://react-icons.github.io](https://react-icons.github.io/), usually from the Simple Icons category.

@@ -153,4 +153,21 @@ mod tests {
         }
         assert!(!nodes.is_empty());
     }
+
+    #[test]
+    fn simple_yaml_metavariable() {
+        let snippet = "$list";
+        let lang = Yaml::new(None);
+        let snippets = lang.parse_snippet_contexts(snippet);
+        let nodes = nodes_from_indices(&snippets);
+        let mut found_metavar = false;
+        for node in &nodes {
+            print_node(&node.node);
+            if node.node.kind_id() == lang.metavariable_sort() {
+                found_metavar = true;
+            }
+        }
+        assert!(!nodes.is_empty());
+        assert!(found_metavar);
+    }
 }
@@ -92,13 +92,13 @@ const copyMvGrammar = async (lang, dest) => {
   if (languagesWithoutMetaVariables.includes(lang)) {
     return;
   }
-  await fs.copyFile(
-    `${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-grammar.js`,
-    path.join(
-      LANGUAGE_METAVARIABLES_DIR,
-      `tree-sitter-${dest ?? lang}/grammar.js`
-    )
+  let from = `${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-grammar.js`;
+  let to = path.join(
+    LANGUAGE_METAVARIABLES_DIR,
+    `tree-sitter-${dest ?? lang}/grammar.js`
   );
+  await fs.copyFile(from, to);
+  console.log(`Copied ${from} to ${to}`);
-  let from = `${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-grammar.js`;
-  let to = path.join(
-    LANGUAGE_METAVARIABLES_DIR,
-    `tree-sitter-${dest ?? lang}/grammar.js`
-  );
-  await fs.copyFile(from, to);
-  console.log(`Copied ${from} to ${to}`);
+  let from = `${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-grammar.js`;
+  let to = path.join(
+    LANGUAGE_METAVARIABLES_DIR,
+    `tree-sitter-${dest ?? lang}/grammar.js`
+  );
+  try {
+    await fs.copyFile(from, to);
+    console.log(`Copied ${from} to ${to}`);
+  } catch (error) {
+    console.error(`Failed to copy ${from} to ${to}:`, error);
+  }
-  let from = `${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-grammar.js`;
-  let to = path.join(
-    LANGUAGE_METAVARIABLES_DIR,
-    `tree-sitter-${dest ?? lang}/grammar.js`
-  );
-  await fs.copyFile(from, to);
-  console.log(`Copied ${from} to ${to}`);
+  let from = `${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-grammar.js`;
+  let to = path.join(
+    LANGUAGE_METAVARIABLES_DIR,
+    `tree-sitter-${dest ?? lang}/grammar.js`
+  );
+  try {
+    await fs.copyFile(from, to);
+    console.log(`Copied ${from} to ${to}`);
+  } catch (error) {
+    console.error(`Failed to copy ${from} to ${to}:`, error);
+  }
 };
 
 /**
@@ -118,15 +118,6 @@ const copyMyBuild = async (c, lang, dest) =>
     )
   );
 
-const copyMvScanner = async (lang, dest) =>
-  fs.copyFile(
-    `${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-scanner.cc`,
-    path.join(
-      LANGUAGE_METAVARIABLES_DIR,
-      `tree-sitter-${dest ?? lang}/src/scanner.cc`
-    )
-  );
-
 const treeSitterGenerate = async (dir, buildWasm = true) => {
   const andMaybeBuildWasm = buildWasm ? "&& tree-sitter build-wasm " : "";
   await execPromise(
@@ -339,12 +330,12 @@ async function buildLanguage(language) {
       `${tsLangDir}/bindings/rust/build.rs`
     );
   } else if (language === "yaml") {
-    await copyMvScanner(language);
-    await buildSimpleLanguage(log, language);
     await fs.copyFile(
-      `${METAVARIABLE_GRAMMARS_DIR}/cc_build.rs`,
-      `${tsLangDir}/bindings/rust/build.rs`
+      `${METAVARIABLE_GRAMMARS_DIR}/${language}-metavariable-scanner.c`,
+      `${tsLangDir}/src/scanner.c`
     );
+    await buildSimpleLanguage(log, language);
+    await copyMyBuild("c", language);
-      `${METAVARIABLE_GRAMMARS_DIR}/${language}-metavariable-scanner.c`,
-      `${tsLangDir}/src/scanner.c`
-    );
-    await buildSimpleLanguage(log, language);
-    await copyMyBuild("c", language);
+    try {
+      await fs.copyFile(
+        `${METAVARIABLE_GRAMMARS_DIR}/${language}-metavariable-scanner.c`,
+        `${tsLangDir}/src/scanner.c`
+      );
+      await buildSimpleLanguage(log, language);
+      await copyMyBuild("c", language);
+    } catch (error) {
+      log(`Error handling YAML language build steps:`, error);
+    }
-      `${METAVARIABLE_GRAMMARS_DIR}/${language}-metavariable-scanner.c`,
-      `${tsLangDir}/src/scanner.c`
-    );
-    await buildSimpleLanguage(log, language);
-    await copyMyBuild("c", language);
+    try {
+      await fs.copyFile(
+        `${METAVARIABLE_GRAMMARS_DIR}/${language}-metavariable-scanner.c`,
+        `${tsLangDir}/src/scanner.c`
+      );
+      await buildSimpleLanguage(log, language);
+      await copyMyBuild("c", language);
+    } catch (error) {
+      log(`Error handling YAML language build steps:`, error);
+    }
   } else if (language === "hcl") {
     //HCL's mv grammar goes into `make_grammar.js`, not `grammar.js`
     await fs.copyFile(

@@ -3,10 +3,38 @@ root = true
 [*]
 charset = utf-8
 end_of_line = lf
-indent_size = 2
-indent_style = space
+max_line_length = 120
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[corpus/*]
-trim_trailing_whitespace = false
+[*.{json,toml,yml,gyp}]
+indent_style = space
+indent_size = 2
+
+[*.js]
+indent_style = space
+indent_size = 2
+
+[*.{c,cc,h}]
+indent_style = space
+indent_size = 4
+
+[*.rs]
+indent_style = space
+indent_size = 4
+
+[*.{py,pyi}]
+indent_style = space
+indent_size = 4
+
+[*.swift]
+indent_style = space
+indent_size = 4
+
+[*.go]
+indent_style = tab
+indent_size = 8
+
+[Makefile]
+indent_style = tab
+indent_size = 8
@@ -0,0 +1,11 @@
+* text eol=lf
+
+src/*.json linguist-generated
+src/parser.c linguist-generated
+src/tree_sitter/* linguist-generated
+
+bindings/** linguist-generated
+binding.gyp linguist-generated
+setup.py linguist-generated
+Makefile linguist-generated
+Package.swift linguist-generated
@@ -0,0 +1,38 @@
+# Rust artifacts
+Cargo.lock
+target/
+
+# Node artifacts
+build/
+prebuilds/
+node_modules/
+*.tgz
+
+# Swift artifacts
+.build/
+
+# Go artifacts
+go.sum
+_obj/
+
+# Python artifacts
+.venv/
+dist/
+*.egg-info
+*.whl
+
+# C artifacts
+*.a
+*.so
+*.so.*
+*.dylib
+*.dll
+*.pc
+
+# Example dirs
+/examples/*/
+
+# Grammar volatiles
+*.wasm
+*.obj
+*.o