From ea55d5987caa321fe2fcf142024c8aef96c39fad Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 2 Jul 2024 22:06:16 -0700 Subject: [PATCH 1/7] fix: strip path prefix in all cases when normalizing (#395) --- crates/core/src/api.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/crates/core/src/api.rs b/crates/core/src/api.rs index 52a6ba6ba..49f566f51 100644 --- a/crates/core/src/api.rs +++ b/crates/core/src/api.rs @@ -62,18 +62,12 @@ impl MatchResult { /// Make a path look the way provolone expects it to /// Removes leading "./", or the root path if it's provided fn normalize_path_in_project<'a>(path: &'a str, root_path: Option<&'a PathBuf>) -> &'a str { - #[cfg(debug_assertions)] if let Some(root_path) = root_path { - if !root_path.to_str().unwrap_or_default().ends_with('/') { - panic!( - "root_path '{}' must end with a slash.", - root_path.to_str().unwrap_or_default() - ); - } - } - if let Some(root_path) = root_path { - let root_path = root_path.to_str().unwrap(); - path.strip_prefix(root_path).unwrap_or(path) + let basic = path + .strip_prefix(root_path.to_string_lossy().as_ref()) + .unwrap_or(path); + // Stip the leading / if it's there + basic.strip_prefix('/').unwrap_or(basic) } else { path.strip_prefix("./").unwrap_or(path) } @@ -795,11 +789,13 @@ mod tests { } #[test] - #[should_panic] fn test_normalize_path_in_project_with_root_no_slash() { let path = "/home/user/project/src/main.rs"; let root_path = PathBuf::from("/home/user/project"); - normalize_path_in_project(path, Some(&root_path)); + assert_eq!( + normalize_path_in_project(path, Some(&root_path)), + "src/main.rs", + ); } #[test] From 265f882f5cccf79648e86691b07cf589d48a630a Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Wed, 3 Jul 2024 22:58:22 -0700 Subject: [PATCH 2/7] feat: add more info to match reasons (#396) --- crates/cli/src/commands/check.rs | 1 + crates/cli/src/result_formatting.rs | 40 +++++++++++++++++++++-------- crates/core/src/api.rs | 4 +++ crates/gritmodule/src/config.rs | 2 +- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/crates/cli/src/commands/check.rs b/crates/cli/src/commands/check.rs index 644bbf022..18ea79cd7 100644 --- a/crates/cli/src/commands/check.rs +++ b/crates/cli/src/commands/check.rs @@ -287,6 +287,7 @@ pub(crate) async fn run_check( let reason = Some(MatchReason { metadata_json: None, source: RewriteSource::Gritql, + title: result.pattern.title().map(|s| s.to_string()), name: Some(result.pattern.local_name.to_string()), level: Some(result.pattern.level()), explanation: None, diff --git a/crates/cli/src/result_formatting.rs b/crates/cli/src/result_formatting.rs index b3be4c155..2f3ee5e96 100644 --- a/crates/cli/src/result_formatting.rs +++ b/crates/cli/src/result_formatting.rs @@ -5,7 +5,7 @@ use core::fmt; use log::{debug, error, info, warn}; use marzano_core::api::{ AllDone, AnalysisLog, AnalysisLogLevel, CreateFile, DoneFile, FileMatchResult, InputFile, - Match, MatchResult, PatternInfo, RemoveFile, Rewrite, + Match, MatchReason, MatchResult, PatternInfo, RemoveFile, Rewrite, }; use marzano_core::constants::DEFAULT_FILE_NAME; use marzano_messenger::output_mode::OutputMode; @@ -137,6 +137,28 @@ pub fn get_human_error(mut log: AnalysisLog, input_pattern: &str) -> String { result } +/// Print a header for a match, with the path and (maybe) a title/explanation +pub fn print_file_header( + path: &str, + reason: &Option, + f: &mut fmt::Formatter<'_>, +) -> fmt::Result { + let path_title = path.bold(); + if let Some(r) = reason { + if let Some(title) = &r.title { + writeln!(f, "{}: {}", path_title, title)?; + } else { + writeln!(f, "{}", path_title)?; + } + if let Some(explanation) = &r.explanation { + writeln!(f, " {}", explanation.italic())?; + } + } else { + writeln!(f, "{}", path_title)?; + } + Ok(()) +} + impl fmt::Display for FormattedResult { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -165,12 +187,12 @@ impl fmt::Display for FormattedResult { Ok(()) } FormattedResult::Match(m) => { - let path_title = m.file_name().bold(); - writeln!(f, "{}", path_title)?; + print_file_header(m.file_name(), &m.reason, f)?; + let source = m.content(); match source { Err(e) => { - writeln!(f, "Could not read file: {}", e)?; + writeln!(f, "Could not read flie: {}", e)?; return Ok(()); } Ok(source) => { @@ -232,24 +254,22 @@ impl fmt::Display for FormattedResult { item.original.source_file, item.rewritten.source_file ) }; - let path_title = path_name.bold(); - writeln!(f, "{}", path_title)?; + print_file_header(&path_name, &item.reason, f)?; + let result: MatchResult = item.clone().into(); let diff = format_result_diff(&result, None); write!(f, "{}", diff)?; Ok(()) } FormattedResult::CreateFile(item) => { - let path_title = item.file_name().bold(); + print_file_header(item.file_name(), &item.reason, f)?; let result: MatchResult = item.clone().into(); - writeln!(f, "{}", path_title)?; let diff = format_result_diff(&result, None); write!(f, "{}", diff)?; Ok(()) } FormattedResult::RemoveFile(item) => { - let path_title = item.file_name().bold(); - writeln!(f, "{}", path_title)?; + print_file_header(item.file_name(), &item.reason, f)?; let result: MatchResult = item.clone().into(); let diff = format_result_diff(&result, None); write!(f, "{}", diff)?; diff --git a/crates/core/src/api.rs b/crates/core/src/api.rs index 49f566f51..d8a1c4993 100644 --- a/crates/core/src/api.rs +++ b/crates/core/src/api.rs @@ -500,7 +500,11 @@ impl FileMatchResult for Rewrite { pub struct MatchReason { pub metadata_json: Option, pub source: RewriteSource, + /// The name of the pattern that matched, or another programmatic identifier pub name: Option, + /// A human-readable title for the match + pub title: Option, + /// A human-readable explanation of the match pub explanation: Option, pub level: Option, } diff --git a/crates/gritmodule/src/config.rs b/crates/gritmodule/src/config.rs index 6e1b066d8..c6accbbe8 100644 --- a/crates/gritmodule/src/config.rs +++ b/crates/gritmodule/src/config.rs @@ -78,7 +78,7 @@ pub struct GritDefinitionConfig { pub name: String, pub body: Option, #[serde(flatten)] - pub(crate) meta: GritPatternMetadata, + pub meta: GritPatternMetadata, #[serde(skip)] pub kind: Option, pub samples: Option>, From de9061013e185c7911d83aafc1171a61376c5444 Mon Sep 17 00:00:00 2001 From: Seren Kwok <86184664+seren5240@users.noreply.github.com> Date: Thu, 4 Jul 2024 20:52:10 +0100 Subject: [PATCH 3/7] feat: pattern files (#397) --- .../fixtures/pattern_files/.grit/grit.yaml | 12 + .../pattern_files/docs/guides/something.md | 82 +++++ .../docs/guides/version_5_upgrade.md | 82 +++++ crates/gritmodule/src/config.rs | 2 + crates/gritmodule/src/resolver.rs | 20 +- ...ds_patterns_from_custom_pattern_files.snap | 284 ++++++++++++++++++ crates/gritmodule/src/yaml.rs | 50 ++- 7 files changed, 522 insertions(+), 10 deletions(-) create mode 100644 crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml create mode 100644 crates/gritmodule/fixtures/pattern_files/docs/guides/something.md create mode 100644 crates/gritmodule/fixtures/pattern_files/docs/guides/version_5_upgrade.md create mode 100644 crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap diff --git a/crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml b/crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml new file mode 100644 index 000000000..818de7336 --- /dev/null +++ b/crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml @@ -0,0 +1,12 @@ +version: 0.0.1 +patterns: + - name: remove_console_error + level: error + body: | + engine marzano(0.1) + language js + + `console.error($_)` => . +pattern_files: + - docs/guides/version_5_upgrade.md + - docs/guides/something.md diff --git a/crates/gritmodule/fixtures/pattern_files/docs/guides/something.md b/crates/gritmodule/fixtures/pattern_files/docs/guides/something.md new file mode 100644 index 000000000..a4dca26bc --- /dev/null +++ b/crates/gritmodule/fixtures/pattern_files/docs/guides/something.md @@ -0,0 +1,82 @@ +--- +title: Compare `null` using `===` or `!==` +--- + +Comparing to `null` needs a type-checking operator (=== or !==), to avoid incorrect results when the value is `undefined`. + +tags: #good + +```grit +engine marzano(0.1) +language js + +// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by "==" or "!=". +// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns. +binary_expression($operator, $left, $right) where { + $operator <: or { "==" => `===` , "!=" => `!==` }, + or { $left <: `null`, $right <: `null`} +} + +``` + +``` + +``` + +## `$val == null` => `$val === null` + +```javascript +if (val == null) { + done(); +} +``` + +```typescript +if (val === null) { + done(); +} +``` + +## `$val != null` => `$val !== null` + +```javascript +if (val != null) { + done(); +} +``` + +```typescript +if (val !== null) { + done(); +} +``` + +## `$val != null` => `$val !== null` into `while` + +```javascript +while (val != null) { + did(); +} +``` + +```typescript +while (val !== null) { + did(); +} +``` + +## Do not change `$val === null` + +```javascript +if (val === null) { + done(); +} +``` + +## Do not change `$val !== null` + +``` +while (val !== null) { + doSomething(); +} +``` diff --git a/crates/gritmodule/fixtures/pattern_files/docs/guides/version_5_upgrade.md b/crates/gritmodule/fixtures/pattern_files/docs/guides/version_5_upgrade.md new file mode 100644 index 000000000..a4dca26bc --- /dev/null +++ b/crates/gritmodule/fixtures/pattern_files/docs/guides/version_5_upgrade.md @@ -0,0 +1,82 @@ +--- +title: Compare `null` using `===` or `!==` +--- + +Comparing to `null` needs a type-checking operator (=== or !==), to avoid incorrect results when the value is `undefined`. + +tags: #good + +```grit +engine marzano(0.1) +language js + +// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by "==" or "!=". +// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns. +binary_expression($operator, $left, $right) where { + $operator <: or { "==" => `===` , "!=" => `!==` }, + or { $left <: `null`, $right <: `null`} +} + +``` + +``` + +``` + +## `$val == null` => `$val === null` + +```javascript +if (val == null) { + done(); +} +``` + +```typescript +if (val === null) { + done(); +} +``` + +## `$val != null` => `$val !== null` + +```javascript +if (val != null) { + done(); +} +``` + +```typescript +if (val !== null) { + done(); +} +``` + +## `$val != null` => `$val !== null` into `while` + +```javascript +while (val != null) { + did(); +} +``` + +```typescript +while (val !== null) { + did(); +} +``` + +## Do not change `$val === null` + +```javascript +if (val === null) { + done(); +} +``` + +## Do not change `$val !== null` + +``` +while (val !== null) { + doSomething(); +} +``` diff --git a/crates/gritmodule/src/config.rs b/crates/gritmodule/src/config.rs index c6accbbe8..62dcb6564 100644 --- a/crates/gritmodule/src/config.rs +++ b/crates/gritmodule/src/config.rs @@ -35,12 +35,14 @@ pub struct GritGitHubConfig { #[derive(Debug, Deserialize)] pub struct GritConfig { pub patterns: Vec, + pub pattern_files: Option>, pub github: Option, } #[derive(Debug, Deserialize)] pub struct SerializedGritConfig { pub patterns: Vec, + pub pattern_files: Option>, pub github: Option, } diff --git a/crates/gritmodule/src/resolver.rs b/crates/gritmodule/src/resolver.rs index aeafdca76..3c8abaa86 100644 --- a/crates/gritmodule/src/resolver.rs +++ b/crates/gritmodule/src/resolver.rs @@ -535,7 +535,7 @@ async fn get_grit_files_for_module( Some(config) => { if let Some(module) = module { let repo_root = find_repo_root_from(repo_path).await?; - get_patterns_from_yaml(&config, module, &repo_root)? + get_patterns_from_yaml(&config, module, &repo_root, repo_dir).await? } else { vec![] } @@ -586,7 +586,7 @@ async fn resolve_patterns_for_module( Some(config) => { if let Some(module) = module { let repo_root = find_repo_root_from(repo_path).await?; - get_patterns_from_yaml(&config, module, &repo_root)? + get_patterns_from_yaml(&config, module, &repo_root, repo_dir).await? } else { vec![] } @@ -934,4 +934,20 @@ mod tests { resolved_patterns.sort_by(|a, b| a.language.to_string().cmp(&b.language.to_string())); assert_yaml_snapshot!(resolved_patterns); } + + #[tokio::test] + async fn finds_patterns_from_custom_pattern_files() { + let module_repo = ModuleRepo::from_host_repo("github.com", "getgrit/rewriter").unwrap(); + let repo_dir = "fixtures/pattern_files"; + let (mut resolved_patterns, errored_patterns) = + super::resolve_patterns(&module_repo, repo_dir, None) + .await + .unwrap(); + + assert_eq!(resolved_patterns.len(), 3); + assert_eq!(errored_patterns.len(), 0); + + resolved_patterns.sort_by(|a, b| a.local_name.cmp(&b.local_name)); + assert_yaml_snapshot!(resolved_patterns); + } } diff --git a/crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap b/crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap new file mode 100644 index 000000000..d656f9e97 --- /dev/null +++ b/crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap @@ -0,0 +1,284 @@ +--- +source: crates/gritmodule/src/resolver.rs +expression: resolved_patterns +--- +- config: + name: remove_console_error + body: "engine marzano(0.1)\nlanguage js\n\n`console.error($_)` => .\n" + level: error + title: ~ + description: ~ + tags: ~ + samples: ~ + path: ".grit/grit.yaml" + position: + line: 3 + column: 11 + raw: ~ + module: + type: Module + host: github.com + fullName: getgrit/rewriter + remote: "https://github.com/getgrit/rewriter.git" + providerName: github.com/getgrit/rewriter + localName: remove_console_error + body: "engine marzano(0.1)\nlanguage js\n\n`console.error($_)` => .\n" + kind: pattern + language: js + visibility: public +- config: + name: something + body: "engine marzano(0.1)\nlanguage js\n\n// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by \"==\" or \"!=\".\n// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns.\nbinary_expression($operator, $left, $right) where {\n $operator <: or { \"==\" => `===` , \"!=\" => `!==` },\n or { $left <: `null`, $right <: `null`}\n}\n\n" + level: info + title: "Compare `null` using `===` or `!==`" + description: "Comparing to `null` needs a type-checking operator (=== or !==), to avoid incorrect results when the value is `undefined`." + tags: ~ + samples: + - name: ~ + input: "\n" + output: ~ + input_range: + start: + line: 23 + column: 1 + end: + line: 24 + column: 1 + startByte: 657 + endByte: 658 + output_range: ~ + - name: "`$val == null` => `$val === null`" + input: "if (val == null) {\n done();\n}\n" + output: "if (val === null) {\n done();\n}\n" + input_range: + start: + line: 29 + column: 1 + end: + line: 32 + column: 1 + startByte: 715 + endByte: 746 + output_range: + start: + line: 35 + column: 1 + end: + line: 38 + column: 1 + startByte: 765 + endByte: 797 + - name: "`$val != null` => `$val !== null`" + input: "if (val != null) {\n done();\n}\n" + output: "if (val !== null) {\n done();\n}\n" + input_range: + start: + line: 43 + column: 1 + end: + line: 46 + column: 1 + startByte: 854 + endByte: 885 + output_range: + start: + line: 49 + column: 1 + end: + line: 52 + column: 1 + startByte: 904 + endByte: 936 + - name: "`$val != null` => `$val !== null` into `while`" + input: "while (val != null) {\n did();\n}\n" + output: "while (val !== null) {\n did();\n}\n" + input_range: + start: + line: 57 + column: 1 + end: + line: 60 + column: 1 + startByte: 1006 + endByte: 1039 + output_range: + start: + line: 63 + column: 1 + end: + line: 66 + column: 1 + startByte: 1058 + endByte: 1092 + - name: "Do not change `$val === null`" + input: "if (val === null) {\n done();\n}\n" + output: ~ + input_range: + start: + line: 71 + column: 1 + end: + line: 74 + column: 1 + startByte: 1145 + endByte: 1177 + output_range: ~ + - name: "Do not change `$val !== null`" + input: "while (val !== null) {\n doSomething();\n}\n" + output: ~ + input_range: + start: + line: 79 + column: 1 + end: + line: 82 + column: 1 + startByte: 1220 + endByte: 1262 + output_range: ~ + path: fixtures/pattern_files/docs/guides/something.md + position: + line: 10 + column: 1 + raw: + format: markdown + content: "---\ntitle: Compare `null` using `===` or `!==`\n---\n\nComparing to `null` needs a type-checking operator (=== or !==), to avoid incorrect results when the value is `undefined`.\n\ntags: #good\n\n```grit\nengine marzano(0.1)\nlanguage js\n\n// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by \"==\" or \"!=\".\n// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns.\nbinary_expression($operator, $left, $right) where {\n $operator <: or { \"==\" => `===` , \"!=\" => `!==` },\n or { $left <: `null`, $right <: `null`}\n}\n\n```\n\n```\n\n```\n\n## `$val == null` => `$val === null`\n\n```javascript\nif (val == null) {\n done();\n}\n```\n\n```typescript\nif (val === null) {\n done();\n}\n```\n\n## `$val != null` => `$val !== null`\n\n```javascript\nif (val != null) {\n done();\n}\n```\n\n```typescript\nif (val !== null) {\n done();\n}\n```\n\n## `$val != null` => `$val !== null` into `while`\n\n```javascript\nwhile (val != null) {\n did();\n}\n```\n\n```typescript\nwhile (val !== null) {\n did();\n}\n```\n\n## Do not change `$val === null`\n\n```javascript\nif (val === null) {\n done();\n}\n```\n\n## Do not change `$val !== null`\n\n```\nwhile (val !== null) {\n doSomething();\n}\n```\n" + module: + type: Module + host: github.com + fullName: getgrit/rewriter + remote: "https://github.com/getgrit/rewriter.git" + providerName: github.com/getgrit/rewriter + localName: something + body: "engine marzano(0.1)\nlanguage js\n\n// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by \"==\" or \"!=\".\n// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns.\nbinary_expression($operator, $left, $right) where {\n $operator <: or { \"==\" => `===` , \"!=\" => `!==` },\n or { $left <: `null`, $right <: `null`}\n}\n\n" + kind: pattern + language: js + visibility: public +- config: + name: version_5_upgrade + body: "engine marzano(0.1)\nlanguage js\n\n// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by \"==\" or \"!=\".\n// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns.\nbinary_expression($operator, $left, $right) where {\n $operator <: or { \"==\" => `===` , \"!=\" => `!==` },\n or { $left <: `null`, $right <: `null`}\n}\n\n" + level: info + title: "Compare `null` using `===` or `!==`" + description: "Comparing to `null` needs a type-checking operator (=== or !==), to avoid incorrect results when the value is `undefined`." + tags: ~ + samples: + - name: ~ + input: "\n" + output: ~ + input_range: + start: + line: 23 + column: 1 + end: + line: 24 + column: 1 + startByte: 657 + endByte: 658 + output_range: ~ + - name: "`$val == null` => `$val === null`" + input: "if (val == null) {\n done();\n}\n" + output: "if (val === null) {\n done();\n}\n" + input_range: + start: + line: 29 + column: 1 + end: + line: 32 + column: 1 + startByte: 715 + endByte: 746 + output_range: + start: + line: 35 + column: 1 + end: + line: 38 + column: 1 + startByte: 765 + endByte: 797 + - name: "`$val != null` => `$val !== null`" + input: "if (val != null) {\n done();\n}\n" + output: "if (val !== null) {\n done();\n}\n" + input_range: + start: + line: 43 + column: 1 + end: + line: 46 + column: 1 + startByte: 854 + endByte: 885 + output_range: + start: + line: 49 + column: 1 + end: + line: 52 + column: 1 + startByte: 904 + endByte: 936 + - name: "`$val != null` => `$val !== null` into `while`" + input: "while (val != null) {\n did();\n}\n" + output: "while (val !== null) {\n did();\n}\n" + input_range: + start: + line: 57 + column: 1 + end: + line: 60 + column: 1 + startByte: 1006 + endByte: 1039 + output_range: + start: + line: 63 + column: 1 + end: + line: 66 + column: 1 + startByte: 1058 + endByte: 1092 + - name: "Do not change `$val === null`" + input: "if (val === null) {\n done();\n}\n" + output: ~ + input_range: + start: + line: 71 + column: 1 + end: + line: 74 + column: 1 + startByte: 1145 + endByte: 1177 + output_range: ~ + - name: "Do not change `$val !== null`" + input: "while (val !== null) {\n doSomething();\n}\n" + output: ~ + input_range: + start: + line: 79 + column: 1 + end: + line: 82 + column: 1 + startByte: 1220 + endByte: 1262 + output_range: ~ + path: fixtures/pattern_files/docs/guides/version_5_upgrade.md + position: + line: 10 + column: 1 + raw: + format: markdown + content: "---\ntitle: Compare `null` using `===` or `!==`\n---\n\nComparing to `null` needs a type-checking operator (=== or !==), to avoid incorrect results when the value is `undefined`.\n\ntags: #good\n\n```grit\nengine marzano(0.1)\nlanguage js\n\n// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by \"==\" or \"!=\".\n// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns.\nbinary_expression($operator, $left, $right) where {\n $operator <: or { \"==\" => `===` , \"!=\" => `!==` },\n or { $left <: `null`, $right <: `null`}\n}\n\n```\n\n```\n\n```\n\n## `$val == null` => `$val === null`\n\n```javascript\nif (val == null) {\n done();\n}\n```\n\n```typescript\nif (val === null) {\n done();\n}\n```\n\n## `$val != null` => `$val !== null`\n\n```javascript\nif (val != null) {\n done();\n}\n```\n\n```typescript\nif (val !== null) {\n done();\n}\n```\n\n## `$val != null` => `$val !== null` into `while`\n\n```javascript\nwhile (val != null) {\n did();\n}\n```\n\n```typescript\nwhile (val !== null) {\n did();\n}\n```\n\n## Do not change `$val === null`\n\n```javascript\nif (val === null) {\n done();\n}\n```\n\n## Do not change `$val !== null`\n\n```\nwhile (val !== null) {\n doSomething();\n}\n```\n" + module: + type: Module + host: github.com + fullName: getgrit/rewriter + remote: "https://github.com/getgrit/rewriter.git" + providerName: github.com/getgrit/rewriter + localName: version_5_upgrade + body: "engine marzano(0.1)\nlanguage js\n\n// We use the syntax-tree node binary_expression to capture all expressions where $a and $b are operated on by \"==\" or \"!=\".\n// This code takes advantage of Grit's allowing us to nest rewrites inside match conditions and to match syntax-tree fields on patterns.\nbinary_expression($operator, $left, $right) where {\n $operator <: or { \"==\" => `===` , \"!=\" => `!==` },\n or { $left <: `null`, $right <: `null`}\n}\n\n" + kind: pattern + language: js + visibility: public diff --git a/crates/gritmodule/src/yaml.rs b/crates/gritmodule/src/yaml.rs index 123cecfbd..b93260af1 100644 --- a/crates/gritmodule/src/yaml.rs +++ b/crates/gritmodule/src/yaml.rs @@ -1,7 +1,10 @@ use anyhow::{bail, Result}; use grit_util::Position; use marzano_util::rich_path::RichFile; -use std::{collections::HashSet, path::Path}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; use tokio::fs; use crate::{ @@ -10,7 +13,7 @@ use crate::{ ModuleGritPattern, SerializedGritConfig, CONFIG_FILE_NAMES, REPO_CONFIG_DIR_NAME, }, fetcher::ModuleRepo, - parser::extract_relative_file_path, + parser::{extract_relative_file_path, get_patterns_from_file, PatternFileExt}, }; pub fn get_grit_config(source: &str, source_path: &str) -> Result { @@ -27,6 +30,7 @@ pub fn get_grit_config(source: &str, source_path: &str) -> Result { let new_config = GritConfig { github: serialized.github, + pattern_files: serialized.pattern_files, patterns: serialized .patterns .into_iter() @@ -37,10 +41,11 @@ pub fn get_grit_config(source: &str, source_path: &str) -> Result { Ok(new_config) } -pub fn get_patterns_from_yaml( +pub async fn get_patterns_from_yaml( file: &RichFile, source_module: &ModuleRepo, root: &Option, + repo_dir: &str, ) -> Result> { let mut config = get_grit_config(&file.content, &extract_relative_file_path(file, root))?; @@ -50,11 +55,38 @@ pub fn get_patterns_from_yaml( pattern.position = Some(Position::from_byte_index(&file.content, offset)); } - config + let patterns = config .patterns .into_iter() .map(|pattern| pattern_config_to_model(pattern, source_module)) - .collect() + .collect(); + + if config.pattern_files.is_none() { + return patterns; + } + + let mut patterns = patterns?; + let mut file_readers = Vec::new(); + + for pattern_file in config.pattern_files.unwrap() { + let pattern_file = PathBuf::from(repo_dir).join(&pattern_file); + let extension = PatternFileExt::from_path(&pattern_file); + if extension.is_none() { + continue; + } + let extension = extension.unwrap(); + file_readers.push(tokio::spawn(get_patterns_from_file( + pattern_file, + Some(source_module.clone()), + extension, + ))); + } + + for file_reader in file_readers { + patterns.extend(file_reader.await??); + } + + Ok(patterns) } pub fn extract_grit_modules(content: &str, path: &str) -> Result> { @@ -136,8 +168,8 @@ patterns: } } - #[test] - fn gets_module_patterns() { + #[tokio::test] + async fn gets_module_patterns() { let grit_yaml = RichFile { path: String::new(), content: r#"version: 0.0.1 @@ -163,7 +195,9 @@ github: .to_string(), }; let repo = Default::default(); - let patterns = get_patterns_from_yaml(&grit_yaml, &repo, &None).unwrap(); + let patterns = get_patterns_from_yaml(&grit_yaml, &repo, &None, "getgrit/rewriter") + .await + .unwrap(); assert_eq!(patterns.len(), 4); assert_yaml_snapshot!(patterns); } From bd58a1ba490ab9b779b97eca6aba7147da6391dc Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 4 Jul 2024 13:28:18 -0700 Subject: [PATCH 4/7] chore: switch other tests to use git cache (#399) --- .github/workflows/main.yaml | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 33179486d..bafe950db 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -10,11 +10,7 @@ jobs: test-rust: name: Rust tests (marzano) timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - os: [nscloud-ubuntu-22.04-amd64-8x32] - runs-on: ${{ matrix.os }} + runs-on: namespace-profile-standard-ubuntu22-amd64 permissions: contents: "read" id-token: "write" @@ -25,10 +21,8 @@ jobs: BUILD_PLATFORM: amd64 steps: - name: clone code - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 + uses: namespacelabs/nscloud-checkout-action@v2 + - run: nsc git-checkout update-submodules --mirror_base_path=${NSC_GIT_MIRROR} --repository_path=${GITHUB_WORKSPACE} --recurse - name: Install Protoc run: sudo apt-get install -y protobuf-compiler - name: install Rust @@ -74,19 +68,14 @@ jobs: test-rust-wasm: name: Rust wasm timeout-minutes: 15 - strategy: - fail-fast: false - runs-on: - - nscloud-ubuntu-22.04-amd64-8x32 + runs-on: namespace-profile-standard-ubuntu22-amd64 permissions: contents: "read" id-token: "write" steps: - name: clone code - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 + uses: namespacelabs/nscloud-checkout-action@v2 + - run: nsc git-checkout update-submodules --mirror_base_path=${NSC_GIT_MIRROR} --repository_path=${GITHUB_WORKSPACE} --recurse - name: install Rust uses: actions-rs/toolchain@v1 with: From 4165037d1d5cf30b60568c1367a30dcf1ad198fb Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 4 Jul 2024 13:29:59 -0700 Subject: [PATCH 5/7] chore: checkout stdlib too --- .github/workflows/main.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bafe950db..71f6ea762 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -91,18 +91,14 @@ jobs: test-stdlib: name: Test the standard library timeout-minutes: 30 - strategy: - fail-fast: false - runs-on: - - nscloud-ubuntu-22.04-amd64-4x16 permissions: contents: "read" id-token: "write" + runs-on: namespace-profile-standard-ubuntu22-amd64 steps: - name: clone code - uses: actions/checkout@v3 - with: - submodules: true + uses: namespacelabs/nscloud-checkout-action@v2 + - run: nsc git-checkout update-submodules --mirror_base_path=${NSC_GIT_MIRROR} --repository_path=${GITHUB_WORKSPACE} --recurse - name: install Rust uses: actions-rs/toolchain@v1 with: From f850d3aaaefeefedad2446a230728636632b1484 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 4 Jul 2024 13:34:39 -0700 Subject: [PATCH 6/7] feat: move pattern files to the main patterns array (#398) --- .../fixtures/pattern_files/.grit/grit.yaml | 5 ++- crates/gritmodule/src/config.rs | 20 +++++++++-- ...ds_patterns_from_custom_pattern_files.snap | 6 ++-- crates/gritmodule/src/yaml.rs | 36 ++++++++++++++----- 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml b/crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml index 818de7336..d58831a5d 100644 --- a/crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml +++ b/crates/gritmodule/fixtures/pattern_files/.grit/grit.yaml @@ -1,5 +1,7 @@ version: 0.0.1 patterns: + - file: ../docs/guides/version_5_upgrade.md + - file: ../docs/guides/something.md - name: remove_console_error level: error body: | @@ -7,6 +9,3 @@ patterns: language js `console.error($_)` => . -pattern_files: - - docs/guides/version_5_upgrade.md - - docs/guides/something.md diff --git a/crates/gritmodule/src/config.rs b/crates/gritmodule/src/config.rs index 62dcb6564..03735b8c7 100644 --- a/crates/gritmodule/src/config.rs +++ b/crates/gritmodule/src/config.rs @@ -32,17 +32,31 @@ pub struct GritGitHubConfig { pub reviewers: Vec, } +/// Represents a reference to an external pattern file #[derive(Debug, Deserialize)] +pub struct GritPatternFile { + pub file: PathBuf, +} + +/// Pure in-memory representation of the grit config +#[derive(Debug)] pub struct GritConfig { pub patterns: Vec, - pub pattern_files: Option>, + pub pattern_files: Option>, pub github: Option, } +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum GritPatternConfig { + File(GritPatternFile), + Pattern(GritSerializedDefinitionConfig), +} + +/// Compacted / serialized version of the GritConfig #[derive(Debug, Deserialize)] pub struct SerializedGritConfig { - pub patterns: Vec, - pub pattern_files: Option>, + pub patterns: Vec, pub github: Option, } diff --git a/crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap b/crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap index d656f9e97..1ac553545 100644 --- a/crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap +++ b/crates/gritmodule/src/snapshots/marzano_gritmodule__resolver__tests__finds_patterns_from_custom_pattern_files.snap @@ -12,7 +12,7 @@ expression: resolved_patterns samples: ~ path: ".grit/grit.yaml" position: - line: 3 + line: 5 column: 11 raw: ~ module: @@ -136,7 +136,7 @@ expression: resolved_patterns startByte: 1220 endByte: 1262 output_range: ~ - path: fixtures/pattern_files/docs/guides/something.md + path: ".grit/../docs/guides/something.md" position: line: 10 column: 1 @@ -264,7 +264,7 @@ expression: resolved_patterns startByte: 1220 endByte: 1262 output_range: ~ - path: fixtures/pattern_files/docs/guides/version_5_upgrade.md + path: ".grit/../docs/guides/version_5_upgrade.md" position: line: 10 column: 1 diff --git a/crates/gritmodule/src/yaml.rs b/crates/gritmodule/src/yaml.rs index b93260af1..eddf0dcdf 100644 --- a/crates/gritmodule/src/yaml.rs +++ b/crates/gritmodule/src/yaml.rs @@ -28,14 +28,31 @@ pub fn get_grit_config(source: &str, source_path: &str) -> Result { } }; + let mut patterns = Vec::new(); + let mut pattern_files = Vec::new(); + + for pattern in serialized.patterns.into_iter() { + match pattern { + crate::config::GritPatternConfig::File(file) => { + pattern_files.push(file); + } + crate::config::GritPatternConfig::Pattern(p) => { + patterns.push(GritDefinitionConfig::from_serialized( + p, + source_path.to_string(), + )); + } + } + } + let new_config = GritConfig { github: serialized.github, - pattern_files: serialized.pattern_files, - patterns: serialized - .patterns - .into_iter() - .map(|p| GritDefinitionConfig::from_serialized(p, source_path.to_string())) - .collect(), + pattern_files: if pattern_files.is_empty() { + None + } else { + Some(pattern_files) + }, + patterns, }; Ok(new_config) @@ -47,7 +64,8 @@ pub async fn get_patterns_from_yaml( root: &Option, repo_dir: &str, ) -> Result> { - let mut config = get_grit_config(&file.content, &extract_relative_file_path(file, root))?; + let grit_path = extract_relative_file_path(file, root); + let mut config = get_grit_config(&file.content, &grit_path)?; for pattern in config.patterns.iter_mut() { pattern.kind = Some(DefinitionKind::Pattern); @@ -69,7 +87,9 @@ pub async fn get_patterns_from_yaml( let mut file_readers = Vec::new(); for pattern_file in config.pattern_files.unwrap() { - let pattern_file = PathBuf::from(repo_dir).join(&pattern_file); + let pattern_file = PathBuf::from(repo_dir) + .join(REPO_CONFIG_DIR_NAME) + .join(&pattern_file.file); let extension = PatternFileExt::from_path(&pattern_file); if extension.is_none() { continue; From 632cea02e947935ff55ad40fb8f6a19a5237cc6e Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Fri, 5 Jul 2024 11:13:44 +0200 Subject: [PATCH 7/7] fix: Add .tfvars as a valid hcl file (#401) --- crates/language/src/target_language.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/language/src/target_language.rs b/crates/language/src/target_language.rs index 3dd0852db..6b6040dcf 100644 --- a/crates/language/src/target_language.rs +++ b/crates/language/src/target_language.rs @@ -205,7 +205,7 @@ impl PatternLanguage { PatternLanguage::Rust => &["rs"], PatternLanguage::Ruby => &["rb"], PatternLanguage::Solidity => &["sol"], - PatternLanguage::Hcl => &["hcl", "tf"], + PatternLanguage::Hcl => &["hcl", "tf", "tfvars"], PatternLanguage::Yaml => &["yaml", "yml"], PatternLanguage::Sql => &["sql"], PatternLanguage::Vue => &["vue"], @@ -260,7 +260,7 @@ impl PatternLanguage { "rs" => Some(Self::Rust), "rb" => Some(Self::Ruby), "sol" => Some(Self::Solidity), - "hcl" | "tf" => Some(Self::Hcl), + "hcl" | "tf" | "tfvars" => Some(Self::Hcl), "yaml" | "yml" => Some(Self::Yaml), "sql" => Some(Self::Sql), "vue" => Some(Self::Vue),