From 0b6ae6b5592e5a318b24c2105cc22999fb78e19f Mon Sep 17 00:00:00 2001 From: Nathan Byrd Date: Thu, 12 Aug 2021 15:18:44 -0500 Subject: [PATCH 1/5] Added ability to use regex rules --- docs/rules.md | 10 ++++++ docs/usage.md | 1 + pkg/printer/json_test.go | 4 +-- pkg/rule/options.go | 1 + pkg/rule/rule.go | 19 ++++++++--- pkg/rule/rule_test.go | 68 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 6 deletions(-) diff --git a/docs/rules.md b/docs/rules.md index 70aca199..e8b3fad5 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -19,6 +19,7 @@ rules: # word_boundary_start: false # word_boundary_end: false # include_note: false + # regex_terms: false ``` A set of default rules is provided in [`pkg/rule/default.yaml`]({{config.repo_url}}blob/main/pkg/rule/default.yaml). @@ -60,6 +61,15 @@ You can configure options for each rule. Add an `options` key to your rule defin * If `false`, the rule note will not be included in the output message * If `not set`, `include_note` in your `woke` config file (ie `.woke.yml`) regulates if the note should be included in the output message (default: `false`). +### `regex_terms` + +:octicons-milestone-24: Default: `false` + +* If `true`, terms will be evaluated as regular expressions +* If `false`, terms will be treated as plain-text values +* **NOTE** this is an advanced feature. Rules will be skipped if they do not compile. Only use non-capturing groups in patterns. Look-around assertions are not supported. + + ## Disabling Default Rules You can disable default rules by providing a rule in your `woke` config file (ie `.woke.yml`), with no terms or alternatives. diff --git a/docs/usage.md b/docs/usage.md index 7a0824bd..5a127113 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -168,6 +168,7 @@ Outputs the results as a series of [`json`](https://www.json.org/json-en.html) f "WordBoundary": , "WordBoundaryStart": , "WordBoundaryEnd": , + "RegexTerms": , "IncludeNote": } }, diff --git a/pkg/printer/json_test.go b/pkg/printer/json_test.go index a698abe7..f18198b2 100644 --- a/pkg/printer/json_test.go +++ b/pkg/printer/json_test.go @@ -12,7 +12,7 @@ func TestJSON_Print_JSON(t *testing.T) { res := generateFileResult() p := NewJSON(buf) assert.NoError(t, p.Print(res)) - expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n" + expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n" got := buf.String() assert.Equal(t, expected, got) } @@ -57,6 +57,6 @@ func TestJSON_Multiple(t *testing.T) { p.End() got := buf.String() - expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n{\"Filename\":\"bar.txt\",\"Results\":[{\"Rule\":{\"Name\":\"slave\",\"Terms\":[\"slave\"],\"Alternatives\":[\"follower\"],\"Note\":\"\",\"Severity\":\"error\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"slave\",\"Line\":\"this slave term must change\",\"StartPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`slave` may be insensitive, use `follower` instead\"}]}\n{\"Filename\":\"barfoo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"test\",\"Terms\":[\"test\"],\"Alternatives\":[\"alternative\"],\"Note\":\"\",\"Severity\":\"info\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"test\",\"Line\":\"this test must change\",\"StartPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`test` may be insensitive, use `alternative` instead\"}]}\n" + expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n{\"Filename\":\"bar.txt\",\"Results\":[{\"Rule\":{\"Name\":\"slave\",\"Terms\":[\"slave\"],\"Alternatives\":[\"follower\"],\"Note\":\"\",\"Severity\":\"error\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"slave\",\"Line\":\"this slave term must change\",\"StartPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`slave` may be insensitive, use `follower` instead\"}]}\n{\"Filename\":\"barfoo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"test\",\"Terms\":[\"test\"],\"Alternatives\":[\"alternative\"],\"Note\":\"\",\"Severity\":\"info\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"test\",\"Line\":\"this test must change\",\"StartPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`test` may be insensitive, use `alternative` instead\"}]}\n" assert.Equal(t, expected, got) } diff --git a/pkg/rule/options.go b/pkg/rule/options.go index c394f07b..60308cb6 100644 --- a/pkg/rule/options.go +++ b/pkg/rule/options.go @@ -5,5 +5,6 @@ type Options struct { WordBoundary bool `yaml:"word_boundary"` WordBoundaryStart bool `yaml:"word_boundary_start"` WordBoundaryEnd bool `yaml:"word_boundary_end"` + RegexTerms bool `yaml:"regex_terms"` IncludeNote *bool `yaml:"include_note"` } diff --git a/pkg/rule/rule.go b/pkg/rule/rule.go index 0e67138d..eb6205db 100644 --- a/pkg/rule/rule.go +++ b/pkg/rule/rule.go @@ -5,6 +5,8 @@ import ( "regexp" "strings" + "github.com/rs/zerolog/log" + "github.com/get-woke/woke/pkg/util" ) @@ -81,8 +83,13 @@ func (r *Rule) SetOptions(o Options) { } func (r *Rule) setRegex() { - group := strings.Join(escape(r.Terms), "|") - r.re = regexp.MustCompile(fmt.Sprintf(r.regexString(), group)) + var err error + group := strings.Join(escape(r, r.Terms), "|") + r.re, err = regexp.Compile(fmt.Sprintf(r.regexString(), group)) + if err != nil { + log.Error().Err(err).Str("Rule", r.Name).Msg("Unable to compile regular expression, disabling rule") + r.Terms = nil // Disable the rule + } } func (r *Rule) regexString() string { @@ -189,9 +196,13 @@ func IsDirectiveOnlyLine(line string) bool { return !util.ContainsAlphanumeric(leftText) } -func escape(ss []string) []string { +func escape(r *Rule, ss []string) []string { for i, s := range ss { - ss[i] = regexp.QuoteMeta(s) + if r.Options.RegexTerms { + ss[i] = s + } else { + ss[i] = regexp.QuoteMeta(s) + } } return ss } diff --git a/pkg/rule/rule_test.go b/pkg/rule/rule_test.go index d2dfae5e..b9517932 100644 --- a/pkg/rule/rule_test.go +++ b/pkg/rule/rule_test.go @@ -21,6 +21,32 @@ func testRule() Rule { } } +func testRegexRuleWithOptions(o Options) Rule { + r := testRegexRule() + r.SetOptions(o) + return r +} + +func testRegexRule() Rule { + return Rule{ + Name: "ruleregex", + Terms: []string{`\d+`}, + Alternatives: []string{"alt-regex1", "alt-regex-1"}, + Severity: SevWarn, + } +} + +func testInvalidRegexRule() Rule { + r := Rule{ + Name: "invalidrule", + Terms: []string{"("}, + Alternatives: []string{"alt-rule1", "alt-rule-1"}, + Severity: SevWarn, + } + r.SetOptions(Options{RegexTerms: true}) + return r +} + func TestRule_FindMatchIndexes(t *testing.T) { tests := []struct { text string @@ -49,6 +75,43 @@ func TestRule_FindMatchIndexes(t *testing.T) { assert.Equal(t, [][]int(nil), e.FindMatchIndexes("rule1")) } +func TestRule_InvalidRegexRule(t *testing.T) { + r := testInvalidRegexRule() + + // Verify rule is compiled + r.setRegex() + + // Validate that terms are now empty / rule is disabled + assert.Empty(t, r.Terms) + assert.True(t, r.Disabled()) +} + +func TestRule_FindMatchRegexIndexes(t *testing.T) { + tests := []struct { + text string + expected [][]int + expectedRe [][]int + }{ + {"this string has 123456 and 56789 included", [][]int(nil), [][]int{{16, 22}, {27, 32}}}, + {"this string does not have any findings", [][]int(nil), [][]int(nil)}, + {`this string has finding with \d+ \d+`, [][]int{{29, 32}, {33, 36}}, [][]int(nil)}, + } + for _, test := range tests { + r := testRegexRule() // Default to non regular expression matching + got := r.FindMatchIndexes(test.text) + assert.Equal(t, test.expected, got) + } + + for _, test := range tests { + r := testRegexRuleWithOptions(Options{RegexTerms: true}) + got := r.FindMatchIndexes(test.text) + assert.Equal(t, test.expectedRe, got) + } + + e := Rule{Name: "rule1"} + assert.Equal(t, [][]int(nil), e.FindMatchIndexes("rule1")) +} + func TestRule_Reason(t *testing.T) { r := testRule() assert.Equal(t, "`rule-1` may be insensitive, use `alt-rule1`, `alt-rule-1` instead", r.Reason("rule-1")) @@ -133,6 +196,11 @@ func TestRule_regexString(t *testing.T) { rule: testRuleWithOptions(Options{WordBoundary: true}), expected: `(?i)\b(%s)\b`, }, + { + desc: "regex rule", + rule: testRegexRuleWithOptions(Options{RegexTerms: true}), + expected: `(?i)(%s)`, + }, { desc: "word boundary start", rule: testRuleWithOptions(Options{WordBoundaryStart: true}), From 7814e7f4626629b7fcd677e55fd5b34754c0d286 Mon Sep 17 00:00:00 2001 From: Nathan Byrd Date: Thu, 12 Aug 2021 15:54:09 -0500 Subject: [PATCH 2/5] Removed extra line on markdown --- docs/rules.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/rules.md b/docs/rules.md index e8b3fad5..f2eec6c3 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -69,7 +69,6 @@ You can configure options for each rule. Add an `options` key to your rule defin * If `false`, terms will be treated as plain-text values * **NOTE** this is an advanced feature. Rules will be skipped if they do not compile. Only use non-capturing groups in patterns. Look-around assertions are not supported. - ## Disabling Default Rules You can disable default rules by providing a rule in your `woke` config file (ie `.woke.yml`), with no terms or alternatives. From c8e59afbdd7e24b0ec9758c490d32ca1f28f58b0 Mon Sep 17 00:00:00 2001 From: Nathan Byrd Date: Sun, 5 Sep 2021 14:13:15 -0500 Subject: [PATCH 3/5] Changed regex to use preferred format --- docs/rules.md | 16 ++++++++-------- docs/usage.md | 3 ++- pkg/printer/json_test.go | 4 ++-- pkg/rule/options.go | 1 - pkg/rule/rule.go | 24 +++++++++--------------- pkg/rule/rule_test.go | 35 ++++++++++++----------------------- 6 files changed, 33 insertions(+), 50 deletions(-) diff --git a/docs/rules.md b/docs/rules.md index f2eec6c3..9408f168 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -13,13 +13,13 @@ rules: - white-list alternatives: - allowlist + # regex: regexterm note: An optional description why these terms are not inclusive. It can be optionally included in the output message. # options: # word_boundary: false # word_boundary_start: false # word_boundary_end: false # include_note: false - # regex_terms: false ``` A set of default rules is provided in [`pkg/rule/default.yaml`]({{config.repo_url}}blob/main/pkg/rule/default.yaml). @@ -27,6 +27,13 @@ A set of default rules is provided in [`pkg/rule/default.yaml`]({{config.repo_ur !!! tip If you copy these rules into your config file, be sure to put them under the `rules:` key. +## `regex` + +Allows the definition of a regular expression (regex) directly. If specified, +any terms in the rule definition as well as word boundary options are ignored. +This is an advanced feature. Only use non-capturing groups in patterns. +Look-around assertions are not supported. + ## Options You can configure options for each rule. Add an `options` key to your rule definition to customize. @@ -61,13 +68,6 @@ You can configure options for each rule. Add an `options` key to your rule defin * If `false`, the rule note will not be included in the output message * If `not set`, `include_note` in your `woke` config file (ie `.woke.yml`) regulates if the note should be included in the output message (default: `false`). -### `regex_terms` - -:octicons-milestone-24: Default: `false` - -* If `true`, terms will be evaluated as regular expressions -* If `false`, terms will be treated as plain-text values -* **NOTE** this is an advanced feature. Rules will be skipped if they do not compile. Only use non-capturing groups in patterns. Look-around assertions are not supported. ## Disabling Default Rules diff --git a/docs/usage.md b/docs/usage.md index 5a127113..5e6bb346 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -86,6 +86,7 @@ The following fields are supported, depending on format: | alternative | List of alternative terms to use instead | | note | Note about reasoning for inclusion | | severity | From config, one of "error", "warning", or "info" | +| regex | Optional regular expression defined by rule | | optionbool | Option value, true or false | | linecontents | Contents of the line with finding | | lineno | Line number, 1 based | @@ -162,13 +163,13 @@ Outputs the results as a series of [`json`](https://www.json.org/json-en.html) f "", ... ], + "Regex": "", "Note": "", "Severity": "", "Options": { "WordBoundary": , "WordBoundaryStart": , "WordBoundaryEnd": , - "RegexTerms": , "IncludeNote": } }, diff --git a/pkg/printer/json_test.go b/pkg/printer/json_test.go index f18198b2..923b3138 100644 --- a/pkg/printer/json_test.go +++ b/pkg/printer/json_test.go @@ -12,7 +12,7 @@ func TestJSON_Print_JSON(t *testing.T) { res := generateFileResult() p := NewJSON(buf) assert.NoError(t, p.Print(res)) - expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n" + expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Regex\":\"\",\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n" got := buf.String() assert.Equal(t, expected, got) } @@ -57,6 +57,6 @@ func TestJSON_Multiple(t *testing.T) { p.End() got := buf.String() - expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n{\"Filename\":\"bar.txt\",\"Results\":[{\"Rule\":{\"Name\":\"slave\",\"Terms\":[\"slave\"],\"Alternatives\":[\"follower\"],\"Note\":\"\",\"Severity\":\"error\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"slave\",\"Line\":\"this slave term must change\",\"StartPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`slave` may be insensitive, use `follower` instead\"}]}\n{\"Filename\":\"barfoo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"test\",\"Terms\":[\"test\"],\"Alternatives\":[\"alternative\"],\"Note\":\"\",\"Severity\":\"info\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"RegexTerms\":false,\"IncludeNote\":null}},\"Finding\":\"test\",\"Line\":\"this test must change\",\"StartPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`test` may be insensitive, use `alternative` instead\"}]}\n" + expected := "{\"Filename\":\"foo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"whitelist\",\"Terms\":[\"whitelist\",\"white-list\",\"whitelisted\",\"white-listed\"],\"Alternatives\":[\"allowlist\"],\"Regex\":\"\",\"Note\":\"\",\"Severity\":\"warning\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"whitelist\",\"Line\":\"this whitelist must change\",\"StartPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"foo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`whitelist` may be insensitive, use `allowlist` instead\"}]}\n{\"Filename\":\"bar.txt\",\"Results\":[{\"Rule\":{\"Name\":\"slave\",\"Terms\":[\"slave\"],\"Alternatives\":[\"follower\"],\"Regex\":\"\",\"Note\":\"\",\"Severity\":\"error\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"slave\",\"Line\":\"this slave term must change\",\"StartPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"bar.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`slave` may be insensitive, use `follower` instead\"}]}\n{\"Filename\":\"barfoo.txt\",\"Results\":[{\"Rule\":{\"Name\":\"test\",\"Terms\":[\"test\"],\"Alternatives\":[\"alternative\"],\"Regex\":\"\",\"Note\":\"\",\"Severity\":\"info\",\"Options\":{\"WordBoundary\":false,\"WordBoundaryStart\":false,\"WordBoundaryEnd\":false,\"IncludeNote\":null}},\"Finding\":\"test\",\"Line\":\"this test must change\",\"StartPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":6},\"EndPosition\":{\"Filename\":\"barfoo.txt\",\"Offset\":0,\"Line\":1,\"Column\":15},\"Reason\":\"`test` may be insensitive, use `alternative` instead\"}]}\n" assert.Equal(t, expected, got) } diff --git a/pkg/rule/options.go b/pkg/rule/options.go index 60308cb6..c394f07b 100644 --- a/pkg/rule/options.go +++ b/pkg/rule/options.go @@ -5,6 +5,5 @@ type Options struct { WordBoundary bool `yaml:"word_boundary"` WordBoundaryStart bool `yaml:"word_boundary_start"` WordBoundaryEnd bool `yaml:"word_boundary_end"` - RegexTerms bool `yaml:"regex_terms"` IncludeNote *bool `yaml:"include_note"` } diff --git a/pkg/rule/rule.go b/pkg/rule/rule.go index eb6205db..ddb4aff8 100644 --- a/pkg/rule/rule.go +++ b/pkg/rule/rule.go @@ -5,8 +5,6 @@ import ( "regexp" "strings" - "github.com/rs/zerolog/log" - "github.com/get-woke/woke/pkg/util" ) @@ -19,6 +17,7 @@ type Rule struct { Name string `yaml:"name"` Terms []string `yaml:"terms"` Alternatives []string `yaml:"alternatives"` + Regex string `yaml:"regex"` Note string `yaml:"note"` Severity Severity `yaml:"severity"` Options Options `yaml:"options"` @@ -83,12 +82,11 @@ func (r *Rule) SetOptions(o Options) { } func (r *Rule) setRegex() { - var err error - group := strings.Join(escape(r, r.Terms), "|") - r.re, err = regexp.Compile(fmt.Sprintf(r.regexString(), group)) - if err != nil { - log.Error().Err(err).Str("Rule", r.Name).Msg("Unable to compile regular expression, disabling rule") - r.Terms = nil // Disable the rule + if len(r.Regex) != 0 { + r.re = regexp.MustCompile(fmt.Sprintf(`(%s)`, r.Regex)) + } else { + group := strings.Join(escape(r.Terms), "|") + r.re = regexp.MustCompile(fmt.Sprintf(r.regexString(), group)) } } @@ -196,13 +194,9 @@ func IsDirectiveOnlyLine(line string) bool { return !util.ContainsAlphanumeric(leftText) } -func escape(r *Rule, ss []string) []string { +func escape(ss []string) []string { for i, s := range ss { - if r.Options.RegexTerms { - ss[i] = s - } else { - ss[i] = regexp.QuoteMeta(s) - } + ss[i] = regexp.QuoteMeta(s) } return ss } @@ -234,7 +228,7 @@ func maskInlineIgnore(line string) string { // which is helpful for disabling default rules. Eventually, there should be a better // way to disable a default rule, and then, if a rule has no Terms, it falls back to the Name. func (r *Rule) Disabled() bool { - return len(r.Terms) == 0 + return len(r.Terms) == 0 && len(r.Regex) == 0 } // SetIncludeNote populates IncludeNote attributte in Options diff --git a/pkg/rule/rule_test.go b/pkg/rule/rule_test.go index b9517932..eb64b157 100644 --- a/pkg/rule/rule_test.go +++ b/pkg/rule/rule_test.go @@ -21,16 +21,10 @@ func testRule() Rule { } } -func testRegexRuleWithOptions(o Options) Rule { - r := testRegexRule() - r.SetOptions(o) - return r -} - func testRegexRule() Rule { return Rule{ Name: "ruleregex", - Terms: []string{`\d+`}, + Regex: `\d+`, Alternatives: []string{"alt-regex1", "alt-regex-1"}, Severity: SevWarn, } @@ -39,11 +33,10 @@ func testRegexRule() Rule { func testInvalidRegexRule() Rule { r := Rule{ Name: "invalidrule", - Terms: []string{"("}, + Regex: `(`, Alternatives: []string{"alt-rule1", "alt-rule-1"}, Severity: SevWarn, } - r.SetOptions(Options{RegexTerms: true}) return r } @@ -76,34 +69,30 @@ func TestRule_FindMatchIndexes(t *testing.T) { } func TestRule_InvalidRegexRule(t *testing.T) { + // turn off the panic + defer func() { _ = recover() }() + r := testInvalidRegexRule() - // Verify rule is compiled + // Verify rule is compiled - should panic r.setRegex() // Validate that terms are now empty / rule is disabled - assert.Empty(t, r.Terms) - assert.True(t, r.Disabled()) + t.Errorf("Invalid rule should have panicked") } func TestRule_FindMatchRegexIndexes(t *testing.T) { tests := []struct { text string - expected [][]int expectedRe [][]int }{ - {"this string has 123456 and 56789 included", [][]int(nil), [][]int{{16, 22}, {27, 32}}}, - {"this string does not have any findings", [][]int(nil), [][]int(nil)}, - {`this string has finding with \d+ \d+`, [][]int{{29, 32}, {33, 36}}, [][]int(nil)}, - } - for _, test := range tests { - r := testRegexRule() // Default to non regular expression matching - got := r.FindMatchIndexes(test.text) - assert.Equal(t, test.expected, got) + {"this string has 123456 and 56789 included", [][]int{{16, 22}, {27, 32}}}, + {"this string does not have any findings", [][]int(nil)}, + {`this string has finding with \d+ \d+`, [][]int(nil)}, } for _, test := range tests { - r := testRegexRuleWithOptions(Options{RegexTerms: true}) + r := testRegexRule() got := r.FindMatchIndexes(test.text) assert.Equal(t, test.expectedRe, got) } @@ -198,7 +187,7 @@ func TestRule_regexString(t *testing.T) { }, { desc: "regex rule", - rule: testRegexRuleWithOptions(Options{RegexTerms: true}), + rule: testRegexRule(), expected: `(?i)(%s)`, }, { From 8401b28013eff89baa7b60fb816219fae196d066 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 5 Sep 2021 19:21:21 +0000 Subject: [PATCH 4/5] docs: autogenerated update for docs/snippets/woke.md: `1fe097` GitHub Action run 1, commit: https://github.com/cognitivegears/woke/commit/1fe097d643f0ec3072866246c7e14fd6f1ef930a --- docs/snippets/woke.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/snippets/woke.md b/docs/snippets/woke.md index f3c4e558..c29c2d20 100644 --- a/docs/snippets/woke.md +++ b/docs/snippets/woke.md @@ -7,6 +7,7 @@ Check for usage of non-inclusive language in your code and provide alternatives ### Synopsis + woke is a linter that will check your source code for usage of non-inclusive language and provide suggestions for alternatives. Rules can be customized to suit your needs. @@ -29,4 +30,4 @@ woke [globs ...] [flags] --stdin Read from stdin ``` -###### Auto generated by spf13/cobra on 13-Aug-2021 +###### Auto generated by spf13/cobra on 5-Sep-2021 From e3fdd008f9a692024110f1e313dcb01ea18bcb93 Mon Sep 17 00:00:00 2001 From: Nathan Byrd Date: Sun, 5 Sep 2021 14:38:56 -0500 Subject: [PATCH 5/5] Resolved markdownlint --- docs/rules.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/rules.md b/docs/rules.md index 661e0740..e5d710a8 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -30,9 +30,9 @@ A set of default rules is provided in [`pkg/rule/default.yaml`]({{config.repo_ur ## `regex` -Allows the definition of a regular expression (regex) directly. If specified, -any terms in the rule definition as well as word boundary options are ignored. -This is an advanced feature. Only use non-capturing groups in patterns. +Allows the definition of a regular expression (regex) directly. If specified, +any terms in the rule definition as well as word boundary options are ignored. +This is an advanced feature. Only use non-capturing groups in patterns. Look-around assertions are not supported. ## Options