diff --git a/remediation/precommit/precommit-config.yml b/remediation/precommit/precommit-config.yml new file mode 100644 index 00000000..9aca0535 --- /dev/null +++ b/remediation/precommit/precommit-config.yml @@ -0,0 +1,61 @@ +hooks: + common: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/gitleaks/gitleaks + rev: v8.16.3 + hooks: + - id: gitleaks + Python: + - repo: https://github.com/pylint-dev/pylint + rev: v2.17.2 + hooks: + - id: pylint + JavaScript: + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.38.0 + hooks: + - id: eslint + TypeScript: + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.38.0 + hooks: + - id: eslint + Java: + - repo: https://github.com/gherynos/pre-commit-java + rev: v0.2.4 + hooks: + - id: Checkstyle + C: + - repo: https://github.com/pocc/pre-commit-hooks + rev: v1.3.5 + hooks: + - id: cpplint + C++: + - repo: https://github.com/pocc/pre-commit-hooks + rev: v1.3.5 + hooks: + - id: cpplint + PHP: + - repo: https://github.com/digitalpulp/pre-commit-php + rev: 1.4.0 + hooks: + - id: php-lint-all + Ruby: + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: RuboCop + Go: + - repo: https://github.com/golangci/golangci-lint + rev: v1.52.2 + hooks: + - id: golangci-lint + Shell: + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: shellcheck \ No newline at end of file diff --git a/remediation/precommit/precommitconfig.go b/remediation/precommit/precommitconfig.go new file mode 100644 index 00000000..b733de9b --- /dev/null +++ b/remediation/precommit/precommitconfig.go @@ -0,0 +1,254 @@ +package precommit + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "sort" + "strings" + + "github.com/step-security/secure-repo/remediation/workflow/permissions" + "gopkg.in/yaml.v3" +) + +type UpdatePrecommitConfigResponse struct { + OriginalInput string + FinalOutput string + IsChanged bool + ConfigfileFetchError bool +} + +type UpdatePrecommitConfigRequest struct { + Content string + Languages []string +} + +type PrecommitConfig struct { + Repos []Repo `yaml:"repos"` +} + +type Repo struct { + Repo string `yaml:"repo"` + Rev string `yaml:"rev"` + Hooks []Hook `yaml:"hooks"` +} + +type Hook struct { + Id string `yaml:"id"` +} + +type FetchPrecommitConfig struct { + Hooks Hooks `yaml:"hooks"` +} + +type Hooks map[string][]Repo + +func getConfigFile() (string, error) { + filePath := os.Getenv("PRECOMMIT_CONFIG") + + if filePath == "" { + filePath = "./precommit-config.yml" + } + + configFile, err := ioutil.ReadFile(filePath) + if err != nil { + return "", err + } + + return string(configFile), nil +} + +func GetHooks(precommitConfig string) ([]Repo, error) { + var updatePrecommitConfigRequest UpdatePrecommitConfigRequest + json.Unmarshal([]byte(precommitConfig), &updatePrecommitConfigRequest) + inputConfigFile := []byte(updatePrecommitConfigRequest.Content) + configMetadata := PrecommitConfig{} + err := yaml.Unmarshal(inputConfigFile, &configMetadata) + if err != nil { + return nil, err + } + + alreadyPresentHooks := make(map[string]bool) + for _, repos := range configMetadata.Repos { + for _, hook := range repos.Hooks { + alreadyPresentHooks[hook.Id] = true + } + } + + configFile, err := getConfigFile() + if err != nil { + return nil, err + } + var fetchPrecommitConfig FetchPrecommitConfig + yaml.Unmarshal([]byte(configFile), &fetchPrecommitConfig) + newHooks := make(map[string]Repo) + for _, lang := range updatePrecommitConfigRequest.Languages { + if _, isSupported := fetchPrecommitConfig.Hooks[lang]; !isSupported { + continue + } + if _, ok := alreadyPresentHooks[fetchPrecommitConfig.Hooks[lang][0].Hooks[0].Id]; !ok { + if repo, ok := newHooks[fetchPrecommitConfig.Hooks[lang][0].Repo]; ok { + repo.Hooks = append(repo.Hooks, fetchPrecommitConfig.Hooks[lang][0].Hooks...) + newHooks[fetchPrecommitConfig.Hooks[lang][0].Repo] = repo + } else { + newHooks[fetchPrecommitConfig.Hooks[lang][0].Repo] = fetchPrecommitConfig.Hooks[lang][0] + } + alreadyPresentHooks[fetchPrecommitConfig.Hooks[lang][0].Hooks[0].Id] = true + } + } + // Adding common hooks + var repos []Repo + for _, repo := range fetchPrecommitConfig.Hooks["common"] { + tempRepo := repo + tempRepo.Hooks = nil + hookPresent := false + for _, hook := range repo.Hooks { + if _, ok := alreadyPresentHooks[hook.Id]; !ok { + tempRepo.Hooks = append(tempRepo.Hooks, hook) + hookPresent = true + } + } + if hookPresent { + repos = append(repos, tempRepo) + } + } + for _, repo := range newHooks { + repos = append(repos, repo) + } + sort.Slice(repos, func(i, j int) bool { + return repos[i].Repo < repos[j].Repo + }) + return repos, nil +} + +func UpdatePrecommitConfig(precommitConfig string, Hooks []Repo) (*UpdatePrecommitConfigResponse, error) { + var updatePrecommitConfigRequest UpdatePrecommitConfigRequest + json.Unmarshal([]byte(precommitConfig), &updatePrecommitConfigRequest) + inputConfigFile := []byte(updatePrecommitConfigRequest.Content) + configMetadata := PrecommitConfig{} + err := yaml.Unmarshal(inputConfigFile, &configMetadata) + if err != nil { + return nil, err + } + + response := new(UpdatePrecommitConfigResponse) + response.FinalOutput = updatePrecommitConfigRequest.Content + response.OriginalInput = updatePrecommitConfigRequest.Content + response.IsChanged = false + + response.FinalOutput = strings.TrimSuffix(response.FinalOutput, "\n") + repoIndent := 0 + repoGap := 1 + hooksIndent := 2 + hooksGap := 1 + if updatePrecommitConfigRequest.Content == "" { + response.FinalOutput = "repos:" + } else { + repoIndent, repoGap, hooksIndent, hooksGap, err = getPrecommitIndentation(response.FinalOutput) + if err != nil { + return nil, err + } + } + + for _, Update := range Hooks { + repoAlreadyExist := false + for _, update := range configMetadata.Repos { + if update.Repo == Update.Repo { + repoAlreadyExist = true + } + if repoAlreadyExist { + break + } + } + response.FinalOutput, err = addHook(Update, repoAlreadyExist, response.FinalOutput, + repoIndent, repoGap, hooksIndent, hooksGap) + if err != nil { + return nil, err + } + response.IsChanged = true + } + + if !strings.HasSuffix(response.FinalOutput, "\n") { + response.FinalOutput = response.FinalOutput + "\n" + } + + return response, nil +} + +func getPrecommitIndentation(content string) (int, int, int, int, error) { + lines := strings.Split(content, "\n") + + var repoIndent, repoGap, hooksIndent, hooksGap int + repoFound, hooksFound := false, false + for _, line := range lines { + if strings.Contains(line, "repo:") && !repoFound { + repoIndent = strings.Index(line, "-") + repoGap = strings.Index(line, "repo:") - repoIndent - 1 + repoFound = true + } else if strings.Contains(line, "id:") && !hooksFound { + hooksIndent = strings.Index(line, "-") + hooksGap = strings.Index(line, "id:") - hooksIndent - 1 + hooksFound = true + } + + if repoFound && hooksFound { + break + } + } + + return repoIndent, repoGap, hooksIndent, hooksGap, nil +} + +func addHook(Update Repo, repoAlreadyExist bool, inputYaml string, repoIndent, repoGap, hooksIndent, hooksGap int) (string, error) { + t := yaml.Node{} + + err := yaml.Unmarshal([]byte(inputYaml), &t) + if err != nil { + return "", fmt.Errorf("unable to parse yaml %v", err) + } + + if repoAlreadyExist { + jobNode := permissions.IterateNode(&t, Update.Repo, "!!str", 0) + if jobNode == nil { + return "", fmt.Errorf("Repo Name %s not found in the input yaml", Update.Repo) + } + + // TODO: Also update rev version for already exist repo + inputLines := strings.Split(inputYaml, "\n") + var output []string + for i := 0; i < jobNode.Line+1; i++ { + output = append(output, inputLines[i]) + } + + for _, hook := range Update.Hooks { + hookIndentStr := strings.Repeat(" ", hooksIndent) + hookGapStr := strings.Repeat(" ", hooksGap) + output = append(output, fmt.Sprintf("%s-%sid: %s", hookIndentStr, hookGapStr, hook.Id)) + } + + for i := jobNode.Line + 1; i < len(inputLines); i++ { + output = append(output, inputLines[i]) + } + return strings.Join(output, "\n"), nil + } else { + inputLines := strings.Split(inputYaml, "\n") + + repoIndentStr := strings.Repeat(" ", repoIndent) + repoGapStr := strings.Repeat(" ", repoGap) + inputLines = append(inputLines, fmt.Sprintf("%s-%srepo: %s", repoIndentStr, repoGapStr, Update.Repo)) + + revIndentStr := strings.Repeat(" ", repoIndent+repoGap+1) + inputLines = append(inputLines, fmt.Sprintf("%srev: %s", revIndentStr, Update.Rev)) + + inputLines = append(inputLines, fmt.Sprintf("%shooks:", revIndentStr)) + + hookIndentStr := strings.Repeat(" ", hooksIndent) + hookGapStr := strings.Repeat(" ", hooksGap) + for _, hook := range Update.Hooks { + inputLines = append(inputLines, fmt.Sprintf("%s-%sid: %s", hookIndentStr, hookGapStr, hook.Id)) + } + + return strings.Join(inputLines, "\n"), nil + } +} diff --git a/remediation/precommit/precommitconfig_test.go b/remediation/precommit/precommitconfig_test.go new file mode 100644 index 00000000..3621f8ac --- /dev/null +++ b/remediation/precommit/precommitconfig_test.go @@ -0,0 +1,81 @@ +package precommit + +import ( + "encoding/json" + "io/ioutil" + "log" + "path" + "testing" +) + +func TestUpdatePrecommitConfig(t *testing.T) { + + const inputDirectory = "../../testfiles/precommit/input" + const outputDirectory = "../../testfiles/precommit/output" + + tests := []struct { + fileName string + Languages []string + isChanged bool + }{ + { + fileName: "basic.yml", + Languages: []string{"JavaScript", "C++"}, + isChanged: true, + }, + { + fileName: "file-not-exit.yml", + Languages: []string{"JavaScript", "C++"}, + isChanged: true, + }, + { + fileName: "same-repo-different-hooks.yml", + Languages: []string{"Ruby", "Shell"}, + isChanged: true, + }, + { + fileName: "style1.yml", + Languages: []string{"Ruby", "Shell"}, + isChanged: true, + }, + } + + for _, test := range tests { + var updatePrecommitConfigRequest UpdatePrecommitConfigRequest + input, err := ioutil.ReadFile(path.Join(inputDirectory, test.fileName)) + if err != nil { + log.Fatal(err) + } + updatePrecommitConfigRequest.Content = string(input) + updatePrecommitConfigRequest.Languages = test.Languages + inputRequest, err := json.Marshal(updatePrecommitConfigRequest) + if err != nil { + log.Fatal(err) + } + + hooks, err := GetHooks(string(inputRequest)) + if err != nil { + log.Fatal(err) + } + output, err := UpdatePrecommitConfig(string(inputRequest), hooks) + if err != nil { + t.Fatalf("Error not expected: %s", err) + } + + expectedOutput, err := ioutil.ReadFile(path.Join(outputDirectory, test.fileName)) + if err != nil { + log.Fatal(err) + } + + if string(expectedOutput) != output.FinalOutput { + t.Errorf("test failed %s did not match expected output\n%s", test.fileName, output.FinalOutput) + } + + if output.IsChanged != test.isChanged { + t.Errorf("test failed %s did not match IsChanged, Expected: %v Got: %v", test.fileName, test.isChanged, output.IsChanged) + + } + + } + +} diff --git a/testfiles/precommit/input/basic.yml b/testfiles/precommit/input/basic.yml new file mode 100644 index 00000000..65ae8148 --- /dev/null +++ b/testfiles/precommit/input/basic.yml @@ -0,0 +1,19 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: check-json + - id: trailing-whitespace +- repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.23.0 + hooks: + - id: eslint +- repo: https://github.com/ejba/pre-commit-maven + rev: v0.3.3 + hooks: + - id: maven-test +- repo: https://github.com/zricethezav/gitleaks + rev: v8.12.0 + hooks: + - id: gitleaks \ No newline at end of file diff --git a/testfiles/precommit/input/file-not-exit.yml b/testfiles/precommit/input/file-not-exit.yml new file mode 100644 index 00000000..e69de29b diff --git a/testfiles/precommit/input/same-repo-different-hooks.yml b/testfiles/precommit/input/same-repo-different-hooks.yml new file mode 100644 index 00000000..8f9857be --- /dev/null +++ b/testfiles/precommit/input/same-repo-different-hooks.yml @@ -0,0 +1,19 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: check-json + - id: trailing-whitespace +- repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.23.0 + hooks: + - id: eslint +- repo: https://github.com/ejba/pre-commit-maven + rev: v0.3.3 + hooks: + - id: maven-test +- repo: https://github.com/zricethezav/gitleaks + rev: v8.12.0 + hooks: + - id: gitleaks diff --git a/testfiles/precommit/input/style1.yml b/testfiles/precommit/input/style1.yml new file mode 100644 index 00000000..cbc34112 --- /dev/null +++ b/testfiles/precommit/input/style1.yml @@ -0,0 +1,11 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black \ No newline at end of file diff --git a/testfiles/precommit/output/basic.yml b/testfiles/precommit/output/basic.yml new file mode 100644 index 00000000..e3c9a5df --- /dev/null +++ b/testfiles/precommit/output/basic.yml @@ -0,0 +1,24 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: trailing-whitespace +- repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.23.0 + hooks: + - id: eslint +- repo: https://github.com/ejba/pre-commit-maven + rev: v0.3.3 + hooks: + - id: maven-test +- repo: https://github.com/zricethezav/gitleaks + rev: v8.12.0 + hooks: + - id: gitleaks +- repo: https://github.com/pocc/pre-commit-hooks + rev: v1.3.5 + hooks: + - id: cpplint diff --git a/testfiles/precommit/output/file-not-exit.yml b/testfiles/precommit/output/file-not-exit.yml new file mode 100644 index 00000000..0e46c4da --- /dev/null +++ b/testfiles/precommit/output/file-not-exit.yml @@ -0,0 +1,18 @@ +repos: +- repo: https://github.com/gitleaks/gitleaks + rev: v8.16.3 + hooks: + - id: gitleaks +- repo: https://github.com/pocc/pre-commit-hooks + rev: v1.3.5 + hooks: + - id: cpplint +- repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.38.0 + hooks: + - id: eslint +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/testfiles/precommit/output/same-repo-different-hooks.yml b/testfiles/precommit/output/same-repo-different-hooks.yml new file mode 100644 index 00000000..4d275c0b --- /dev/null +++ b/testfiles/precommit/output/same-repo-different-hooks.yml @@ -0,0 +1,25 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: trailing-whitespace +- repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.23.0 + hooks: + - id: eslint +- repo: https://github.com/ejba/pre-commit-maven + rev: v0.3.3 + hooks: + - id: maven-test +- repo: https://github.com/zricethezav/gitleaks + rev: v8.12.0 + hooks: + - id: gitleaks +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: RuboCop + - id: shellcheck diff --git a/testfiles/precommit/output/style1.yml b/testfiles/precommit/output/style1.yml new file mode 100644 index 00000000..06489ca1 --- /dev/null +++ b/testfiles/precommit/output/style1.yml @@ -0,0 +1,20 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black +- repo: https://github.com/gitleaks/gitleaks + rev: v8.16.3 + hooks: + - id: gitleaks +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: RuboCop + - id: shellcheck