diff --git a/.github/actions/runleaks/.gitignore b/.github/actions/runleaks/.gitignore new file mode 100644 index 00000000000..47b70ae9777 --- /dev/null +++ b/.github/actions/runleaks/.gitignore @@ -0,0 +1,4 @@ +.git +_git +.github +_github diff --git a/.github/actions/runleaks/Dockerfile b/.github/actions/runleaks/Dockerfile new file mode 100644 index 00000000000..cb3b4e0ccb1 --- /dev/null +++ b/.github/actions/runleaks/Dockerfile @@ -0,0 +1,8 @@ +FROM cgr.dev/chainguard/wolfi-base:latest +RUN apk add git gh make parallel jq + +COPY git-secrets /git-secrets +RUN make -C /git-secrets install +COPY lib/* / + +ENTRYPOINT ["bash", "/scan.sh"] diff --git a/.github/actions/runleaks/LICENSE b/.github/actions/runleaks/LICENSE new file mode 100644 index 00000000000..251ac3d0313 --- /dev/null +++ b/.github/actions/runleaks/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Josiah Siegel + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.github/actions/runleaks/README.md b/.github/actions/runleaks/README.md new file mode 100644 index 00000000000..d86f92d2062 --- /dev/null +++ b/.github/actions/runleaks/README.md @@ -0,0 +1,163 @@ +# runleaks + +[![Scan Action Logs](https://github.com/CDCgov/prime-reportstream/.github/workflows/runleaks--main.yml/badge.svg?branch=main)](https://github.com/CDCgov/prime-reportstream/.github/workflows/runleaks--main.yml) + +Leverages [git-secrets](https://github.com/awslabs/git-secrets) to identify potential leaks in GitHub action run logs. + + * Common Azure and Google Cloud patterns are available, thanks to fork [msalemcode/git-secrets](https://github.com/msalemcode/git-secrets). + + +## Inputs +```yml + github-token: + description: 'Token used to login to GitHub' + required: true + repo: + description: 'Repo to scan run logs for exceptions' + required: false + default: ${{ github.repository }} + run-limit: + description: 'Limit on how many runs to scan' + required: false + default: '50' + min-days-old: + description: 'Min age of runs in days' + required: false + default: '0' + max-days-old: + description: 'Max age of runs in days' + required: false + default: '3' + patterns-path: + description: 'Patterns file path' + required: false + default: ".runleaks/patterns.txt" + exclusions-path: + description: 'Excluded patterns file path' + required: false + default: ".runleaks/exclusions.txt" + fail-on-leak: + description: 'Fail action if leak is found' + required: false + default: true +``` + +## Outputs +```yml + exceptions: + description: 'Json output of run logs with exceptions' + count: + description: 'Count of exceptions' +``` + +## Usage + * Note: [GitHub rate limits](#rate-limits) +```yml + - name: Checkout + uses: actions/checkout@v3 + - name: Scan run logs + uses: josiahsiegel/runleaks@4dd30d107c03b6ade87978e10c94a77015e488f9 + id: scan + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-limit: 500 + fail-on-leak: false + - name: Get scan exceptions + if: steps.scan.outputs.count > 0 + run: echo "${{ steps.scan.outputs.exceptions }}" +``` +or +```yml + - name: Checkout + uses: actions/checkout@v3 + - name: Scan run logs + uses: josiahsiegel/runleaks@4dd30d107c03b6ade87978e10c94a77015e488f9 + id: scan + with: + github-token: ${{ secrets.MY_TOKEN }} + patterns-path: ".github/patterns.txt" + exclusions-path: ".github/exclusions.txt" + fail-on-leak: false + - name: Get scan exceptions + if: steps.scan.outputs.count > 0 + run: echo "${{ steps.scan.outputs.exceptions }}" +``` +or +```yml + - name: Checkout + uses: actions/checkout@v3 + with: + repository: 'me/my-repo' + - name: Scan run logs + uses: josiahsiegel/runleaks@4dd30d107c03b6ade87978e10c94a77015e488f9 + id: scan + with: + github-token: ${{ secrets.MY_TOKEN }} + repo: 'me/my-repo' + run-limit: 200 + min-days-old: 0 + max-days-old: 4 + fail-on-leak: true +``` + +## Local testing + * Registers default patterns +```sh +git clone https://github.com/CDCgov/prime-reportstream/.github/actions/runleaks.git +cd runleaks/ +docker build -t runleaks . +docker run scan "" "" +``` + +## Pattern file + * Default location: `.runleaks/patterns.txt` + +``` +#################################################################### + +# Register a secret provider +#--register-azure +#--register-gcp +--register-aws + +#################################################################### + +# Add a prohibited pattern +--add [A-Z0-9]{20} +--add Account[k|K]ey +--add Shared[a|A]ccessSignature + +#################################################################### + +# Add a string that is scanned for literally (+ is escaped): +--add --literal foo+bar + +#################################################################### +``` + +## Exclusion file + * Default location: `.runleaks/exclusions.txt` +``` +#################################################################### + +# Add regular expressions patterns to filter false positives. + +# Allow GUID +("|')[0-9A-Fa-f]{8}-([0-9A-Fa-f]{4}-){3}[0-9A-Fa-f]{12}("|') + +#################################################################### +``` + +## Performance + + * Scan 50 runs = 1 min + + * Scan 500 runs = 8 mins + +* Scan 3000 runs = 50 mins + +## Rate limits + +Built-in secret `GITHUB_TOKEN` is [limited to 1,000 requests per hour per repository](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#requests-from-github-actions). + +To avoid repo-wide rate limiting, personal access tokens can be added to secrets, which are [limited to 5,000 requests per hour and per authenticated user](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#requests-from-personal-accounts). diff --git a/.github/actions/runleaks/action.yml b/.github/actions/runleaks/action.yml new file mode 100644 index 00000000000..d431caf1cf6 --- /dev/null +++ b/.github/actions/runleaks/action.yml @@ -0,0 +1,55 @@ +# action.yml +name: 'runleaks' +description: 'Identify potential leaks in GitHub action logs' +branding: + icon: 'search' + color: 'red' +inputs: + github-token: + description: 'Token used to login to GitHub' + required: true + repo: + description: 'Repo to scan run logs for exceptions' + required: false + default: ${{ github.repository }} + run-limit: + description: 'Limit on how many runs to scan' + required: false + default: '100' + min-days-old: + description: 'Min age of runs in days' + required: false + default: '0' + max-days-old: + description: 'Max age of runs in days' + required: false + default: '3' + patterns-path: + description: 'Patterns file path' + required: false + default: ".github/runleaks/patterns.txt" + exclusions-path: + description: 'Excluded patterns file path' + required: false + default: ".github/runleaks/exclusions.txt" + fail-on-leak: + description: 'Fail action if leak is found' + required: false + default: true +outputs: + exceptions: + description: 'Json output of run logs with exceptions' + count: + description: 'Count of exceptions' +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.github-token }} + - ${{ inputs.repo }} + - ${{ inputs.run-limit }} + - ${{ inputs.min-days-old }} + - ${{ inputs.max-days-old }} + - ${{ inputs.patterns-path }} + - ${{ inputs.exclusions-path }} + - ${{ inputs.fail-on-leak }} diff --git a/.github/actions/runleaks/git-secrets/.pre-commit-hooks.yaml b/.github/actions/runleaks/git-secrets/.pre-commit-hooks.yaml new file mode 100644 index 00000000000..d313836246f --- /dev/null +++ b/.github/actions/runleaks/git-secrets/.pre-commit-hooks.yaml @@ -0,0 +1,5 @@ +- id: git-secrets + name: Git Secrets + description: git-secrets scans commits, commit messages, and --no-ff merges to prevent adding secrets into your git repositories. + entry: 'git-secrets --pre_commit_hook' + language: script diff --git a/.github/actions/runleaks/git-secrets/CHANGELOG.md b/.github/actions/runleaks/git-secrets/CHANGELOG.md new file mode 100644 index 00000000000..cfcae4e818c --- /dev/null +++ b/.github/actions/runleaks/git-secrets/CHANGELOG.md @@ -0,0 +1,49 @@ +# CHANGELOG + +## 1.3.0 - 2019-02-10 + +* Empty provider output is now excluded + (https://github.com/awslabs/git-secrets/issues/34) +* Spaces are now supported in git exec path, making more Windows + paths execute properly. +* Patterns with newlines and carriage returns are now loaded properly. +* Patterns that contain only "\n" are now ignored. +* Various Bash 4 fixes (https://github.com/awslabs/git-secrets/issues/66). +* Make IAM key scanning much more targeted. + +## 1.2.1 - 2016-06-27 + +* Fixed an issue where secret provider commands were causing "command not + found" errors due to a previously set IFS variable. + https://github.com/awslabs/git-secrets/pull/30 + +## 1.2.0 - 2016-05-23 + +* Fixed an issue where spaces files with spaces in their names were not being + properly scanned in the pre-commit hook. +* Now ignoring empty lines and comments (e.g., `#`) in the .gitallowed file. +* Fixed an issue where numbers were being compared to strings causing failures + on some platforms. + +## 1.1.0 - 2016-04-06 + +* Bug fix: the pre-commit hook previously only scanned the working directory + rather than staged files. This release updates the pre-commit hook to instead + scan staged files so that git-secrets will detect violations if the working + directory drifts from the staging directory. +* Added the `--scan-history` subcommand so that you can scan your entire + git history for violations. +* Added the ability to filter false positives by using a .gitallowed file. +* Added support for `--cached`, `--no-index`, and `--untracked` to the `--scan` + subcommand. + +## 1.0.1 - 2016-01-11 + +* Now works correctly with filenames in a repository that contain spaces when + executing `git secrets --scan` with no provided filename (via `git grep`). +* Now works with git repositories with hundreds of thousands of files when + using `git secrets --scan` with no provided filename (via `git grep`). + +## 1.0.0 - 2015-12-10 + +* Initial release of ``git-secrets``. diff --git a/.github/actions/runleaks/git-secrets/CODE_OF_CONDUCT.md b/.github/actions/runleaks/git-secrets/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..3b64466870c --- /dev/null +++ b/.github/actions/runleaks/git-secrets/CODE_OF_CONDUCT.md @@ -0,0 +1,4 @@ +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. diff --git a/.github/actions/runleaks/git-secrets/CONTRIBUTING.md b/.github/actions/runleaks/git-secrets/CONTRIBUTING.md new file mode 100644 index 00000000000..de6d3d38075 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Contributing Guidelines + +Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional +documentation, we greatly value feedback and contributions from our community. + +Please read through this document before submitting any issues or pull requests to ensure we have all the necessary +information to effectively respond to your bug report or contribution. + + +## Reporting Bugs/Feature Requests + +We welcome you to use the GitHub issue tracker to report bugs or suggest features. + +When filing an issue, please check [existing open](https://github.com/awslabs/git-secrets/issues), or [recently closed](https://github.com/awslabs/git-secrets/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already +reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: + +* A reproducible test case or series of steps +* The version of our code being used +* Any modifications you've made relevant to the bug +* Anything unusual about your environment or deployment + + +## Contributing via Pull Requests +Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: + +1. You are working against the latest source on the *master* branch. +2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. +3. You open an issue to discuss any significant work - we would hate for your time to be wasted. + +To send us a pull request, please: + +1. Fork the repository. +2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. +3. Ensure local tests pass. +4. Commit to your fork using clear commit messages. +5. Send us a pull request, answering any default questions in the pull request interface. +6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. + +GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and +[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). + + +## Finding contributions to work on +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/git-secrets/labels/help%20wanted) issues is a great place to start. + + +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. + + +## Security issue notifications +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. + + +## Licensing + +See the [LICENSE](https://github.com/awslabs/git-secrets/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. + +We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. diff --git a/.github/actions/runleaks/git-secrets/LICENSE.txt b/.github/actions/runleaks/git-secrets/LICENSE.txt new file mode 100644 index 00000000000..de96b9473c9 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/LICENSE.txt @@ -0,0 +1,208 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Note: Other license terms may apply to certain, identified software files +contained within or distributed with the accompanying software if such terms +are included in the directory containing the accompanying software. Such other +license terms will then apply in lieu of the terms of the software license +above. diff --git a/.github/actions/runleaks/git-secrets/Makefile b/.github/actions/runleaks/git-secrets/Makefile new file mode 100644 index 00000000000..a67eee2dd23 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/Makefile @@ -0,0 +1,25 @@ +PREFIX ?= /usr/local +MANPREFIX ?= "${PREFIX}/share/man/man1" + +help: + @echo "Please use \`make ' where is one of" + @echo " test to perform unit tests." + @echo " man to build the man file from README.rst" + @echo " install to install. Use PREFIX and MANPREFIX to customize." + +# We use bats for testing: https://github.com/sstephenson/bats +test: + LANG=C test/bats/bin/bats test/ + +# The man page is completely derived from README.rst. Edits to +# README.rst require a rebuild of the man page. +man: + rst2man.py README.rst > git-secrets.1 + +install: + @mkdir -p ${DESTDIR}${MANPREFIX} + @mkdir -p ${DESTDIR}${PREFIX}/bin + @cp -f git-secrets ${DESTDIR}${PREFIX}/bin + @cp -f git-secrets.1 ${DESTDIR}${MANPREFIX} + +.PHONY: help test man diff --git a/.github/actions/runleaks/git-secrets/NOTICE.txt b/.github/actions/runleaks/git-secrets/NOTICE.txt new file mode 100644 index 00000000000..a5e5da9ba01 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/NOTICE.txt @@ -0,0 +1,6 @@ +git-secrets +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +bats +This product bundles bats, which is available under a "MIT" license. +For details, see test/bats. diff --git a/.github/actions/runleaks/git-secrets/README.rst b/.github/actions/runleaks/git-secrets/README.rst new file mode 100644 index 00000000000..1be1691ab8a --- /dev/null +++ b/.github/actions/runleaks/git-secrets/README.rst @@ -0,0 +1,565 @@ +=========== +git-secrets +=========== + +------------------------------------------------------------------------------------------- +Prevent committing AWS , AZURE and GCP sensitive creds to a git repository. +------------------------------------------------------------------------------------------- + +.. contents:: :depth: 2 + +Synopsis +-------- + +:: + + git secrets --scan [-r|--recursive] [--cached] [--no-index] [--untracked] [...] + git secrets --scan-history + git secrets --install [-f|--force] [] + git secrets --list [--global] + git secrets --add [-a|--allowed] [-l|--literal] [--global] + git secrets --add-provider [--global] [arguments...] + git secrets --register-aws [--global] + git secrets --register-gcp [--global] + git secrets --register-azure [--global] + git secrets --aws-provider [] + + +Description +----------- + +``git-secrets`` scans commits, commit messages, and ``--no-ff`` merges to +prevent adding secrets into your git repositories. If a commit, +commit message, or any commit in a ``--no-ff`` merge history matches one of +your configured prohibited regular expression patterns, then the commit is +rejected. + + +Installing git-secrets +---------------------- + +``git-secrets`` must be placed somewhere in your PATH so that it is picked up +by ``git`` when running ``git secrets``. + +\*nix (Linux/macOS) +~~~~~~~~~~~~~~~~~~~ + +You can use the ``install`` target of the provided Makefile to install ``git secrets`` and the man page. +You can customize the install path using the PREFIX and MANPREFIX variables. + +:: + + make install + +Windows +~~~~~~~ + +Run the provided ``install.ps1`` powershell script. This will copy the needed files +to an installation directory (``%USERPROFILE%/.git-secrets`` by default) and add +the directory to the current user ``PATH``. + +:: + + PS > ./install.ps1 + +Homebrew (for macOS users) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + brew install git-secrets + +.. warning:: + + You're not done yet! You MUST install the git hooks for every repo that + you wish to use with ``git secrets --install``. + +Here's a quick example of how to ensure a git repository is scanned for secrets +on each commit:: + + cd /path/to/my/repo + git secrets --install + git secrets --register-aws + git secrets --register-azure + git secrets --register-gcp + + +Advanced configuration +---------------------- + +Add a configuration template if you want to add hooks to all repositories you +initialize or clone in the future. + +:: + + git secrets --register-(aws/azure/gcp) --global + + +Add hooks to all your local repositories. + +:: + + git secrets --install ~/.git-templates/git-secrets + git config --global init.templateDir ~/.git-templates/git-secrets + + +Add custom providers to scan for security credentials. + +:: + + git secrets --add-provider -- cat /path/to/secret/file/patterns + + +Before making public a repository +--------------------------------- + +With git-secrets is also possible to scan a repository including all revisions: + +:: + + git secrets --scan-history + + +Options +------- + +Operation Modes +~~~~~~~~~~~~~~~ + +Each of these options must appear first on the command line. + +``--install`` + Installs git hooks for a repository. Once the hooks are installed for a git + repository, commits and non-fast-forward merges for that repository will be prevented + from committing secrets. + +``--scan`` + Scans one or more files for secrets. When a file contains a secret, the + matched text from the file being scanned will be written to stdout and the + script will exit with a non-zero status. Each matched line will be written with + the name of the file that matched, a colon, the line number that matched, + a colon, and then the line of text that matched. If no files are provided, + all files returned by ``git ls-files`` are scanned. + +``--scan-history`` + Scans repository including all revisions. When a file contains a secret, the + matched text from the file being scanned will be written to stdout and the + script will exit with a non-zero status. Each matched line will be written with + the name of the file that matched, a colon, the line number that matched, + a colon, and then the line of text that matched. + +``--list`` + Lists the ``git-secrets`` configuration for the current repo or in the global + git config. + +``--add`` + Adds a prohibited or allowed pattern. + +``--add-provider`` + Registers a secret provider. Secret providers are executables that when + invoked output prohibited patterns that ``git-secrets`` should treat as + prohibited. + +``--register-aws`` + Adds common AWS patterns to the git config and ensures that keys present + in ``~/.aws/credentials`` are not found in any commit. The following + checks are added: + + - AWS Access Key IDs via ``(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}`` + - AWS Secret Access Key assignments via ":" or "=" surrounded by optional + quotes + - AWS account ID assignments via ":" or "=" surrounded by optional quotes + - Allowed patterns for example AWS keys (``AKIAIOSFODNN7EXAMPLE`` and + ``wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY``) + - Known credentials from ``~/.aws/credentials`` + + .. note:: + + While the patterns registered by this command should catch most + instances of AWS credentials, these patterns are **not** guaranteed to + catch them **all**. ``git-secrets`` should be used as an extra means of + insurance -- you still need to do your due diligence to ensure that you + do not commit credentials to a repository. + +``--register-gcp`` + Secret provider which scans files for Google Cloud Platform's (GCP's) crentials JSON files. + +``--register-azure`` + Secret provider which scans files for AZURE credentials + + +``--aws-provider`` + Secret provider that outputs credentials found in an INI file. You can + optionally provide the path to an INI file. + + +Options for ``--install`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +``-f, --force`` + Overwrites existing hooks if present. + +```` + When provided, installs git hooks to the given directory. The current + directory is assumed if ```` is not provided. + + If the provided ```` is not in a git repository, the + directory will be created and hooks will be placed in + ``/hooks``. This can be useful for creating git template + directories using with ``git init --template ``. + + You can run ``git init`` on a repository that has already been initialized. + From the `git init documentation `_: + + From the git documentation: Running ``git init`` in an existing repository + is safe. It will not overwrite things that are already there. The + primary reason for rerunning ``git init`` is to pick up newly added + templates (or to move the repository to another place if + ``--separate-git-dir`` is given). + + The following git hooks are installed: + + 1. ``pre-commit``: Used to check if any of the files changed in the commit + use prohibited patterns. + 2. ``commit-msg``: Used to determine if a commit message contains a + prohibited patterns. + 3. ``prepare-commit-msg``: Used to determine if a merge commit will + introduce a history that contains a prohibited pattern at any point. + Please note that this hook is only invoked for non fast-forward merges. + + .. note:: + + Git only allows a single script to be executed per hook. If the + repository contains Debian-style subdirectories like ``pre-commit.d`` + and ``commit-msg.d``, then the git hooks will be installed into these + directories, which assumes that you've configured the corresponding + hooks to execute all of the scripts found in these directories. If + these git subdirectories are not present, then the git hooks will be + installed to the git repo's ``.git/hooks`` directory. + + +Examples +^^^^^^^^ + +Install git hooks to the current directory:: + + cd /path/to/my/repository + git secrets --install + +Install git hooks to a repository other than the current directory:: + + git secrets --install /path/to/my/repository + +Create a git template that has ``git-secrets`` installed, and then copy that +template into a git repository:: + + git secrets --install ~/.git-templates/git-secrets + git init --template ~/.git-templates/git-secrets + +Overwrite existing hooks if present:: + + git secrets --install -f + + +Options for ``--scan`` +~~~~~~~~~~~~~~~~~~~~~~ + +``-r, --recursive`` + Scans the given files recursively. If a directory is encountered, the + directory will be scanned. If ``-r`` is not provided, directories will be + ignored. + + ``-r`` cannot be used alongside ``--cached``, ``--no-index``, or + ``--untracked``. + +``--cached`` + Searches blobs registered in the index file. + +``--no-index`` + Searches files in the current directory that is not managed by git. + +``--untracked`` + In addition to searching in the tracked files in the working tree, + ``--scan`` also in untracked files. + +``...`` + The path to one or more files on disk to scan for secrets. + + If no files are provided, all files returned by ``git ls-files`` are + scanned. + + +Examples +^^^^^^^^ + +Scan all files in the repo:: + + git secrets --scan + +Scans a single file for secrets:: + + git secrets --scan /path/to/file + +Scans a directory recursively for secrets:: + + git secrets --scan -r /path/to/directory + +Scans multiple files for secrets:: + + git secrets --scan /path/to/file /path/to/other/file + +You can scan by globbing:: + + git secrets --scan /path/to/directory/* + +Scan from stdin:: + + echo 'hello!' | git secrets --scan - + + +Options for ``--list`` +~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Lists only git-secrets configuration in the global git config. + + +Options for ``--add`` +~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds patterns to the global git config + +``-l, --literal`` + Escapes special regular expression characters in the provided pattern so + that the pattern is searched for literally. + +``-a, --allowed`` + Mark the pattern as allowed instead of prohibited. Allowed patterns are + used to filter our false positives. + +```` + The regex pattern to search. + + +Examples +^^^^^^^^ + +Adds a prohibited pattern to the current repo:: + + git secrets --add '[A-Z0-9]{20}' + +Adds a prohibited pattern to the global git config:: + + git secrets --add --global '[A-Z0-9]{20}' + +Adds a string that is scanned for literally (``+`` is escaped):: + + git secrets --add --literal 'foo+bar' + +Add an allowed pattern:: + + git secrets --add -a 'allowed pattern' + + +Options for ``--register-aws`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds AWS specific configuration variables to the global git config. + +Options for ``--register-gcp`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds GCP specific configuration variables to the global git config. + +Options for ``--register-azure`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds AZURE specific configuration variables to the global git config. + +Options for ``--aws-provider`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``[]`` + If provided, specifies the custom path to an INI file to scan. If not + provided, ``~/.aws/credentials`` is assumed. + + +Options for ``--add-provider`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``--global`` + Adds the provider to the global git config. + +```` + Provider command to invoke. When invoked the command is expected to write + prohibited patterns separated by new lines to stdout. Any extra arguments + provided are passed on to the command. + + +Examples +^^^^^^^^ + +Registers a secret provider with arguments:: + + git secrets --add-provider -- git secrets --aws-provider + +Cats secrets out of a file:: + + git secrets --add-provider -- cat /path/to/secret/file/patterns + + +Defining prohibited patterns +---------------------------- + +``egrep``-compatible regular expressions are used to determine if a commit or +commit message contains any prohibited patterns. These regular expressions are +defined using the ``git config`` command. It is important to note that +different systems use different versions of egrep. For example, when running on +macOS, you will use a different version of ``egrep`` than when running on something +like Ubuntu (BSD vs GNU). + +You can add prohibited regular expression patterns to your git config using +``git secrets --add ``. + + +Ignoring false positives +------------------------ + +Sometimes a regular expression might match false positives. For example, git +commit SHAs look a lot like AWS access keys. You can specify many different +regular expression patterns as false positives using the following command: + +:: + + git secrets --add --allowed 'my regex pattern' + +You can also add regular expressions patterns to filter false positives to a +``.gitallowed`` file located in the repository's root directory. Lines starting +with ``#`` are skipped (comment line) and empty lines are also skipped. + +First, git-secrets will extract all lines from a file that contain a prohibited +match. Included in the matched results will be the full path to the name of +the file that was matched, followed by ':', followed by the line number that was +matched, followed by the entire line from the file that was matched by a secret +pattern. Then, if you've defined allowed regular expressions, git-secrets will +check to see if all of the matched lines match at least one of your registered +allowed regular expressions. If all of the lines that were flagged as secret +are canceled out by an allowed match, then the subject text does not contain +any secrets. If any of the matched lines are not matched by an allowed regular +expression, then git-secrets will fail the commit/merge/message. + +.. important:: + + Just as it is a bad practice to add prohibited patterns that are too + greedy, it is also a bad practice to add allowed patterns that are too + forgiving. Be sure to test out your patterns using ad-hoc calls to + ``git secrets --scan $filename`` to ensure they are working as intended. + + +Secret providers +---------------- + +Sometimes you want to check for an exact pattern match against a set of known +secrets. For example, you might want to ensure that no credentials present in +``~/.aws/credentials`` ever show up in a commit. In these cases, it's better to +leave these secrets in one location rather than spread them out across git +repositories in git configs. You can use "secret providers" to fetch these +types of credentials. A secret provider is an executable that when invoked +outputs prohibited patterns separated by new lines. + +You can add secret providers using the ``--add-provider`` command:: + + git secrets --add-provider -- git secrets --aws-provider + +Notice the use of ``--``. This ensures that any arguments associated with the +provider are passed to the provider each time it is invoked when scanning +for secrets. + + +Example walkthrough +------------------- + +Let's take a look at an example. Given the following subject text (stored in +``/tmp/example``):: + + This is a test! + password=ex@mplepassword + password=****** + More test... + +And the following registered patterns: + +:: + + git secrets --add 'password\s*=\s*.+' + git secrets --add --allowed --literal 'ex@mplepassword' + +Running ``git secrets --scan /tmp/example``, the result will +result in the following error output:: + + /tmp/example:3:password=****** + + [ERROR] Matched prohibited pattern + + Possible mitigations: + - Mark false positives as allowed using: git config --add secrets.allowed ... + - List your configured patterns: git config --get-all secrets.patterns + - List your configured allowed patterns: git config --get-all secrets.allowed + - Use --no-verify if this is a one-time false positive + +Breaking this down, the prohibited pattern value of ``password\s*=\s*.+`` will +match the following lines:: + + /tmp/example:2:password=ex@mplepassword + /tmp/example:3:password=****** + +...But the first match will be filtered out due to the fact that it matches the +allowed regular expression of ``ex@mplepassword``. Because there is still a +remaining line that did not match, it is considered a secret. + +Because that matching lines are placed on lines that start with the filename +and line number (e.g., ``/tmp/example:3:...``), you can create allowed +patterns that take filenames and line numbers into account in the regular +expression. For example, you could whitelist an entire file using something +like:: + + git secrets --add --allowed '/tmp/example:.*' + git secrets --scan /tmp/example && echo $? + # Outputs: 0 + +Alternatively, you could allow a specific line number of a file if that +line is unlikely to change using something like the following: + +:: + + git secrets --add --allowed '/tmp/example:3:.*' + git secrets --scan /tmp/example && echo $? + # Outputs: 0 + +Keep this in mind when creating allowed patterns to ensure that your allowed +patterns are not inadvertently matched due to the fact that the filename is +included in the subject text that allowed patterns are matched against. + + +Skipping validation +------------------- + +Use the ``--no-verify`` option in the event of a false positive match in a +commit, merge, or commit message. This will skip the execution of the +git hook and allow you to make the commit or merge. + + +About +------ + +- Author: `Michael Dowling `_ +- Issue tracker: This project's source code and issue tracker can be found at + `https://github.com/awslabs/git-secrets `_ +- Special thanks to Adrian Vatchinsky and Ari Juels of Cornell University for + providing suggestions and feedback. + +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/.github/actions/runleaks/git-secrets/git-secrets b/.github/actions/runleaks/git-secrets/git-secrets new file mode 100755 index 00000000000..236d03b5cd1 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/git-secrets @@ -0,0 +1,409 @@ +#!/usr/bin/env bash +# Copyright 2010-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +NONGIT_OK=1 OPTIONS_SPEC="\ +git secrets --scan [-r|--recursive] [--cached] [--no-index] [--untracked] [...] +git secrets --scan-history +git secrets --install [-f|--force] [] +git secrets --list [--global] +git secrets --add [-a|--allowed] [-l|--literal] [--global] +git secrets --add-provider [--global] [arguments...] +git secrets --register-aws [--global] +git secrets --register-azure [--global] +git secrets --register-gcp [--global] +git secrets --aws-provider [] +-- +scan Scans for prohibited patterns +scan-history Scans repo for prohibited patterns +install Installs git hooks for Git repository or Git template directory +list Lists secret patterns +add Adds a prohibited or allowed pattern, ensuring to de-dupe with existing patterns +add-provider Adds a secret provider that when called outputs secret patterns on new lines +aws-provider Secret provider that outputs credentials found in an ini file +register-aws Adds common AWS patterns to the git config and scans for ~/.aws/credentials +register-azure Adds common Azure patterns to the git config +register-gcp Adds common GCP patterns to the git config +r,recursive --scan scans directories recursively +cached --scan scans searches blobs registered in the index file +no-index --scan searches files in the current directory that is not managed by Git +untracked In addition to searching in the tracked files in the working tree, --scan also in untracked files +f,force --install overwrites hooks if the hook already exists +l,literal --add and --add-allowed patterns are escaped so that they are literal +a,allowed --add adds an allowed pattern instead of a prohibited pattern +global Uses the --global git config +commit_msg_hook* commit-msg hook (internal only) +pre_commit_hook* pre-commit hook (internal only) +prepare_commit_msg_hook* prepare-commit-msg hook (internal only)" + +# Include the git setup script. This parses and normalized CLI arguments. +. "$(git --exec-path)"/git-sh-setup + +load_patterns() { + git config --get-all secrets.patterns + # Execute each provider and use their output to build up patterns + git config --get-all secrets.providers | while read -r cmd; do + # Only split words on '\n\t ' and strip "\r" from the output to account + # for carriage returns being added on Windows systems. Note that this + # trimming is done before the test to ensure that the string is not empty. + local result="$(export IFS=$'\n\t '; $cmd | tr -d $'\r')" + # Do not add empty lines from providers as that would match everything. + if [ -n "${result}" ]; then + echo "${result}" + fi + done +} + +load_allowed() { + git config --get-all secrets.allowed + local gitallowed="$(git rev-parse --show-toplevel)/.gitallowed" + if [ -e "$gitallowed" ]; then + cat $gitallowed | awk 'NF && $1!~/^#/' + fi +} + +# load patterns and combine them with | +load_combined_patterns() { + local patterns=$(load_patterns) + local combined_patterns='' + for pattern in $patterns; do + combined_patterns=${combined_patterns}${pattern}"|" + done + combined_patterns=${combined_patterns%?} + echo $combined_patterns +} + +# Scans files or a repo using patterns. +scan() { + local files=("${@}") options="" + [ "${SCAN_CACHED}" == 1 ] && options+="--cached" + [ "${SCAN_UNTRACKED}" == 1 ] && options+=" --untracked" + [ "${SCAN_NO_INDEX}" == 1 ] && options+=" --no-index" + # Scan using git-grep if there are no files or if git options are applied. + if [ ${#files[@]} -eq 0 ] || [ ! -z "${options}" ]; then + output=$(git_grep $options "${files[@]}") + else + output=$(regular_grep "${files[@]}") + fi + process_output $? "${output}" +} + +# Scans through history using patterns +scan_history() { + # git log does not support multiple patterns, so we need to combine them + local combined_patterns=$(load_combined_patterns) + [ -z "${combined_patterns}" ] && return 0 + # Looks for differences matching the patterns, reduces the number of revisions to scan + local to_scan=$(git log --all -G"${combined_patterns}" --pretty=%H) + # Scan through revisions with findings to normalize output + output=$(GREP_OPTIONS= LC_ALL=C git grep -nwHEI "${combined_patterns}" $to_scan) + process_output $? "${output}" +} + +# Performs a git-grep, taking into account patterns and options. +# Note: this function returns 1 on success, 0 on error. +git_grep() { + local options="$1"; shift + local files=("${@}") combined_patterns=$(load_combined_patterns) + + [ -z "${combined_patterns}" ] && return 1 + GREP_OPTIONS= LC_ALL=C git grep -nwHEI ${options} "${combined_patterns}" -- "${files[@]}" +} + +# Performs a regular grep, taking into account patterns and recursion. +# Note: this function returns 1 on success, 0 on error. +regular_grep() { + local files=("${@}") patterns=$(load_patterns) action='skip' + [ -z "${patterns}" ] && return 1 + [ ${RECURSIVE} -eq 1 ] && action="recurse" + GREP_OPTIONS= LC_ALL=C grep -d "${action}" -nwHEI "${patterns}" "${files[@]}" +} + +# Process the given status ($1) and output variables ($2). +# Takes into account allowed patterns, and if a bad match is found, +# prints an error message and exits 1. +process_output() { + local status="$1" output="$2" + local allowed=$(load_allowed) + case "$status" in + 0) + [ -z "${allowed}" ] && echo "${output}" >&2 && return 1 + # Determine with a negative grep if the found matches are allowed + echo "${output}" | GREP_OPTIONS= LC_ALL=C grep -Ev "${allowed}" >&2 \ + && return 1 || return 0 + ;; + 1) return 0 ;; + *) exit $status + esac +} + +# Calls the given scanning function at $1, shifts, and passes to it $@. +# Exit 0 if success, otherwise exit 1 with error message. +scan_with_fn_or_die() { + local fn="$1"; shift + $fn "$@" && exit 0 + echo >&2 + echo "[ERROR] Matched one or more prohibited patterns" >&2 + echo >&2 + echo "Possible mitigations:" >&2 + echo "- Mark false positives as allowed using: git config --add secrets.allowed ..." >&2 + echo "- Mark false positives as allowed by adding regular expressions to .gitallowed at repository's root directory" >&2 + echo "- List your configured patterns: git config --get-all secrets.patterns" >&2 + echo "- List your configured allowed patterns: git config --get-all secrets.allowed" >&2 + echo "- List your configured allowed patterns in .gitallowed at repository's root directory" >&2 + echo "- Use --no-verify if this is a one-time false positive" >&2 + exit 1 +} + +# Scans a commit message, passed in the path to a file. +commit_msg_hook() { + scan_with_fn_or_die "scan" "$1" +} + +# Scans all files that are about to be committed. +pre_commit_hook() { + SCAN_CACHED=1 + local files=() file found_match=0 rev="4b825dc642cb6eb9a060e54bf8d69288fbee4904" + # Diff against HEAD if this is not the first commit in the repo. + git rev-parse --verify HEAD >/dev/null 2>&1 && rev="HEAD" + # Filter out deleted files using --diff-filter + while IFS= read -r file; do + [ -n "$file" ] && files+=("$file") + done <<< "$(git diff-index --diff-filter 'ACMU' --name-only --cached $rev --)" + scan_with_fn_or_die "scan" "${files[@]}" +} + +# Determines if merging in a commit will introduce tainted history. +prepare_commit_msg_hook() { + case "$2,$3" in + merge,) + local git_head=$(env | grep GITHEAD) # e.g. GITHEAD_=release/1.43 + local sha="${git_head##*=}" # Get just the SHA + local branch=$(git symbolic-ref HEAD) # e.g. refs/heads/master + local dest="${branch#refs/heads/}" # cut out "refs/heads" + git log "${dest}".."${sha}" -p | scan_with_fn_or_die "scan" - + ;; + esac +} + +install_hook() { + local path="$1" hook="$2" cmd="$3" dest + # Determines the approriate path for a hook to be installed + if [ -d "${path}/hooks/${hook}.d" ]; then + dest="${path}/hooks/${hook}.d/git-secrets" + else + dest="${path}/hooks/${hook}" + fi + [ -f "${dest}" ] && [ "${FORCE}" -ne 1 ] \ + && die "${dest} already exists. Use -f to force" + echo "#!/usr/bin/env bash" > "${dest}" + echo "git secrets --${cmd} -- \"\$@\"" >> "${dest}" + chmod +x "${dest}" + say "$(tput setaf 2)✓$(tput sgr 0) Installed ${hook} hook to ${dest}" +} + +install_all_hooks() { + install_hook "$1" "commit-msg" "commit_msg_hook" + install_hook "$1" "pre-commit" "pre_commit_hook" + install_hook "$1" "prepare-commit-msg" "prepare_commit_msg_hook" +} + +# Adds a git config pattern, ensuring to de-dupe +add_config() { + local key="$1"; shift + local value="$@" + if [ ${LITERAL} -eq 1 ]; then + value=$(sed 's/[\.|$(){}?+*^]/\\&/g' <<< "${value}") + fi + if [ ${GLOBAL} -eq 1 ]; then + git config --global --get-all $key | grep -Fq "${value}" && return 1 + git config --global --add "${key}" "${value}" + else + git config --get-all $key | grep -Fq "${value}" && return 1 + git config --add "${key}" "${value}" + fi +} + +register_aws() { + # Reusable regex patterns + local aws="(AWS|aws|Aws)?_?" quote="(\"|')" connect="\s*(:|=>|=)\s*" + local opt_quote="${quote}?" + add_config 'secrets.providers' 'git secrets --aws-provider' + add_config 'secrets.patterns' '(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}' + add_config 'secrets.patterns' "${opt_quote}${aws}(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)${opt_quote}${connect}${opt_quote}[A-Za-z0-9/\+=]{40}${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}${aws}(ACCOUNT|account|Account)_?(ID|id|Id)?${opt_quote}${connect}${opt_quote}[0-9]{4}\-?[0-9]{4}\-?[0-9]{4}${opt_quote}" + add_config 'secrets.allowed' 'AKIAIOSFODNN7EXAMPLE' + add_config 'secrets.allowed' "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + + if [[ $? == 0 ]]; then + echo 'OK' + fi + + exit $? +} + +register_azure() { + # Reusable regex patterns + local azure="(AZURE|azure|Azure)?_?" quote="(\"|')" connect="\s*(:|=>|=)\s*" + local opt_quote="${quote}" + add_config 'secrets.patterns' "${opt_quote}[0-9A-Fa-f]{8}-([0-9A-Fa-f]{4}-){3}[0-9A-Fa-f]{12}${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][o|O][n|N][m|M][i|I][c|C][r|R][o|O][s|S][o|O][f|F][t|T][.][c|C][o|O][m|M]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][b|B][l|L][o|O][b|B][.][c|C][o|O][r|R][e|E][.][w|W][i|I][n|N][d|D][o|O][w|W][s|S][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][q|Q][u|U][e|E][u|U][e|E][.][c|C][o|O][r|R][e|E][.][w|W][i|I][n|N][d|D][o|O][w|W][s|S][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][t|T][a|A][b|B][l|L][e|E][.][c|C][o|O][r|R][e|E][.][w|W][i|I][n|N][d|D][o|O][w|W][s|S][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][d|D][a|A][t|T][a|A][b|B][a|A][s|S][e|E][.][w|W][i|I][n|N][d|D][o|O][w|W][s|S][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][s|S][e|E][r|R][v|V][i|I][c|C][e|E][b|B][u|U][s|S][.][w|W][i|I][n|N][d|D][o|O][w|W][s|S][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][t|T][i|I][m|M][e|E][s|S][e|E][r|R][i|I][e|E][s|S][.][a|A][z|Z][u|U][r|R][e|E][.][c|C][o|O][m|M]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][a|T][c|C][c|C][e|E][s|S][s|S][c|C][o|O][n|N][t|T][r|R][o|O][l|L][.][w|W][i|I][n|N][d|D][o|O][w|W][s|S][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][a|A][z|Z][u|U][r|R][e|E][h|H][d|D][i|I][n|N][s|S][i|I][g|G][h|H][t|T][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][c|C][l|L][o|O][u|U][d|D][a|A][p|P][p|P][.][a|A][z|Z][u|U][r|R][e|E][.][c|C][o|O][m|M]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][c|C][l|L][o|O][u|U][d|D][a|A][p|P][p|P][.][n|N][e|E][t|T]${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}.*[0-9a-zA-Z]{2,256}[.][d|D][o|O][c|C][u|U][m|M][e|E][n|N][t|T][s|S][.][a|A][z|Z][u|U][r|R][e|E][.][c|C][o|O][m|M]${opt_quote}" + add_config 'secrets.patterns' "^-----BEGIN (RSA|EC|DSA|GPP) PRIVATE KEY-----$" + add_config 'secrets.patterns' "(\"|')[A-Z0-9a-z[:punct:]]{32}(\"|')$" + add_config 'secrets.patterns' "(\"|')[A-Z0-9a-z[:punct:]]{88}(\"|')$" + + + + + if [[ $? == 0 ]]; then + echo 'OK' + fi + + exit $? +} + + + + +register_gcp() { + # Reusable regex patterns + add_config 'secrets.patterns' '\bprivate_key.*\b' + + + if [[ $? == 0 ]]; then + echo 'OK' + fi + + exit $? +} + +aws_provider() { + local fi="$1" + [ -z "$fi" ] && fi=~/.aws/credentials + # Find keys and ensure that special characters are escaped. + if [ -f $fi ]; then + awk -F "=" '/aws_access_key_id|aws_secret_access_key/ {print $2}' $fi \ + | tr -d ' "' \ + | sed 's/[]\.|$(){}?+*^]/\\&/g' + fi +} + +# Ensures that the command is what was expected for an option. +assert_option_for_command() { + local expected_command="$1" + local option_name="$2" + if [ "${COMMAND}" != "${expected_command}" ]; then + die "${option_name} can only be supplied with the ${expected_command} subcommand" + fi +} + +declare COMMAND="$1" FORCE=0 RECURSIVE=0 LITERAL=0 GLOBAL=0 ALLOWED=0 +declare SCAN_CACHED=0 SCAN_NO_INDEX=0 SCAN_UNTRACKED=0 + +# Shift off the command name +shift 1 +while [ "$#" -ne 0 ]; do + case "$1" in + -f) + assert_option_for_command "--install" "-f|--force" + FORCE=1 + ;; + -r) + assert_option_for_command "--scan" "-r|--recursive" + RECURSIVE=1 + ;; + -a) + assert_option_for_command "--add" "-a|--allowed" + ALLOWED=1 + ;; + -l) + assert_option_for_command "--add" "-l|--literal" + LITERAL=1 + ;; + --cached) + assert_option_for_command "--scan" "--cached" + SCAN_CACHED=1 + ;; + --no-index) + assert_option_for_command "--scan" "--no-index" + SCAN_NO_INDEX=1 + ;; + --untracked) + assert_option_for_command "--scan" "--untracked" + SCAN_UNTRACKED=1 + ;; + --global) GLOBAL=1 ;; + --) shift; break ;; + esac + shift +done + +# Ensure that recursive is not applied with mutually exclusive options. +if [ ${RECURSIVE} -eq 1 ]; then + if [ ${SCAN_CACHED} -eq 1 ] \ + || [ ${SCAN_NO_INDEX} -eq 1 ] \ + || [ ${SCAN_UNTRACKED} -eq 1 ]; + then + die "-r|--recursive cannot be supplied with --cached, --no-index, or --untracked" + fi +fi + +case "${COMMAND}" in + -h|--help|--) "$0" -h; exit 0 ;; + --add-provider) add_config "secrets.providers" "$@" ;; + --register-aws) register_aws ;; + --register-azure) register_azure ;; + --register-gcp) register_gcp ;; + --aws-provider) aws_provider "$1" ;; + --commit_msg_hook|--pre_commit_hook|--prepare_commit_msg_hook) + ${COMMAND:2} "$@" + ;; + --add) + if [ ${ALLOWED} -eq 1 ]; then + add_config "secrets.allowed" "$1" + else + add_config "secrets.patterns" "$1" + fi + ;; + --scan) scan_with_fn_or_die "scan" "$@" ;; + --scan-history) scan_with_fn_or_die "scan_history" "$@" ;; + --list) + if [ ${GLOBAL} -eq 1 ]; then + git config --global --get-regex secrets.* + else + git config --get-regex secrets.* + fi + ;; + --install) + DIRECTORY="$1" + if [ -z "${DIRECTORY}" ]; then + DIRECTORY=$(git rev-parse --git-dir) || die "Not in a Git repository" + elif [ -d "${DIRECTORY}"/.git ]; then + DIRECTORY="${DIRECTORY}/.git" + fi + mkdir -p "${DIRECTORY}/hooks" || die "Could not create dir: ${DIRECTORY}" + install_all_hooks "${DIRECTORY}" + ;; + *) echo "Unknown option: ${COMMAND}" && "$0" -h ;; +esac diff --git a/.github/actions/runleaks/git-secrets/git-secrets.1 b/.github/actions/runleaks/git-secrets/git-secrets.1 new file mode 100644 index 00000000000..1c6d25c3413 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/git-secrets.1 @@ -0,0 +1,843 @@ +.\" Man page generated from reStructuredText. +. +.TH GIT-SECRETS "" "" "" +.SH NAME +git-secrets \- Prevents you from committing passwords and other sensitive information to a git repository. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SS Contents +.INDENT 0.0 +.IP \(bu 2 +\fI\%Synopsis\fP +.IP \(bu 2 +\fI\%Description\fP +.IP \(bu 2 +\fI\%Installing git\-secrets\fP +.INDENT 2.0 +.IP \(bu 2 +\fI\%*nix (Linux/macOS)\fP +.IP \(bu 2 +\fI\%Windows\fP +.IP \(bu 2 +\fI\%Homebrew (for macOS users)\fP +.UNINDENT +.IP \(bu 2 +\fI\%Advanced configuration\fP +.IP \(bu 2 +\fI\%Before making public a repository\fP +.IP \(bu 2 +\fI\%Options\fP +.INDENT 2.0 +.IP \(bu 2 +\fI\%Operation Modes\fP +.IP \(bu 2 +\fI\%Options for \fB\-\-install\fP\fP +.IP \(bu 2 +\fI\%Options for \fB\-\-scan\fP\fP +.IP \(bu 2 +\fI\%Options for \fB\-\-list\fP\fP +.IP \(bu 2 +\fI\%Options for \fB\-\-add\fP\fP +.IP \(bu 2 +\fI\%Options for \fB\-\-register\-aws\fP\fP +.IP \(bu 2 +\fI\%Options for \fB\-\-aws\-provider\fP\fP +.IP \(bu 2 +\fI\%Options for \fB\-\-add\-provider\fP\fP +.UNINDENT +.IP \(bu 2 +\fI\%Defining prohibited patterns\fP +.IP \(bu 2 +\fI\%Ignoring false positives\fP +.IP \(bu 2 +\fI\%Secret providers\fP +.IP \(bu 2 +\fI\%Example walkthrough\fP +.IP \(bu 2 +\fI\%Skipping validation\fP +.IP \(bu 2 +\fI\%About\fP +.UNINDENT +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan [\-r|\-\-recursive] [\-\-cached] [\-\-no\-index] [\-\-untracked] [...] +git secrets \-\-scan\-history +git secrets \-\-install [\-f|\-\-force] [] +git secrets \-\-list [\-\-global] +git secrets \-\-add [\-a|\-\-allowed] [\-l|\-\-literal] [\-\-global] +git secrets \-\-add\-provider [\-\-global] [arguments...] +git secrets \-\-register\-aws [\-\-global] +git secrets \-\-aws\-provider [] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +\fBgit\-secrets\fP scans commits, commit messages, and \fB\-\-no\-ff\fP merges to +prevent adding secrets into your git repositories. If a commit, +commit message, or any commit in a \fB\-\-no\-ff\fP merge history matches one of +your configured prohibited regular expression patterns, then the commit is +rejected. +.SH INSTALLING GIT-SECRETS +.sp +\fBgit\-secrets\fP must be placed somewhere in your PATH so that it is picked up +by \fBgit\fP when running \fBgit secrets\fP\&. +.SS *nix (Linux/macOS) +.IP "System Message: WARNING/2 (README.rst:, line 43)" +Title underline too short. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\e*nix (Linux/macOS) +~~~~~~~~~~~~~~~~~ +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You can use the \fBinstall\fP target of the provided Makefile to install \fBgit secrets\fP and the man page. +You can customize the install path using the PREFIX and MANPREFIX variables. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +make install +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Windows +.sp +Run the provided \fBinstall.ps1\fP powershell script. This will copy the needed files +to an installation directory (\fB%USERPROFILE%/.git\-secrets\fP by default) and add +the directory to the current user \fBPATH\fP\&. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +PS > ./install.ps1 +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Homebrew (for macOS users) +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +brew install git\-secrets +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBWARNING:\fP +.INDENT 0.0 +.INDENT 3.5 +You\(aqre not done yet! You MUST install the git hooks for every repo that +you wish to use with \fBgit secrets \-\-install\fP\&. +.UNINDENT +.UNINDENT +.sp +Here\(aqs a quick example of how to ensure a git repository is scanned for secrets +on each commit: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +cd /path/to/my/repo +git secrets \-\-install +git secrets \-\-register\-aws +.ft P +.fi +.UNINDENT +.UNINDENT +.SH ADVANCED CONFIGURATION +.sp +Add a configuration template if you want to add hooks to all repositories you +initialize or clone in the future. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-register\-aws \-\-global +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Add hooks to all your local repositories. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-install ~/.git\-templates/git\-secrets +git config \-\-global init.templateDir ~/.git\-templates/git\-secrets +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Add custom providers to scan for security credentials. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add\-provider \-\- cat /path/to/secret/file/patterns +.ft P +.fi +.UNINDENT +.UNINDENT +.SH BEFORE MAKING PUBLIC A REPOSITORY +.sp +With git\-secrets is also possible to scan a repository including all revisions: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan\-history +.ft P +.fi +.UNINDENT +.UNINDENT +.SH OPTIONS +.SS Operation Modes +.sp +Each of these options must appear first on the command line. +.INDENT 0.0 +.TP +.B \fB\-\-install\fP +Installs git hooks for a repository. Once the hooks are installed for a git +repository, commits and non\-fast\-forward merges for that repository will be prevented +from committing secrets. +.TP +.B \fB\-\-scan\fP +Scans one or more files for secrets. When a file contains a secret, the +matched text from the file being scanned will be written to stdout and the +script will exit with a non\-zero status. Each matched line will be written with +the name of the file that matched, a colon, the line number that matched, +a colon, and then the line of text that matched. If no files are provided, +all files returned by \fBgit ls\-files\fP are scanned. +.TP +.B \fB\-\-scan\-history\fP +Scans repository including all revisions. When a file contains a secret, the +matched text from the file being scanned will be written to stdout and the +script will exit with a non\-zero status. Each matched line will be written with +the name of the file that matched, a colon, the line number that matched, +a colon, and then the line of text that matched. +.TP +.B \fB\-\-list\fP +Lists the \fBgit\-secrets\fP configuration for the current repo or in the global +git config. +.TP +.B \fB\-\-add\fP +Adds a prohibited or allowed pattern. +.TP +.B \fB\-\-add\-provider\fP +Registers a secret provider. Secret providers are executables that when +invoked output prohibited patterns that \fBgit\-secrets\fP should treat as +prohibited. +.TP +.B \fB\-\-register\-aws\fP +Adds common AWS patterns to the git config and ensures that keys present +in \fB~/.aws/credentials\fP are not found in any commit. The following +checks are added: +.INDENT 7.0 +.IP \(bu 2 +AWS Access Key IDs via \fB(A3T[A\-Z0\-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A\-Z0\-9]{16}\fP +.IP \(bu 2 +AWS Secret Access Key assignments via ":" or "=" surrounded by optional +quotes +.IP \(bu 2 +AWS account ID assignments via ":" or "=" surrounded by optional quotes +.IP \(bu 2 +Allowed patterns for example AWS keys (\fBAKIAIOSFODNN7EXAMPLE\fP and +\fBwJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\fP) +.IP \(bu 2 +Known credentials from \fB~/.aws/credentials\fP +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 7.0 +.INDENT 3.5 +While the patterns registered by this command should catch most +instances of AWS credentials, these patterns are \fBnot\fP guaranteed to +catch them \fBall\fP\&. \fBgit\-secrets\fP should be used as an extra means of +insurance \-\- you still need to do your due diligence to ensure that you +do not commit credentials to a repository. +.UNINDENT +.UNINDENT +.TP +.B \fB\-\-aws\-provider\fP +Secret provider that outputs credentials found in an INI file. You can +optionally provide the path to an INI file. +.UNINDENT +.SS Options for \fB\-\-install\fP +.INDENT 0.0 +.TP +.B \fB\-f, \-\-force\fP +Overwrites existing hooks if present. +.TP +.B \fB\fP +When provided, installs git hooks to the given directory. The current +directory is assumed if \fB\fP is not provided. +.sp +If the provided \fB\fP is not in a git repository, the +directory will be created and hooks will be placed in +\fB/hooks\fP\&. This can be useful for creating git template +directories using with \fBgit init \-\-template \fP\&. +.sp +You can run \fBgit init\fP on a repository that has already been initialized. +From the \fI\%git init documentation\fP: +.INDENT 7.0 +.INDENT 3.5 +From the git documentation: Running \fBgit init\fP in an existing repository +is safe. It will not overwrite things that are already there. The +primary reason for rerunning \fBgit init\fP is to pick up newly added +templates (or to move the repository to another place if +\fB\-\-separate\-git\-dir\fP is given). +.UNINDENT +.UNINDENT +.sp +The following git hooks are installed: +.INDENT 7.0 +.IP 1. 3 +\fBpre\-commit\fP: Used to check if any of the files changed in the commit +use prohibited patterns. +.IP 2. 3 +\fBcommit\-msg\fP: Used to determine if a commit message contains a +prohibited patterns. +.IP 3. 3 +\fBprepare\-commit\-msg\fP: Used to determine if a merge commit will +introduce a history that contains a prohibited pattern at any point. +Please note that this hook is only invoked for non fast\-forward merges. +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 7.0 +.INDENT 3.5 +Git only allows a single script to be executed per hook. If the +repository contains Debian\-style subdirectories like \fBpre\-commit.d\fP +and \fBcommit\-msg.d\fP, then the git hooks will be installed into these +directories, which assumes that you\(aqve configured the corresponding +hooks to execute all of the scripts found in these directories. If +these git subdirectories are not present, then the git hooks will be +installed to the git repo\(aqs \fB\&.git/hooks\fP directory. +.UNINDENT +.UNINDENT +.UNINDENT +.SS Examples +.sp +Install git hooks to the current directory: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +cd /path/to/my/repository +git secrets \-\-install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Install git hooks to a repository other than the current directory: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-install /path/to/my/repository +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Create a git template that has \fBgit\-secrets\fP installed, and then copy that +template into a git repository: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-install ~/.git\-templates/git\-secrets +git init \-\-template ~/.git\-templates/git\-secrets +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Overwrite existing hooks if present: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-install \-f +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Options for \fB\-\-scan\fP +.INDENT 0.0 +.TP +.B \fB\-r, \-\-recursive\fP +Scans the given files recursively. If a directory is encountered, the +directory will be scanned. If \fB\-r\fP is not provided, directories will be +ignored. +.sp +\fB\-r\fP cannot be used alongside \fB\-\-cached\fP, \fB\-\-no\-index\fP, or +\fB\-\-untracked\fP\&. +.TP +.B \fB\-\-cached\fP +Searches blobs registered in the index file. +.TP +.B \fB\-\-no\-index\fP +Searches files in the current directory that is not managed by git. +.TP +.B \fB\-\-untracked\fP +In addition to searching in the tracked files in the working tree, +\fB\-\-scan\fP also in untracked files. +.TP +.B \fB...\fP +The path to one or more files on disk to scan for secrets. +.sp +If no files are provided, all files returned by \fBgit ls\-files\fP are +scanned. +.UNINDENT +.SS Examples +.sp +Scan all files in the repo: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scans a single file for secrets: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan /path/to/file +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scans a directory recursively for secrets: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan \-r /path/to/directory +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scans multiple files for secrets: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan /path/to/file /path/to/other/file +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You can scan by globbing: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-scan /path/to/directory/* +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Scan from stdin: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +echo \(aqhello!\(aq | git secrets \-\-scan \- +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Options for \fB\-\-list\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Lists only git\-secrets configuration in the global git config. +.UNINDENT +.SS Options for \fB\-\-add\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Adds patterns to the global git config +.TP +.B \fB\-l, \-\-literal\fP +Escapes special regular expression characters in the provided pattern so +that the pattern is searched for literally. +.TP +.B \fB\-a, \-\-allowed\fP +Mark the pattern as allowed instead of prohibited. Allowed patterns are +used to filter our false positives. +.TP +.B \fB\fP +The regex pattern to search. +.UNINDENT +.SS Examples +.sp +Adds a prohibited pattern to the current repo: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \(aq[A\-Z0\-9]{20}\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Adds a prohibited pattern to the global git config: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-global \(aq[A\-Z0\-9]{20}\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Adds a string that is scanned for literally (\fB+\fP is escaped): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-literal \(aqfoo+bar\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Add an allowed pattern: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-a \(aqallowed pattern\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Options for \fB\-\-register\-aws\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Adds AWS specific configuration variables to the global git config. +.UNINDENT +.SS Options for \fB\-\-aws\-provider\fP +.INDENT 0.0 +.TP +.B \fB[]\fP +If provided, specifies the custom path to an INI file to scan. If not +provided, \fB~/.aws/credentials\fP is assumed. +.UNINDENT +.SS Options for \fB\-\-add\-provider\fP +.INDENT 0.0 +.TP +.B \fB\-\-global\fP +Adds the provider to the global git config. +.TP +.B \fB\fP +Provider command to invoke. When invoked the command is expected to write +prohibited patterns separated by new lines to stdout. Any extra arguments +provided are passed on to the command. +.UNINDENT +.SS Examples +.sp +Registers a secret provider with arguments: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add\-provider \-\- git secrets \-\-aws\-provider +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Cats secrets out of a file: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add\-provider \-\- cat /path/to/secret/file/patterns +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DEFINING PROHIBITED PATTERNS +.sp +\fBegrep\fP\-compatible regular expressions are used to determine if a commit or +commit message contains any prohibited patterns. These regular expressions are +defined using the \fBgit config\fP command. It is important to note that +different systems use different versions of egrep. For example, when running on +macOS, you will use a different version of \fBegrep\fP than when running on something +like Ubuntu (BSD vs GNU). +.sp +You can add prohibited regular expression patterns to your git config using +\fBgit secrets \-\-add \fP\&. +.SH IGNORING FALSE POSITIVES +.sp +Sometimes a regular expression might match false positives. For example, git +commit SHAs look a lot like AWS access keys. You can specify many different +regular expression patterns as false positives using the following command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-allowed \(aqmy regex pattern\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You can also add regular expressions patterns to filter false positives to a +\fB\&.gitallowed\fP file located in the repository\(aqs root directory. Lines starting +with \fB#\fP are skipped (comment line) and empty lines are also skipped. +.sp +First, git\-secrets will extract all lines from a file that contain a prohibited +match. Included in the matched results will be the full path to the name of +the file that was matched, followed by \(aq:\(aq, followed by the line number that was +matched, followed by the entire line from the file that was matched by a secret +pattern. Then, if you\(aqve defined allowed regular expressions, git\-secrets will +check to see if all of the matched lines match at least one of your registered +allowed regular expressions. If all of the lines that were flagged as secret +are canceled out by an allowed match, then the subject text does not contain +any secrets. If any of the matched lines are not matched by an allowed regular +expression, then git\-secrets will fail the commit/merge/message. +.sp +\fBIMPORTANT:\fP +.INDENT 0.0 +.INDENT 3.5 +Just as it is a bad practice to add prohibited patterns that are too +greedy, it is also a bad practice to add allowed patterns that are too +forgiving. Be sure to test out your patterns using ad\-hoc calls to +\fBgit secrets \-\-scan $filename\fP to ensure they are working as intended. +.UNINDENT +.UNINDENT +.SH SECRET PROVIDERS +.sp +Sometimes you want to check for an exact pattern match against a set of known +secrets. For example, you might want to ensure that no credentials present in +\fB~/.aws/credentials\fP ever show up in a commit. In these cases, it\(aqs better to +leave these secrets in one location rather than spread them out across git +repositories in git configs. You can use "secret providers" to fetch these +types of credentials. A secret provider is an executable that when invoked +outputs prohibited patterns separated by new lines. +.sp +You can add secret providers using the \fB\-\-add\-provider\fP command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add\-provider \-\- git secrets \-\-aws\-provider +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Notice the use of \fB\-\-\fP\&. This ensures that any arguments associated with the +provider are passed to the provider each time it is invoked when scanning +for secrets. +.SH EXAMPLE WALKTHROUGH +.sp +Let\(aqs take a look at an example. Given the following subject text (stored in +\fB/tmp/example\fP): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +This is a test! +password=ex@mplepassword +password=****** +More test... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And the following registered patterns: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \(aqpassword\es*=\es*.+\(aq +git secrets \-\-add \-\-allowed \-\-literal \(aqex@mplepassword\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Running \fBgit secrets \-\-scan /tmp/example\fP, the result will +result in the following error output: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +/tmp/example:3:password=****** + +[ERROR] Matched prohibited pattern + +Possible mitigations: +\- Mark false positives as allowed using: git config \-\-add secrets.allowed ... +\- List your configured patterns: git config \-\-get\-all secrets.patterns +\- List your configured allowed patterns: git config \-\-get\-all secrets.allowed +\- Use \-\-no\-verify if this is a one\-time false positive +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Breaking this down, the prohibited pattern value of \fBpassword\es*=\es*.+\fP will +match the following lines: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +/tmp/example:2:password=ex@mplepassword +/tmp/example:3:password=****** +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\&...But the first match will be filtered out due to the fact that it matches the +allowed regular expression of \fBex@mplepassword\fP\&. Because there is still a +remaining line that did not match, it is considered a secret. +.sp +Because that matching lines are placed on lines that start with the filename +and line number (e.g., \fB/tmp/example:3:...\fP), you can create allowed +patterns that take filenames and line numbers into account in the regular +expression. For example, you could whitelist an entire file using something +like: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-allowed \(aq/tmp/example:.*\(aq +git secrets \-\-scan /tmp/example && echo $? +# Outputs: 0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Alternatively, you could allow a specific line number of a file if that +line is unlikely to change using something like the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +git secrets \-\-add \-\-allowed \(aq/tmp/example:3:.*\(aq +git secrets \-\-scan /tmp/example && echo $? +# Outputs: 0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Keep this in mind when creating allowed patterns to ensure that your allowed +patterns are not inadvertently matched due to the fact that the filename is +included in the subject text that allowed patterns are matched against. +.SH SKIPPING VALIDATION +.sp +Use the \fB\-\-no\-verify\fP option in the event of a false positive match in a +commit, merge, or commit message. This will skip the execution of the +git hook and allow you to make the commit or merge. +.SH ABOUT +.INDENT 0.0 +.IP \(bu 2 +Author: \fI\%Michael Dowling\fP +.IP \(bu 2 +Issue tracker: This project\(aqs source code and issue tracker can be found at +\fI\%https://github.com/awslabs/git\-secrets\fP +.IP \(bu 2 +Special thanks to Adrian Vatchinsky and Ari Juels of Cornell University for +providing suggestions and feedback. +.UNINDENT +.sp +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +.\" Generated by docutils manpage writer. +. diff --git a/.github/actions/runleaks/git-secrets/install.ps1 b/.github/actions/runleaks/git-secrets/install.ps1 new file mode 100644 index 00000000000..fbffbfa9906 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/install.ps1 @@ -0,0 +1,48 @@ +Param([string]$InstallationDirectory = $($Env:USERPROFILE + "\.git-secrets")) + +Write-Host "Checking to see if installation directory already exists..." +if (-not (Test-Path $InstallationDirectory)) +{ + Write-Host "Creating installation directory." + New-Item -ItemType Directory -Path $InstallationDirectory | Out-Null +} +else +{ + Write-Host "Installation directory already exists." +} + +Write-Host "Copying files." +Copy-Item ./git-secrets -Destination $InstallationDirectory -Force +Copy-Item ./git-secrets.1 -Destination $InstallationDirectory -Force + +Write-Host "Checking if directory already exists in Path..." +$currentPath = [Environment]::GetEnvironmentVariable("PATH", "User") +if ($currentPath -notlike "*$InstallationDirectory*") +{ + Write-Host "Adding to path." + $newPath = $currentPath + if(-not ($newPath.EndsWith(";"))) + { + $newPath = $newPath + ";" + } + $newPath = $newPath + $InstallationDirectory + [Environment]::SetEnvironmentVariable("PATH", $newPath, "User") +} +else +{ + Write-Host "Already in Path." +} + +# Adding to Session +Write-Host "Adding to user session." +$currentSessionPath = $Env:Path +if ($currentSessionPath -notlike "*$InstallationDirectory*") +{ + if(-not ($currentSessionPath.EndsWith(";"))) + { + $currentSessionPath = $currentSessionPath + ";" + } + $Env:Path = $currentSessionPath + $InstallationDirectory +} + +Write-Host "Done." \ No newline at end of file diff --git a/.github/actions/runleaks/git-secrets/test/bats/LICENSE b/.github/actions/runleaks/git-secrets/test/bats/LICENSE new file mode 100644 index 00000000000..bac4eb29ccf --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/bats/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2014 Sam Stephenson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.github/actions/runleaks/git-secrets/test/bats/bin/bats b/.github/actions/runleaks/git-secrets/test/bats/bin/bats new file mode 120000 index 00000000000..a50a884e581 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/bats/bin/bats @@ -0,0 +1 @@ +../libexec/bats \ No newline at end of file diff --git a/.github/actions/runleaks/git-secrets/test/bats/libexec/bats b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats new file mode 100755 index 00000000000..71f392f757e --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -e + +version() { + echo "Bats 0.4.0" +} + +usage() { + version + echo "Usage: bats [-c] [-p | -t] [ ...]" +} + +help() { + usage + echo + echo " is the path to a Bats test file, or the path to a directory" + echo " containing Bats test files." + echo + echo " -c, --count Count the number of test cases without running any tests" + echo " -h, --help Display this help message" + echo " -p, --pretty Show results in pretty format (default for terminals)" + echo " -t, --tap Show results in TAP format" + echo " -v, --version Display the version number" + echo + echo " For more information, see https://github.com/sstephenson/bats" + echo +} + +resolve_link() { + $(type -p greadlink readlink | head -1) "$1" +} + +abs_dirname() { + local cwd="$(pwd)" + local path="$1" + + while [ -n "$path" ]; do + cd "${path%/*}" + local name="${path##*/}" + path="$(resolve_link "$name" || true)" + done + + pwd + cd "$cwd" +} + +expand_path() { + { cd "$(dirname "$1")" 2>/dev/null + local dirname="$PWD" + cd "$OLDPWD" + echo "$dirname/$(basename "$1")" + } || echo "$1" +} + +BATS_LIBEXEC="$(abs_dirname "$0")" +export BATS_PREFIX="$(abs_dirname "$BATS_LIBEXEC")" +export BATS_CWD="$(abs_dirname .)" +export PATH="$BATS_LIBEXEC:$PATH" + +options=() +arguments=() +for arg in "$@"; do + if [ "${arg:0:1}" = "-" ]; then + if [ "${arg:1:1}" = "-" ]; then + options[${#options[*]}]="${arg:2}" + else + index=1 + while option="${arg:$index:1}"; do + [ -n "$option" ] || break + options[${#options[*]}]="$option" + let index+=1 + done + fi + else + arguments[${#arguments[*]}]="$arg" + fi +done + +unset count_flag pretty +[ -t 0 ] && [ -t 1 ] && pretty="1" +[ -n "$CI" ] && pretty="" + +for option in "${options[@]}"; do + case "$option" in + "h" | "help" ) + help + exit 0 + ;; + "v" | "version" ) + version + exit 0 + ;; + "c" | "count" ) + count_flag="-c" + ;; + "t" | "tap" ) + pretty="" + ;; + "p" | "pretty" ) + pretty="1" + ;; + * ) + usage >&2 + exit 1 + ;; + esac +done + +if [ "${#arguments[@]}" -eq 0 ]; then + usage >&2 + exit 1 +fi + +filenames=() +for filename in "${arguments[@]}"; do + if [ -d "$filename" ]; then + shopt -s nullglob + for suite_filename in "$(expand_path "$filename")"/*.bats; do + filenames["${#filenames[@]}"]="$suite_filename" + done + shopt -u nullglob + else + filenames["${#filenames[@]}"]="$(expand_path "$filename")" + fi +done + +if [ "${#filenames[@]}" -eq 1 ]; then + command="bats-exec-test" +else + command="bats-exec-suite" +fi + +if [ -n "$pretty" ]; then + extended_syntax_flag="-x" + formatter="bats-format-tap-stream" +else + extended_syntax_flag="" + formatter="cat" +fi + +set -o pipefail execfail +exec "$command" $count_flag $extended_syntax_flag "${filenames[@]}" | "$formatter" diff --git a/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-exec-suite b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-exec-suite new file mode 100755 index 00000000000..29ab255d062 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-exec-suite @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -e + +count_only_flag="" +if [ "$1" = "-c" ]; then + count_only_flag=1 + shift +fi + +extended_syntax_flag="" +if [ "$1" = "-x" ]; then + extended_syntax_flag="-x" + shift +fi + +trap "kill 0; exit 1" int + +count=0 +for filename in "$@"; do + let count+="$(bats-exec-test -c "$filename")" +done + +if [ -n "$count_only_flag" ]; then + echo "$count" + exit +fi + +echo "1..$count" +status=0 +offset=0 +for filename in "$@"; do + index=0 + { + IFS= read -r # 1..n + while IFS= read -r line; do + case "$line" in + "begin "* ) + let index+=1 + echo "${line/ $index / $(($offset + $index)) }" + ;; + "ok "* | "not ok "* ) + [ -n "$extended_syntax_flag" ] || let index+=1 + echo "${line/ $index / $(($offset + $index)) }" + [ "${line:0:6}" != "not ok" ] || status=1 + ;; + * ) + echo "$line" + ;; + esac + done + } < <( bats-exec-test $extended_syntax_flag "$filename" ) + offset=$(($offset + $index)) +done + +exit "$status" diff --git a/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-exec-test b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-exec-test new file mode 100755 index 00000000000..8f3bd5102e4 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-exec-test @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +set -e +set -E +set -T + +BATS_COUNT_ONLY="" +if [ "$1" = "-c" ]; then + BATS_COUNT_ONLY=1 + shift +fi + +BATS_EXTENDED_SYNTAX="" +if [ "$1" = "-x" ]; then + BATS_EXTENDED_SYNTAX="$1" + shift +fi + +BATS_TEST_FILENAME="$1" +if [ -z "$BATS_TEST_FILENAME" ]; then + echo "usage: bats-exec " >&2 + exit 1 +elif [ ! -f "$BATS_TEST_FILENAME" ]; then + echo "bats: $BATS_TEST_FILENAME does not exist" >&2 + exit 1 +else + shift +fi + +BATS_TEST_DIRNAME="$(dirname "$BATS_TEST_FILENAME")" +BATS_TEST_NAMES=() + +load() { + local name="$1" + local filename + + if [ "${name:0:1}" = "/" ]; then + filename="${name}" + else + filename="$BATS_TEST_DIRNAME/${name}.bash" + fi + + [ -f "$filename" ] || { + echo "bats: $filename does not exist" >&2 + exit 1 + } + + source "${filename}" +} + +run() { + local e E T oldIFS + [[ ! "$-" =~ e ]] || e=1 + [[ ! "$-" =~ E ]] || E=1 + [[ ! "$-" =~ T ]] || T=1 + set +e + set +E + set +T + output="$("$@" 2>&1)" + status="$?" + oldIFS=$IFS + IFS=$'\n' lines=($output) + [ -z "$e" ] || set -e + [ -z "$E" ] || set -E + [ -z "$T" ] || set -T + IFS=$oldIFS +} + +setup() { + true +} + +teardown() { + true +} + +skip() { + BATS_TEST_SKIPPED=${1:-1} + BATS_TEST_COMPLETED=1 + exit 0 +} + +bats_test_begin() { + BATS_TEST_DESCRIPTION="$1" + if [ -n "$BATS_EXTENDED_SYNTAX" ]; then + echo "begin $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3 + fi + setup +} + +bats_test_function() { + local test_name="$1" + BATS_TEST_NAMES["${#BATS_TEST_NAMES[@]}"]="$test_name" +} + +bats_capture_stack_trace() { + BATS_PREVIOUS_STACK_TRACE=( "${BATS_CURRENT_STACK_TRACE[@]}" ) + BATS_CURRENT_STACK_TRACE=() + + local test_pattern=" $BATS_TEST_NAME $BATS_TEST_SOURCE" + local setup_pattern=" setup $BATS_TEST_SOURCE" + local teardown_pattern=" teardown $BATS_TEST_SOURCE" + + local frame + local index=1 + + while frame="$(caller "$index")"; do + BATS_CURRENT_STACK_TRACE["${#BATS_CURRENT_STACK_TRACE[@]}"]="$frame" + if [[ "$frame" = *"$test_pattern" || \ + "$frame" = *"$setup_pattern" || \ + "$frame" = *"$teardown_pattern" ]]; then + break + else + let index+=1 + fi + done + + BATS_SOURCE="$(bats_frame_filename "${BATS_CURRENT_STACK_TRACE[0]}")" + BATS_LINENO="$(bats_frame_lineno "${BATS_CURRENT_STACK_TRACE[0]}")" +} + +bats_print_stack_trace() { + local frame + local index=1 + local count="${#@}" + + for frame in "$@"; do + local filename="$(bats_trim_filename "$(bats_frame_filename "$frame")")" + local lineno="$(bats_frame_lineno "$frame")" + + if [ $index -eq 1 ]; then + echo -n "# (" + else + echo -n "# " + fi + + local fn="$(bats_frame_function "$frame")" + if [ "$fn" != "$BATS_TEST_NAME" ]; then + echo -n "from function \`$fn' " + fi + + if [ $index -eq $count ]; then + echo "in test file $filename, line $lineno)" + else + echo "in file $filename, line $lineno," + fi + + let index+=1 + done +} + +bats_print_failed_command() { + local frame="$1" + local status="$2" + local filename="$(bats_frame_filename "$frame")" + local lineno="$(bats_frame_lineno "$frame")" + + local failed_line="$(bats_extract_line "$filename" "$lineno")" + local failed_command="$(bats_strip_string "$failed_line")" + echo -n "# \`${failed_command}' " + + if [ $status -eq 1 ]; then + echo "failed" + else + echo "failed with status $status" + fi +} + +bats_frame_lineno() { + local frame="$1" + local lineno="${frame%% *}" + echo "$lineno" +} + +bats_frame_function() { + local frame="$1" + local rest="${frame#* }" + local fn="${rest%% *}" + echo "$fn" +} + +bats_frame_filename() { + local frame="$1" + local rest="${frame#* }" + local filename="${rest#* }" + + if [ "$filename" = "$BATS_TEST_SOURCE" ]; then + echo "$BATS_TEST_FILENAME" + else + echo "$filename" + fi +} + +bats_extract_line() { + local filename="$1" + local lineno="$2" + sed -n "${lineno}p" "$filename" +} + +bats_strip_string() { + local string="$1" + printf "%s" "$string" | sed -e "s/^[ "$'\t'"]*//" -e "s/[ "$'\t'"]*$//" +} + +bats_trim_filename() { + local filename="$1" + local length="${#BATS_CWD}" + + if [ "${filename:0:length+1}" = "${BATS_CWD}/" ]; then + echo "${filename:length+1}" + else + echo "$filename" + fi +} + +bats_debug_trap() { + if [ "$BASH_SOURCE" != "$1" ]; then + bats_capture_stack_trace + fi +} + +bats_error_trap() { + BATS_ERROR_STATUS="$?" + BATS_ERROR_STACK_TRACE=( "${BATS_PREVIOUS_STACK_TRACE[@]}" ) + trap - debug +} + +bats_teardown_trap() { + trap "bats_exit_trap" exit + local status=0 + teardown >>"$BATS_OUT" 2>&1 || status="$?" + + if [ $status -eq 0 ]; then + BATS_TEARDOWN_COMPLETED=1 + elif [ -n "$BATS_TEST_COMPLETED" ]; then + BATS_ERROR_STATUS="$status" + BATS_ERROR_STACK_TRACE=( "${BATS_CURRENT_STACK_TRACE[@]}" ) + fi + + bats_exit_trap +} + +bats_exit_trap() { + local status + local skipped + trap - err exit + + skipped="" + if [ -n "$BATS_TEST_SKIPPED" ]; then + skipped=" # skip" + if [ "1" != "$BATS_TEST_SKIPPED" ]; then + skipped+=" ($BATS_TEST_SKIPPED)" + fi + fi + + if [ -z "$BATS_TEST_COMPLETED" ] || [ -z "$BATS_TEARDOWN_COMPLETED" ]; then + echo "not ok $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3 + bats_print_stack_trace "${BATS_ERROR_STACK_TRACE[@]}" >&3 + bats_print_failed_command "${BATS_ERROR_STACK_TRACE[${#BATS_ERROR_STACK_TRACE[@]}-1]}" "$BATS_ERROR_STATUS" >&3 + sed -e "s/^/# /" < "$BATS_OUT" >&3 + status=1 + else + echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}" >&3 + status=0 + fi + + rm -f "$BATS_OUT" + exit "$status" +} + +bats_perform_tests() { + echo "1..$#" + test_number=1 + status=0 + for test_name in "$@"; do + "$0" $BATS_EXTENDED_SYNTAX "$BATS_TEST_FILENAME" "$test_name" "$test_number" || status=1 + let test_number+=1 + done + exit "$status" +} + +bats_perform_test() { + BATS_TEST_NAME="$1" + if [ "$(type -t "$BATS_TEST_NAME" || true)" = "function" ]; then + BATS_TEST_NUMBER="$2" + if [ -z "$BATS_TEST_NUMBER" ]; then + echo "1..1" + BATS_TEST_NUMBER="1" + fi + + BATS_TEST_COMPLETED="" + BATS_TEARDOWN_COMPLETED="" + trap "bats_debug_trap \"\$BASH_SOURCE\"" debug + trap "bats_error_trap" err + trap "bats_teardown_trap" exit + "$BATS_TEST_NAME" >>"$BATS_OUT" 2>&1 + BATS_TEST_COMPLETED=1 + + else + echo "bats: unknown test name \`$BATS_TEST_NAME'" >&2 + exit 1 + fi +} + +if [ -z "$TMPDIR" ]; then + BATS_TMPDIR="/tmp" +else + BATS_TMPDIR="${TMPDIR%/}" +fi + +BATS_TMPNAME="$BATS_TMPDIR/bats.$$" +BATS_PARENT_TMPNAME="$BATS_TMPDIR/bats.$PPID" +BATS_OUT="${BATS_TMPNAME}.out" + +bats_preprocess_source() { + BATS_TEST_SOURCE="${BATS_TMPNAME}.src" + { tr -d '\r' < "$BATS_TEST_FILENAME"; echo; } | bats-preprocess > "$BATS_TEST_SOURCE" + trap "bats_cleanup_preprocessed_source" err exit + trap "bats_cleanup_preprocessed_source; exit 1" int +} + +bats_cleanup_preprocessed_source() { + rm -f "$BATS_TEST_SOURCE" +} + +bats_evaluate_preprocessed_source() { + if [ -z "$BATS_TEST_SOURCE" ]; then + BATS_TEST_SOURCE="${BATS_PARENT_TMPNAME}.src" + fi + source "$BATS_TEST_SOURCE" +} + +exec 3<&1 + +if [ "$#" -eq 0 ]; then + bats_preprocess_source + bats_evaluate_preprocessed_source + + if [ -n "$BATS_COUNT_ONLY" ]; then + echo "${#BATS_TEST_NAMES[@]}" + else + bats_perform_tests "${BATS_TEST_NAMES[@]}" + fi +else + bats_evaluate_preprocessed_source + bats_perform_test "$@" +fi diff --git a/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-format-tap-stream b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-format-tap-stream new file mode 100755 index 00000000000..614768f4d9e --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-format-tap-stream @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +set -e + +# Just stream the TAP output (sans extended syntax) if tput is missing +command -v tput >/dev/null || exec grep -v "^begin " + +header_pattern='[0-9]+\.\.[0-9]+' +IFS= read -r header + +if [[ "$header" =~ $header_pattern ]]; then + count="${header:3}" + index=0 + failures=0 + skipped=0 + name="" + count_column_width=$(( ${#count} * 2 + 2 )) +else + # If the first line isn't a TAP plan, print it and pass the rest through + printf "%s\n" "$header" + exec cat +fi + +update_screen_width() { + screen_width="$(tput cols)" + count_column_left=$(( $screen_width - $count_column_width )) +} + +trap update_screen_width WINCH +update_screen_width + +begin() { + go_to_column 0 + printf_with_truncation $(( $count_column_left - 1 )) " %s" "$name" + clear_to_end_of_line + go_to_column $count_column_left + printf "%${#count}s/${count}" "$index" + go_to_column 1 +} + +pass() { + go_to_column 0 + printf " ✓ %s" "$name" + advance +} + +skip() { + local reason="$1" + [ -z "$reason" ] || reason=": $reason" + go_to_column 0 + printf " - %s (skipped%s)" "$name" "$reason" + advance +} + +fail() { + go_to_column 0 + set_color 1 bold + printf " ✗ %s" "$name" + advance +} + +log() { + set_color 1 + printf " %s\n" "$1" + clear_color +} + +summary() { + printf "\n%d test%s" "$count" "$(plural "$count")" + + printf ", %d failure%s" "$failures" "$(plural "$failures")" + + if [ "$skipped" -gt 0 ]; then + printf ", %d skipped" "$skipped" + fi + + printf "\n" +} + +printf_with_truncation() { + local width="$1" + shift + local string="$(printf "$@")" + + if [ "${#string}" -gt "$width" ]; then + printf "%s..." "${string:0:$(( $width - 4 ))}" + else + printf "%s" "$string" + fi +} + +go_to_column() { + local column="$1" + printf "\x1B[%dG" $(( $column + 1 )) +} + +clear_to_end_of_line() { + printf "\x1B[K" +} + +advance() { + clear_to_end_of_line + echo + clear_color +} + +set_color() { + local color="$1" + local weight="$2" + printf "\x1B[%d;%dm" $(( 30 + $color )) "$( [ "$weight" = "bold" ] && echo 1 || echo 22 )" +} + +clear_color() { + printf "\x1B[0m" +} + +plural() { + [ "$1" -eq 1 ] || echo "s" +} + +_buffer="" + +buffer() { + _buffer="${_buffer}$("$@")" +} + +flush() { + printf "%s" "$_buffer" + _buffer="" +} + +finish() { + flush + printf "\n" +} + +trap finish EXIT + +while IFS= read -r line; do + case "$line" in + "begin "* ) + let index+=1 + name="${line#* $index }" + buffer begin + flush + ;; + "ok "* ) + skip_expr="ok $index # skip (\(([^)]*)\))?" + if [[ "$line" =~ $skip_expr ]]; then + let skipped+=1 + buffer skip "${BASH_REMATCH[2]}" + else + buffer pass + fi + ;; + "not ok "* ) + let failures+=1 + buffer fail + ;; + "# "* ) + buffer log "${line:2}" + ;; + esac +done + +buffer summary diff --git a/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-preprocess b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-preprocess new file mode 100755 index 00000000000..04297ed019b --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/bats/libexec/bats-preprocess @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -e + +encode_name() { + local name="$1" + local result="test_" + + if [[ ! "$name" =~ [^[:alnum:]\ _-] ]]; then + name="${name//_/-5f}" + name="${name//-/-2d}" + name="${name// /_}" + result+="$name" + else + local length="${#name}" + local char i + + for ((i=0; i "$BATS_TMPDIR/test.txt" + repo_run git-secrets --scan "$BATS_TMPDIR/test.txt" + [ $status -eq 0 ] +} + +@test "Scans all files when no file provided" { + setup_bad_repo + repo_run git-secrets --scan + [ $status -eq 1 ] +} + +@test "Scans all files including history" { + setup_bad_repo + repo_run git-secrets --scan-history + [ $status -eq 1 ] +} + +@test "Scans all files when no file provided with secret in history" { + setup_bad_repo_history + repo_run git-secrets --scan + [ $status -eq 0 ] +} + +@test "Scans all files including history with secret in history" { + setup_bad_repo_history + repo_run git-secrets --scan-history + [ $status -eq 1 ] +} + +@test "Scans history with secrets distributed among branches in history" { + cd $TEST_REPO + echo '@todo' > $TEST_REPO/history_failure.txt + git add -A + git commit -m "Testing history" + echo 'todo' > $TEST_REPO/history_failure.txt + git add -A + git commit -m "Testing history" + git checkout -b testbranch + echo '@todo' > $TEST_REPO/history_failure.txt + git add -A + git commit -m "Testing history" + git checkout master + cd - + repo_run git-secrets --scan-history + [ $status -eq 1 ] +} + +@test "Scans recursively" { + setup_bad_repo + mkdir -p $TEST_REPO/foo/bar/baz + echo '@todo more stuff' > $TEST_REPO/foo/bar/baz/data.txt + repo_run git-secrets --scan -r $TEST_REPO/foo + [ $status -eq 1 ] +} + +@test "Scans recursively only if -r is given" { + setup_bad_repo + mkdir -p $TEST_REPO/foo/bar/baz + echo '@todo more stuff' > $TEST_REPO/foo/bar/baz/data.txt + repo_run git-secrets --scan $TEST_REPO/foo + [ $status -eq 0 ] +} + +@test "Excludes allowed patterns from failures" { + git config --add secrets.patterns 'foo="baz{1,5}"' + git config --add secrets.allowed 'foo="bazzz"' + echo 'foo="bazzz" is ok because 3 "z"s' > "$BATS_TMPDIR/test.txt" + repo_run git-secrets --scan "$BATS_TMPDIR/test.txt" + [ $status -eq 0 ] + echo 'This is NOT: ok foo="bazzzz"' > "$BATS_TMPDIR/test.txt" + repo_run git-secrets --scan "$BATS_TMPDIR/test.txt" + [ $status -eq 1 ] +} + +@test "Prohibited matches exits 1" { + file="$TEST_REPO/test.txt" + echo '@todo stuff' > $file + echo 'this is forbidden right?' >> $file + repo_run git-secrets --scan $file + [ $status -eq 1 ] + [ "${lines[0]}" == "$file:1:@todo stuff" ] + [ "${lines[1]}" == "$file:2:this is forbidden right?" ] +} + +@test "Only matches on word boundaries" { + file="$TEST_REPO/test.txt" + # Note that the following does not match as it is not a word. + echo 'mesa Jar Jar Binks' > $file + # The following do match because they are in word boundaries. + echo 'foo.me' >> $file + echo '"me"' >> $file + repo_run git-secrets --scan $file + [ $status -eq 1 ] + [ "${lines[0]}" == "$file:2:foo.me" ] + [ "${lines[1]}" == "$file:3:\"me\"" ] +} + +@test "Can scan from stdin using -" { + echo "foo" | "${BATS_TEST_DIRNAME}/../git-secrets" --scan - + echo "me" | "${BATS_TEST_DIRNAME}/../git-secrets" --scan - && exit 1 || true +} + +@test "installs hooks for repo" { + setup_bad_repo + repo_run git-secrets --install $TEST_REPO + [ -f $TEST_REPO/.git/hooks/pre-commit ] + [ -f $TEST_REPO/.git/hooks/prepare-commit-msg ] + [ -f $TEST_REPO/.git/hooks/commit-msg ] +} + +@test "fails if hook exists and no -f" { + repo_run git-secrets --install $TEST_REPO + repo_run git-secrets --install $TEST_REPO + [ $status -eq 1 ] +} + +@test "Overwrites hooks if -f is given" { + repo_run git-secrets --install $TEST_REPO + repo_run git-secrets --install -f $TEST_REPO + [ $status -eq 0 ] +} + +@test "installs hooks for repo with Debian style directories" { + setup_bad_repo + mkdir $TEST_REPO/.git/hooks/pre-commit.d + mkdir $TEST_REPO/.git/hooks/prepare-commit-msg.d + mkdir $TEST_REPO/.git/hooks/commit-msg.d + run git-secrets --install $TEST_REPO + [ -f $TEST_REPO/.git/hooks/pre-commit.d/git-secrets ] + [ -f $TEST_REPO/.git/hooks/prepare-commit-msg.d/git-secrets ] + [ -f $TEST_REPO/.git/hooks/commit-msg.d/git-secrets ] +} + +@test "installs hooks to template directory" { + setup_bad_repo + run git-secrets --install $TEMPLATE_DIR + [ $status -eq 0 ] + run git init --template $TEMPLATE_DIR + [ $status -eq 0 ] + [ -f "${TEST_REPO}/.git/hooks/pre-commit" ] + [ -f "${TEST_REPO}/.git/hooks/prepare-commit-msg" ] + [ -f "${TEST_REPO}/.git/hooks/commit-msg" ] +} + +@test "Scans using keys from credentials file" { + echo 'aws_access_key_id = abc123' > $BATS_TMPDIR/test.ini + echo 'aws_secret_access_key=foobaz' >> $BATS_TMPDIR/test.ini + echo 'aws_access_key_id = "Bernard"' >> $BATS_TMPDIR/test.ini + echo 'aws_secret_access_key= "Laverne"' >> $BATS_TMPDIR/test.ini + echo 'aws_access_key_id= Hoagie+man' >> $BATS_TMPDIR/test.ini + cd $TEST_REPO + run git secrets --aws-provider $BATS_TMPDIR/test.ini + [ $status -eq 0 ] + echo "$output" | grep -F "foobaz" + echo "$output" | grep -F "abc123" + echo "$output" | grep -F "Bernard" + echo "$output" | grep -F "Laverne" + echo "$output" | grep -F 'Hoagie\+man' + run git secrets --add-provider -- git secrets --aws-provider $BATS_TMPDIR/test.ini + [ $status -eq 0 ] + echo '(foobaz) test' > $TEST_REPO/bad_file + echo "abc123 test" >> $TEST_REPO/bad_file + echo 'Bernard test' >> $TEST_REPO/bad_file + echo 'Laverne test' >> $TEST_REPO/bad_file + echo 'Hoagie+man test' >> $TEST_REPO/bad_file + repo_run git-secrets --scan $TEST_REPO/bad_file + [ $status -eq 1 ] + echo "$output" | grep "foobaz" + echo "$output" | grep "abc123" + echo "$output" | grep "Bernard" + echo "$output" | grep "Laverne" + echo "$output" | grep -F 'Hoagie+man' +} + +@test "Lists secrets for a repo" { + repo_run git-secrets --list + [ $status -eq 0 ] + echo "$output" | grep -F 'secrets.patterns @todo' + echo "$output" | grep -F 'secrets.patterns forbidden|me' +} + +@test "Adds secrets to a repo and de-dedupes" { + repo_run git-secrets --add 'testing+123' + [ $status -eq 0 ] + repo_run git-secrets --add 'testing+123' + [ $status -eq 1 ] + repo_run git-secrets --add --literal 'testing+abc' + [ $status -eq 0 ] + repo_run git-secrets --add -l 'testing+abc' + [ $status -eq 1 ] + repo_run git-secrets --list + echo "$output" | grep -F 'secrets.patterns @todo' + echo "$output" | grep -F 'secrets.patterns forbidden|me' + echo "$output" | grep -F 'secrets.patterns testing+123' + echo "$output" | grep -F 'secrets.patterns testing\+abc' +} + +@test "Adds allowed patterns to a repo and de-dedupes" { + repo_run git-secrets --add -a 'testing+123' + [ $status -eq 0 ] + repo_run git-secrets --add --allowed 'testing+123' + [ $status -eq 1 ] + repo_run git-secrets --add -a -l 'testing+abc' + [ $status -eq 0 ] + repo_run git-secrets --add -a -l 'testing+abc' + [ $status -eq 1 ] + repo_run git-secrets --list + echo "$output" | grep -F 'secrets.patterns @todo' + echo "$output" | grep -F 'secrets.patterns forbidden|me' + echo "$output" | grep -F 'secrets.allowed testing+123' + echo "$output" | grep -F 'secrets.allowed testing\+abc' +} + +@test "Empty lines must be ignored in .gitallowed files" { + setup_bad_repo + echo '' >> $TEST_REPO/.gitallowed + repo_run git-secrets --scan + [ $status -eq 1 ] +} + +@test "Comment lines must be ignored in .gitallowed files" { + setup_bad_repo_with_hash + repo_run git-secrets --scan + [ $status -eq 1 ] + echo '#hash' > $TEST_REPO/.gitallowed + repo_run git-secrets --scan + [ $status -eq 1 ] + echo 'hash' > $TEST_REPO/.gitallowed + repo_run git-secrets --scan + [ $status -eq 0 ] +} + +@test "Scans all files and allowing none of the bad patterns in .gitallowed" { + setup_bad_repo + echo 'hello' > $TEST_REPO/.gitallowed + repo_run git-secrets --scan + [ $status -eq 1 ] +} + +@test "Scans all files and allowing all bad patterns in .gitallowed" { + setup_bad_repo + echo '@todo' > $TEST_REPO/.gitallowed + echo 'forbidden' >> $TEST_REPO/.gitallowed + echo 'me' >> $TEST_REPO/.gitallowed + repo_run git-secrets --scan + [ $status -eq 0 ] +} + +@test "Adds common AWS patterns" { + repo_run git config --unset-all secrets + repo_run git-secrets --register-aws + git config --local --get secrets.providers + repo_run git-secrets --list + echo "$output" | grep -F '(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}' + echo "$output" | grep "AKIAIOSFODNN7EXAMPLE" + echo "$output" | grep "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +} + +@test "Adds providers" { + repo_run git-secrets --add-provider -- echo foo baz bar + [ $status -eq 0 ] + repo_run git-secrets --add-provider -- echo bam + [ $status -eq 0 ] + repo_run git-secrets --list + echo "$output" | grep -F 'echo foo baz bar' + echo "$output" | grep -F 'echo bam' + echo 'foo baz bar' > $TEST_REPO/bad_file + echo 'bam' >> $TEST_REPO/bad_file + repo_run git-secrets --scan $TEST_REPO/bad_file + [ $status -eq 1 ] + echo "$output" | grep -F 'foo baz bar' + echo "$output" | grep -F 'bam' +} + +@test "Strips providers that return nothing" { + repo_run git-secrets --add-provider -- 'echo' + [ $status -eq 0 ] + repo_run git-secrets --add-provider -- 'echo 123' + [ $status -eq 0 ] + repo_run git-secrets --list + echo "$output" | grep -F 'echo 123' + echo 'foo' > $TEST_REPO/bad_file + repo_run git-secrets --scan $TEST_REPO/bad_file + [ $status -eq 0 ] +} + +@test "--recursive cannot be used with SCAN_*" { + repo_run git-secrets --scan -r --cached + [ $status -eq 1 ] + repo_run git-secrets --scan -r --no-index + [ $status -eq 1 ] + repo_run git-secrets --scan -r --untracked + [ $status -eq 1 ] +} + +@test "--recursive can be used with --scan" { + repo_run git-secrets --scan -r + [ $status -eq 0 ] +} + +@test "--recursive can't be used with --list" { + repo_run git-secrets --list -r + [ $status -eq 1 ] +} + +@test "-f can only be used with --install" { + repo_run git-secrets --scan -f + [ $status -eq 1 ] +} + +@test "-a can only be used with --add" { + repo_run git-secrets --scan -a + [ $status -eq 1 ] +} + +@test "-l can only be used with --add" { + repo_run git-secrets --scan -l + [ $status -eq 1 ] +} + +@test "--cached can only be used with --scan" { + repo_run git-secrets --list --cached + [ $status -eq 1 ] +} + +@test "--no-index can only be used with --scan" { + repo_run git-secrets --list --no-index + [ $status -eq 1 ] +} + +@test "--untracked can only be used with --scan" { + repo_run git-secrets --list --untracked + [ $status -eq 1 ] +} diff --git a/.github/actions/runleaks/git-secrets/test/pre-commit.bats b/.github/actions/runleaks/git-secrets/test/pre-commit.bats new file mode 100644 index 00000000000..5ace267cbb3 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/pre-commit.bats @@ -0,0 +1,62 @@ +#!/usr/bin/env bats +load test_helper + +@test "Rejects commits with prohibited patterns in changeset" { + setup_bad_repo + repo_run git-secrets --install $TEST_REPO + cd $TEST_REPO + run git commit -m 'Contents are bad not the message' + [ $status -eq 1 ] + [ "${lines[0]}" == "data.txt:1:@todo more stuff" ] + [ "${lines[1]}" == "failure1.txt:1:another line... forbidden" ] + [ "${lines[2]}" == "failure2.txt:1:me" ] +} + +@test "Rejects commits with prohibited patterns in changeset with filename that contain spaces" { + setup_bad_repo_with_spaces + repo_run git-secrets --install $TEST_REPO + cd $TEST_REPO + run git commit -m 'Contents are bad not the message' + [ $status -eq 1 ] + [ "${lines[0]}" == "da ta.txt:1:@todo more stuff" ] +} + +@test "Scans staged files" { + cd $TEST_REPO + repo_run git-secrets --install $TEST_REPO + echo '@todo more stuff' > $TEST_REPO/data.txt + echo 'hi there' > $TEST_REPO/ok.txt + git add -A + echo 'fixed the working directory, but not staged' > $TEST_REPO/data.txt + run git commit -m 'Contents are bad not the message' + [ $status -eq 1 ] + [ "${lines[0]}" == "data.txt:1:@todo more stuff" ] +} + +@test "Allows commits that do not match prohibited patterns" { + setup_good_repo + repo_run git-secrets --install $TEST_REPO + cd $TEST_REPO + run git commit -m 'This is fine' + [ $status -eq 0 ] + # Ensure deleted files are filtered out of the grep + rm $TEST_REPO/data.txt + echo 'aaa' > $TEST_REPO/data_2.txt + run git add -A + run git commit -m 'This is also fine' + [ $status -eq 0 ] +} + +@test "Rejects commits with prohibited patterns in changeset when AWS provider is enabled" { + setup_bad_repo + repo_run git-secrets --install $TEST_REPO + repo_run git-secrets --register-aws $TEST_REPO + cd $TEST_REPO + run git commit -m 'Contents are bad not the message' + [ $status -eq 1 ] + echo "${lines}" | grep -vq 'git secrets --aws-provider: command not found' + + [ "${lines[0]}" == "data.txt:1:@todo more stuff" ] + [ "${lines[1]}" == "failure1.txt:1:another line... forbidden" ] + [ "${lines[2]}" == "failure2.txt:1:me" ] +} diff --git a/.github/actions/runleaks/git-secrets/test/prepare-commit-msg.bats b/.github/actions/runleaks/git-secrets/test/prepare-commit-msg.bats new file mode 100644 index 00000000000..a211c1318a2 --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/prepare-commit-msg.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats +load test_helper + +@test "Rejects merges with prohibited patterns in history" { + setup_good_repo + repo_run git-secrets --install $TEST_REPO + cd $TEST_REPO + git commit -m 'OK' + git checkout -b feature + echo '@todo' > data.txt + git add -A + git commit -m 'Bad commit' --no-verify + echo 'Fixing!' > data.txt + git add -A + git commit -m 'Fixing commit' + git checkout master + run git merge --no-ff feature + [ $status -eq 1 ] +} + +@test "Allows merges that do not match prohibited patterns" { + setup_good_repo + cd $TEST_REPO + repo_run git-secrets --install + git commit -m 'OK' + git checkout -b feature + echo 'Not bad' > data.txt + git add -A + git commit -m 'Good commit' + git checkout master + run git merge --no-ff feature + [ $status -eq 0 ] +} diff --git a/.github/actions/runleaks/git-secrets/test/test_helper.bash b/.github/actions/runleaks/git-secrets/test/test_helper.bash new file mode 100644 index 00000000000..9133e5162ec --- /dev/null +++ b/.github/actions/runleaks/git-secrets/test/test_helper.bash @@ -0,0 +1,94 @@ +#!/bin/bash +export TEST_REPO="$BATS_TMPDIR/test-repo" +export TEMP_HOME="$BATS_TMPDIR/home" +export TEMPLATE_DIR="${BATS_TMPDIR}/template" +INITIAL_PATH="${PATH}" +INITIAL_HOME=${HOME} + +setup() { + setup_repo + [ -d "${TEMPLATE_DIR}" ] && rm -rf "${TEMPLATE_DIR}" + [ -d "${TEMP_HOME}" ] && rm -rf "${TEMP_HOME}" + mkdir -p $TEMP_HOME + export HOME=$TEMP_HOME + export PATH="${BATS_TEST_DIRNAME}/..:${INITIAL_PATH}" + cd $TEST_REPO +} + +teardown() { + delete_repo + export PATH="${INITIAL_PATH}" + export HOME="${INITIAL_HOME}" + [ -d "${TEMP_HOME}" ] && rm -rf "${TEMP_HOME}" +} + +delete_repo() { + [ -d $TEST_REPO ] && rm -rf $TEST_REPO || true +} + +setup_repo() { + delete_repo + mkdir -p $TEST_REPO + cd $TEST_REPO + git init + git config --local --add secrets.patterns '@todo' + git config --local --add secrets.patterns 'forbidden|me' + git config --local --add secrets.patterns '#hash' + git config --local user.email "you@example.com" + git config --local user.name "Your Name" + cd - +} + +repo_run() { + cmd="$1" + shift + cd "${TEST_REPO}" + run "${BATS_TEST_DIRNAME}/../${cmd}" $@ + cd - +} + +# Creates a repo that should fail +setup_bad_repo() { + cd $TEST_REPO + echo '@todo more stuff' > $TEST_REPO/data.txt + echo 'hi there' > $TEST_REPO/ok.txt + echo 'another line... forbidden' > $TEST_REPO/failure1.txt + echo 'me' > $TEST_REPO/failure2.txt + git add -A + cd - +} + +# Creates a repo that should fail +setup_bad_repo_with_spaces() { + cd $TEST_REPO + echo '@todo more stuff' > "$TEST_REPO/da ta.txt" + git add -A + cd - +} + +# Creates a repo that should fail +setup_bad_repo_with_hash() { + cd $TEST_REPO + echo '#hash' > "$TEST_REPO/data.txt" + git add -A + cd - +} + +# Creates a repo that should fail +setup_bad_repo_history() { + cd $TEST_REPO + echo '@todo' > $TEST_REPO/history_failure.txt + git add -A + git commit -m "Testing history" + echo 'todo' > $TEST_REPO/history_failure.txt + git add -A + cd - +} + +# Creates a repo that does not fail +setup_good_repo() { + cd $TEST_REPO + echo 'hello!' > $TEST_REPO/data.txt + git add -A + cd - +} diff --git a/.github/actions/runleaks/lib/scan.sh b/.github/actions/runleaks/lib/scan.sh new file mode 100644 index 00000000000..4a4b9cadb61 --- /dev/null +++ b/.github/actions/runleaks/lib/scan.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# Init +repo="$2" +run_count=$3 +min_days_old=$4 +max_days_old=$5 +fail_on_leak=$8 +max_proc_count=32 +log_file=exceptions_search.txt + +# Set defaults +if [[ -z $min_days_old ]]; then + min_days_old="0" + echo "min_days_old: $min_days_old" +fi + +if [[ -z $max_days_old ]]; then + max_days_old="3" + echo "max_days_old: $max_days_old" +fi + +if [[ -z $fail_on_leak ]]; then + fail_on_leak=true + echo "fail_on_leak: $fail_on_leak" +fi + +# If file path error, use default patterns +if cp "$6" /patterns.txt; then + # Register patterns + grep -v -E '(#.*$)|(^$)' /patterns.txt >/clean_patterns.txt + while read line; do + if [[ "$line" != "#*" ]]; then + git secrets $line --global + fi + done auth.txt +gh auth login --with-token ((now - ('"$max_days_old"'*86400))|strftime("%Y-%m-%dT%H:%M:%S %Z"))) | select(.status =="completed").databaseId' --limit $run_list_limit) +run_ids_limited=$(printf "%s\n" ${run_ids[@]} | head -$run_count) + +echo "Runs to scan: $run_count" +echo $run_ids_limited + +touch "$log_file" + +run_for_each() { + local each=$@ + + # Collect run logs and remove null + log_out=$(gh run view $each --repo "$repo" --log 2>/dev/null | sed 's/\x0//g') + if [[ $? -ne 0 ]]; then + exit 1 + fi + + # Identify potential exceptions + scan_out=$(echo "$log_out" | git secrets --scan - 2>&1) + status=$? + + # If exception, add to array of details + if (($status != 0)); then + raw_log_full=$(echo "$scan_out" | grep '(standard input)') + exception_line=$(echo "$raw_log_full" | awk -F '\t' '{print $2$3}' | sed 's/[^a-zA-Z0-9]/_/g' | sed 's/.*/"&",/') + exception=$(gh run view $each --repo "$repo" --json name,createdAt,databaseId,url,updatedAt,headBranch 2>/dev/null) + if [[ $? -ne 0 ]]; then + exit 1 + fi + exception_with_detail=$(echo $exception | jq '. + {exception_detail: {'"$exception_line"'}}') + echo $exception_with_detail >>"$log_file" + fi +} + +# Make visible to subprocesses +export -f run_for_each +export log_file +export repo + +parallel -0 --jobs $max_proc_count run_for_each ::: $run_ids_limited + +json_out=$(jq -n '.exceptions |= [inputs]' "$log_file") +json_out_length=$(echo $json_out | jq -s '.[].exceptions' | jq length) +echo "$json_out" + +# Make output friendly +json_out="${json_out//'%'/'%25'}" +json_out="${json_out//$'\n'/'%0A'}" +json_out="${json_out//$'\r'/'%0D'}" +echo "exceptions=$json_out" >> $GITHUB_OUTPUT +echo "count=$json_out_length" >> $GITHUB_OUTPUT + +rm "$log_file" + +if [[ $fail_on_leak = true && json_out_length -gt 0 ]]; then + echo "Failing since leak!" + exit 1 +fi diff --git a/prime-router/src/main/resources/metadata/hl7_mapping/receivers/STLTs/WA/WA-receiver-transform.yml b/prime-router/src/main/resources/metadata/hl7_mapping/receivers/STLTs/WA/WA-receiver-transform.yml index c4fc5487234..139c65d91e1 100644 --- a/prime-router/src/main/resources/metadata/hl7_mapping/receivers/STLTs/WA/WA-receiver-transform.yml +++ b/prime-router/src/main/resources/metadata/hl7_mapping/receivers/STLTs/WA/WA-receiver-transform.yml @@ -163,4 +163,5 @@ elements: value: [ 'Bundle.entry.resource.ofType(Organization).telecom.value[x].replaceMatches("[^0-9]", "")' ] hl7Spec: [ '%{ORC}-23-12', '%{ORC}-23-1' ] - + - name: observation-result-with-aoe + resource: "Bundle.entry.resource.ofType(Observation).where(code.coding.extension('https://reportstream.cdc.gov/fhir/StructureDefinition/condition-code').value.where(code in ('AOE')).exists().not())" \ No newline at end of file diff --git a/prime-router/src/testIntegration/resources/datatests/FHIR_to_HL7/sample_WA_20240719-0001.hl7 b/prime-router/src/testIntegration/resources/datatests/FHIR_to_HL7/sample_WA_20240719-0001.hl7 index 5c68836448b..4efac6915de 100644 --- a/prime-router/src/testIntegration/resources/datatests/FHIR_to_HL7/sample_WA_20240719-0001.hl7 +++ b/prime-router/src/testIntegration/resources/datatests/FHIR_to_HL7/sample_WA_20240719-0001.hl7 @@ -4,9 +4,4 @@ PID|1||10083d1d-dc8b-4ea0-91fa-8744cf0f013b^^^Testing Lab&12D4567890&CLIA^PI^Tes ORC|RE|12256b9a-0ca9-47ed-95b4-0d4ca60c4324^Testing Lab^12D4567890^CLIA|12256b9a-0ca9-47ed-95b4-0d4ca60c4324^Testing Lab^12D4567890^CLIA|||||||||1245319599^McTester^Phil^^^^^^NPI&2.16.840.1.113883.4.6&ISO^L^^^NPI||^WPN^PH^^1^530^8675309|20240815054718||||||Testing Lab^L^^^^CLIA&2.16.840.1.113883.4.7&ISO^XX^^^12D4567890|123 Beach Way^^Denver^CO^80210^USA|^WPN^PH^^1^530^8675309|321 Ocean Drive^^Denver^CO^80210^USA OBR|1|12256b9a-0ca9-47ed-95b4-0d4ca60c4324^Testing Lab^12D4567890^CLIA|12256b9a-0ca9-47ed-95b4-0d4ca60c4324^Testing Lab^12D4567890^CLIA|94558-4^SARS-CoV-2 (COVID-19) Ag [Presence] in Respiratory system specimen by Rapid immunoassay^LN|||20240815053218|||||||||1245319599^McTester^Phil^^^^^^NPI&2.16.840.1.113883.4.6&ISO^L^^^NPI|^WPN^PH^^1^530^8675309|||||20240815054718|||F OBX|1|CWE|94558-4^SARS-CoV-2 (COVID-19) Ag [Presence] in Respiratory system specimen by Rapid immunoassay^LN^^^^^^COVID-19||260373001^Detected^SCT|||A^Abnormal^HL70078^^^^2.7|||F|||20240815053218|12D4567890^Testing Lab^CLIA||BinaxNOW COVID-19 Ag 2 Card_Abbott Diagnostics Scarborough, Inc.^BinaxNOW COVID-19 Ag 2 Card^^^^^^^BinaxNOW COVID-19 Ag 2 Card_Abbott Diagnostics Scarborough, Inc.|No Equipment|20240815054718||||Testing Lab^L^^^^CLIA&2.16.840.1.113883.4.7&ISO^XX^^^12D4567890|123 Beach Way^^Denver^CO^80210^USA -OBX|2|CWE|95419-8^Has symptoms related to condition of interest^LN^^^^2.69^^Has symptoms related to condition of interest||N^No^HL70136||||||F|||20240815053218|12D4567890^Testing Lab^CLIA||||||||Testing Lab^L^^^^CLIA&2.16.840.1.113883.4.7&ISO^XX^^^12D4567890|123 Beach Way^^Denver^CO^80210^USA -OBX|3|CWE|82810-3^Pregnancy status^LN^^^^2.68^^Pregnancy status||60001007^Not pregnant^SCT||||||F|||20240815053218|12D4567890^Testing Lab^CLIA||||||||Testing Lab^L^^^^CLIA&2.16.840.1.113883.4.7&ISO^XX^^^12D4567890|123 Beach Way^^Denver^CO^80210^USA -OBX|4|CWE|95418-0^Employed in a healthcare setting^LN^^^^2.69^^Employed in a healthcare setting||Y^Yes^HL70136||||||F|||20240815053218|12D4567890^Testing Lab^CLIA||||||||Testing Lab^L^^^^CLIA&2.16.840.1.113883.4.7&ISO^XX^^^12D4567890|123 Beach Way^^Denver^CO^80210^USA -OBX|5|CWE|95421-4^Resides in a congregate care setting^LN^^^^2.69^^Resides in a congregate care setting||Y^Yes^HL70136||||||F|||20240815053218|12D4567890^Testing Lab^CLIA||||||||Testing Lab^L^^^^CLIA&2.16.840.1.113883.4.7&ISO^XX^^^12D4567890|123 Beach Way^^Denver^CO^80210^USA -OBX|6|CWE|76691-5^Gender identity^LN^^^^^^Gender identity||UNK^Non-binary gender identity^NULLFL||||||F|||20240815053218|12D4567890^Testing Lab^CLIA||||||||Testing Lab^L^^^^CLIA&2.16.840.1.113883.4.7&ISO^XX^^^12D4567890|123 Beach Way^^Denver^CO^80210^USA SPM|1|12256b9a-0ca9-47ed-95b4-0d4ca60c4324&Testing Lab&12D4567890&CLIA^12256b9a-0ca9-47ed-95b4-0d4ca60c4324&Testing Lab&12D4567890&CLIA||433801000124107^Nasopharyngeal and oropharyngeal swab^SCT^^^^2.67^^Nasopharyngeal and oropharyngeal swab||||87100004^Topography unknown (body structure)^SCT^^^^^^Topography unknown (body structure)|||||||||20240815053218|20240815053218 \ No newline at end of file