From 5a7ca780fd5c1eccdaaca58ee740674cfcd638c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20Gr=C3=BCbel?= Date: Sun, 26 Nov 2023 16:16:44 +0100 Subject: [PATCH] drop Python 3.7 and update dependencies (#168) --- .flake8 | 3 - .github/workflows/pr.yml | 16 ++-- .github/workflows/release.yml | 24 +++-- .pre-commit-config.yaml | 19 ++-- README.md | 3 +- detect_secrets/__version__.py | 2 +- detect_secrets/audit/analytics.py | 2 +- detect_secrets/audit/common.py | 4 +- detect_secrets/audit/compare.py | 6 +- detect_secrets/audit/io.py | 4 +- detect_secrets/audit/report.py | 4 +- detect_secrets/constants.py | 2 +- detect_secrets/core/baseline.py | 4 +- detect_secrets/core/plugins/__init__.py | 1 - detect_secrets/core/plugins/initialize.py | 25 +++--- detect_secrets/core/plugins/util.py | 23 ++--- detect_secrets/core/potential_secret.py | 2 +- detect_secrets/core/scan.py | 23 +++-- detect_secrets/core/secrets_collection.py | 25 +++--- detect_secrets/core/usage/__init__.py | 3 +- detect_secrets/core/usage/common.py | 2 +- detect_secrets/core/usage/plugins.py | 6 +- detect_secrets/exceptions.py | 2 +- detect_secrets/filters/common.py | 9 +- detect_secrets/filters/gibberish/__init__.py | 19 ++-- detect_secrets/filters/heuristic.py | 92 +++++++++----------- detect_secrets/filters/regex.py | 15 +--- detect_secrets/filters/util.py | 2 +- detect_secrets/filters/wordlist.py | 4 +- detect_secrets/main.py | 4 +- detect_secrets/plugins/aws.py | 9 +- detect_secrets/plugins/base.py | 6 +- detect_secrets/plugins/cloudant.py | 2 +- detect_secrets/plugins/ibm_cloud_iam.py | 8 +- detect_secrets/plugins/ibm_cos_hmac.py | 16 ++-- detect_secrets/plugins/jwt.py | 4 +- detect_secrets/plugins/keyword.py | 6 +- detect_secrets/plugins/private_key.py | 8 +- detect_secrets/plugins/softlayer.py | 2 +- detect_secrets/settings.py | 9 +- detect_secrets/transformers/__init__.py | 6 +- detect_secrets/transformers/yaml.py | 7 +- detect_secrets/util/code_snippet.py | 11 +-- detect_secrets/util/git.py | 6 +- detect_secrets/util/importlib.py | 6 +- detect_secrets/util/inject.py | 5 +- mypy.ini | 16 +--- pyproject.toml | 54 ++++++++++++ requirements-dev.txt | 19 ++-- setup.py | 3 +- testing/factories.py | 2 +- testing/plugins.py | 11 ++- tox.ini | 8 +- 53 files changed, 293 insertions(+), 281 deletions(-) delete mode 100644 .flake8 create mode 100644 pyproject.toml diff --git a/.flake8 b/.flake8 deleted file mode 100644 index f3839033..00000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length = 100 -extend-exclude = venv/*, test_data/*, bumpity.py diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index abf3c4b5..150a261a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -2,7 +2,11 @@ name: pr on: pull_request -permissions: read-all +permissions: + contents: read + +env: + MIN_PYTHON_VERSION: "3.8" jobs: lint: @@ -16,7 +20,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 with: - python-version: 3.7 + python-version: ${{ env.MIN_PYTHON_VERSION }} - name: Install dependencies run: | pip install --upgrade pip @@ -30,7 +34,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v3 @@ -41,11 +45,5 @@ jobs: run: | pip install --upgrade pip pip install -r requirements-dev.txt - - name: Sets env var for release - if: ${{ matrix.os == 'macos-latest' && matrix.python == '3.7' }} - run: | - # NO_PROXY is needed to call requests API within a forked process - # when using macOS and python version 3.7 - echo "NO_PROXY='*'" >> "$GITHUB_ENV" - name: Run tests run: pytest --strict-markers -W ignore::UserWarning tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f6f369b..386400d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,14 +13,18 @@ on: - '.pre-commit-config.yaml' - 'requirements-dev.txt' -permissions: read-all +permissions: + contents: read + +env: + MIN_PYTHON_VERSION: "3.8" jobs: tests: runs-on: ubuntu-latest strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v3 @@ -31,12 +35,6 @@ jobs: run: | pip install --upgrade pip pip install -r requirements-dev.txt - - name: Sets env var for release - if: ${{ matrix.os == 'macos-latest' && matrix.python == '3.7' }} - run: | - # NO_PROXY is needed to call requests API within a forked process - # when using macOS and python version 3.7 - echo "NO_PROXY='*'" >> "$GITHUB_ENV" - name: Run tests run: pytest --strict-markers -W ignore::UserWarning tests @@ -55,10 +53,10 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 with: token: ${{ secrets.PAT_TOKEN }} - - name: Set up Python 3.7 + - name: Set up Python uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v3 with: - python-version: 3.7 + python-version: ${{ env.MIN_PYTHON_VERSION }} - name: bump version id: version env: @@ -95,8 +93,6 @@ jobs: needs: bump-version runs-on: [self-hosted, public, linux, x64] environment: release - env: - PYTHON_VERSION: "3.7" steps: - name: Checkout checkov uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 @@ -105,14 +101,14 @@ jobs: repository: bridgecrewio/checkov - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: ${{ env.MIN_PYTHON_VERSION }} - name: Prepare PR run: | # install needed tools python -m pip install --no-cache-dir --upgrade pipenv "pipenv-setup[black]" "vistir<0.7.0" # update Pipfile - pipenv --python ${{ env.PYTHON_VERSION }} + pipenv --python ${{ env.MIN_PYTHON_VERSION }} pipenv install bc-detect-secrets==${{ needs.bump-version.outputs.version }} pipenv lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d108c3cc..8e558567 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-builtin-literals args: ['--no-allow-dict-kwargs'] @@ -12,27 +12,22 @@ repos: - id: end-of-file-fixer - id: name-tests-test - id: trailing-whitespace -- repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.1.6 hooks: - - id: flake8 - language_version: python3.8 + - id: ruff - repo: https://github.com/asottile/reorder_python_imports - rev: v3.9.0 + rev: v3.12.0 hooks: - id: reorder-python-imports language_version: python3 exclude: bumpity.py$ - repo: https://github.com/asottile/add-trailing-comma - rev: v2.4.0 + rev: v3.1.0 hooks: - id: add-trailing-comma -- repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.1 - hooks: - - id: autopep8 - repo: https://github.com/rhysd/actionlint - rev: v1.6.23 + rev: v1.6.26 hooks: - id: actionlint-docker # SC2129 - Consider using { cmd1; cmd2; } >> file instead of individual redirects. diff --git a/README.md b/README.md index e7b39c43..96f8d263 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Build Status](https://github.com/bridgecrewio/detect-secrets/actions/workflows/release.yml/badge.svg)](https://github.com/bridgecrewio/detect-secrets/actions/workflows/release.yml?query=branch%3Amaster++) -[![PyPI version](https://badge.fury.io/py/bc-detect-secrets.svg)](https://badge.fury.io/py/bc-detect-secrets) +[![PyPI](https://img.shields.io/pypi/v/bc-detect-secrets)](https://pypi.org/project/bc-detect-secrets/) +[![Python Version](https://img.shields.io/pypi/pyversions/bc-detect-secrets)](#) # detect-secrets diff --git a/detect_secrets/__version__.py b/detect_secrets/__version__.py index 15a6d6ef..84262d36 100644 --- a/detect_secrets/__version__.py +++ b/detect_secrets/__version__.py @@ -1 +1 @@ -VERSION = '1.4.31' +VERSION = '1.5.0' diff --git a/detect_secrets/audit/analytics.py b/detect_secrets/audit/analytics.py index 29facdbb..0cb7ab3a 100644 --- a/detect_secrets/audit/analytics.py +++ b/detect_secrets/audit/analytics.py @@ -14,7 +14,7 @@ def calculate_statistics_for_baseline( filename: str, - **kwargs: Any, + **kwargs: Any, # noqa: ARG001 ) -> 'StatisticsAggregator': """ :raises: InvalidBaselineError diff --git a/detect_secrets/audit/common.py b/detect_secrets/audit/common.py index de5b506d..849b3891 100644 --- a/detect_secrets/audit/common.py +++ b/detect_secrets/audit/common.py @@ -28,7 +28,7 @@ def get_baseline_from_file(filename: str) -> SecretsCollection: try: # TODO: Should we upgrade this? return baseline.load(baseline.load_from_file(filename), filename) - except (IOError, json.decoder.JSONDecodeError): + except (OSError, json.decoder.JSONDecodeError): io.print_error('Not a valid baseline file!') raise InvalidBaselineError except KeyError: @@ -157,7 +157,7 @@ def lines(self) -> List[str]: with self.open_file() as f: lines = get_transformed_file(f, use_eager_transformers=self.use_eager_transformers) - self._lines = self.raw_lines if not lines else lines + self._lines = lines if lines else self.raw_lines return self._lines diff --git a/detect_secrets/audit/compare.py b/detect_secrets/audit/compare.py index 4f62081f..869946e7 100644 --- a/detect_secrets/audit/compare.py +++ b/detect_secrets/audit/compare.py @@ -77,10 +77,10 @@ def _compare_baselines( if `left_secret` is None, then it's a newly added secret; if `right_secret` is None, then it's a deleted secret """ - class LeftSecret(Exception): + class LeftSecret(Exception): # noqa: N818 pass - class RightSecret(Exception): + class RightSecret(Exception): # noqa: N818 pass left_secrets = [secret for _, secret in old_baseline] @@ -173,7 +173,7 @@ def _display_difference_to_user( new_baseline, new_config = new_data iterator = BidirectionalIterator(list(_compare_baselines(old_baseline, new_baseline))) - for filename, left_secret, right_secret in iterator: + for _, left_secret, right_secret in iterator: io.clear_screen() secret = left_secret if left_secret else right_secret diff --git a/detect_secrets/audit/io.py b/detect_secrets/audit/io.py index e5fcd4cf..86e06956 100644 --- a/detect_secrets/audit/io.py +++ b/detect_secrets/audit/io.py @@ -23,7 +23,7 @@ def clear_screen() -> None: # pragma: no cover command = 'clear' if platform.system() == 'Windows': command = 'cls' - os.system(command) + os.system(command) # noqa: S605 def print_context(context: SecretContext) -> None: @@ -96,7 +96,7 @@ def get_user_decision( user_input = None while user_input not in prompter.valid_input: if user_input: - print('Invalid input.') # type: ignore # Statement unreachable? Come on mypy... + print('Invalid input.') user_input = input(str(prompter)) if user_input: diff --git a/detect_secrets/audit/report.py b/detect_secrets/audit/report.py index 2e177135..6a432acb 100644 --- a/detect_secrets/audit/report.py +++ b/detect_secrets/audit/report.py @@ -19,7 +19,7 @@ class SecretClassToPrint(Enum): FALSE_POSITIVE = 2 @staticmethod - def from_class(secret_class: VerifiedResult) -> 'SecretClassToPrint': + def from_class(secret_class: VerifiedResult) -> SecretClassToPrint: if secret_class in [VerifiedResult.UNVERIFIED, VerifiedResult.VERIFIED_TRUE]: return SecretClassToPrint.REAL_SECRET else: @@ -29,7 +29,7 @@ def from_class(secret_class: VerifiedResult) -> 'SecretClassToPrint': def generate_report( baseline_file: str, class_to_print: SecretClassToPrint | None = None, - line_getter_factory: Callable[[str], 'LineGetter'] = open_file, + line_getter_factory: Callable[[str], LineGetter] = open_file, ) -> Dict[str, List[Dict[str, Any]]]: secrets: Dict[Tuple[str, str], Any] = {} diff --git a/detect_secrets/constants.py b/detect_secrets/constants.py index b71493c9..d3a93c56 100644 --- a/detect_secrets/constants.py +++ b/detect_secrets/constants.py @@ -13,7 +13,7 @@ class VerifiedResult(Enum): VERIFIED_TRUE = 3 @staticmethod - def from_secret(secret: PotentialSecret) -> 'VerifiedResult': + def from_secret(secret: PotentialSecret) -> VerifiedResult: if secret.is_secret is None: return VerifiedResult.UNVERIFIED elif secret.is_secret: diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index 16c9f45f..bea2913b 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -60,7 +60,7 @@ def load_from_file(filename: str) -> Dict[str, Any]: try: with open(filename) as f: return cast(Dict[str, Any], json.loads(f.read())) - except (FileNotFoundError, IOError, json.decoder.JSONDecodeError) as e: + except (FileNotFoundError, OSError, json.decoder.JSONDecodeError) as e: raise UnableToReadBaselineError from e @@ -79,7 +79,7 @@ def format_for_output(secrets: SecretsCollection, is_slim_mode: bool = False) -> else: # NOTE: This has a nice little side effect of keeping it ordered by line number, # even though we don't output it. - for filename, secret_list in cast( + for _, secret_list in cast( Dict[str, List[Dict[str, Any]]], output['results'], ).items(): diff --git a/detect_secrets/core/plugins/__init__.py b/detect_secrets/core/plugins/__init__.py index d8c6f588..d0a5f90d 100644 --- a/detect_secrets/core/plugins/__init__.py +++ b/detect_secrets/core/plugins/__init__.py @@ -1,2 +1 @@ from . import initialize # noqa: F401 -from .util import Plugin # noqa: F401 diff --git a/detect_secrets/core/plugins/initialize.py b/detect_secrets/core/plugins/initialize.py index 4de3858c..d9ce3b55 100644 --- a/detect_secrets/core/plugins/initialize.py +++ b/detect_secrets/core/plugins/initialize.py @@ -1,15 +1,19 @@ +from __future__ import annotations + from typing import Any +from typing import cast from typing import Dict from typing import Iterable from typing import List -from typing import Type +from typing import TYPE_CHECKING -from ...plugins.base import BasePlugin from ...settings import get_settings from ..log import log from .util import get_mapping_from_secret_type_to_class from .util import get_plugins_from_file -from .util import Plugin + +if TYPE_CHECKING: + from detect_secrets.plugins.base import BasePlugin def from_secret_type(secret_type: str) -> BasePlugin: @@ -17,7 +21,7 @@ def from_secret_type(secret_type: str) -> BasePlugin: :raises: TypeError """ try: - plugin_type = get_mapping_from_secret_type_to_class()[secret_type] + plugin_type: type[BasePlugin] = get_mapping_from_secret_type_to_class()[secret_type] except KeyError: raise TypeError @@ -51,19 +55,20 @@ def from_plugin_classname(classname: str) -> BasePlugin: raise -def from_file(filename: str) -> Iterable[Type[Plugin]]: +def from_file(filename: str) -> Iterable[type[BasePlugin]]: """ :raises: FileNotFoundError :raises: InvalidFile """ - output: List[Type[Plugin]] = [] - plugin_class: Type[Plugin] + output: List[type[BasePlugin]] = [] + plugin_class: type[BasePlugin] + secret_type_classes = get_mapping_from_secret_type_to_class() for plugin_class in get_plugins_from_file(filename): - secret_type = plugin_class.secret_type # type: ignore - if secret_type in get_mapping_from_secret_type_to_class(): + secret_type = cast('str', plugin_class.secret_type) + if secret_type in secret_type_classes: log.info(f'Duplicate plugin detected: {plugin_class.__name__}. Skipping...') - get_mapping_from_secret_type_to_class()[secret_type] = plugin_class + secret_type_classes[secret_type] = plugin_class output.append(plugin_class) return output diff --git a/detect_secrets/core/plugins/util.py b/detect_secrets/core/plugins/util.py index e2acadfc..695c552c 100644 --- a/detect_secrets/core/plugins/util.py +++ b/detect_secrets/core/plugins/util.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import inspect from functools import lru_cache from types import ModuleType @@ -5,8 +7,6 @@ from typing import cast from typing import Dict from typing import Generator -from typing import Type -from typing import TypeVar from ... import plugins from ...plugins.base import BasePlugin @@ -16,11 +16,8 @@ from ...util.importlib import import_types_from_package -Plugin = TypeVar('Plugin', bound=BasePlugin) - - @lru_cache(maxsize=1) -def get_mapping_from_secret_type_to_class() -> Dict[str, Type[Plugin]]: +def get_mapping_from_secret_type_to_class() -> Dict[str, type[BasePlugin]]: output = {} for plugin_class in import_types_from_package( plugins, @@ -33,27 +30,25 @@ def get_mapping_from_secret_type_to_class() -> Dict[str, Type[Plugin]]: # someone to cause this to break (e.g. arbitrary imports from unexpected places). # However, this falls under the same security assumptions as listed in # `import_file_as_module`. - for classname, config in get_settings().plugins.items(): + for config in get_settings().plugins.values(): if 'path' not in config: continue # Only supporting file schema right now. filename = config['path'][len('file://'):] for plugin_class in get_plugins_from_file(filename): - output[cast(BasePlugin, plugin_class).secret_type] = plugin_class + output[cast('BasePlugin', plugin_class).secret_type] = plugin_class return output -def get_plugins_from_file(filename: str) -> Generator[Type[Plugin], None, None]: - plugin_class: Type[Plugin] - for plugin_class in get_plugins_from_module(import_file_as_module(filename)): - yield plugin_class +def get_plugins_from_file(filename: str) -> Generator[type[BasePlugin], None, None]: + yield from get_plugins_from_module(import_file_as_module(filename)) -def get_plugins_from_module(module: ModuleType) -> Generator[Type[Plugin], None, None]: +def get_plugins_from_module(module: ModuleType) -> Generator[type[BasePlugin], None, None]: for plugin_class in import_types_from_module(module, filter=lambda x: not _is_valid_plugin(x)): - yield cast(Type[Plugin], plugin_class) + yield cast('type[BasePlugin]', plugin_class) def _is_valid_plugin(attribute: Any) -> bool: diff --git a/detect_secrets/core/potential_secret.py b/detect_secrets/core/potential_secret.py index deb48944..0257911f 100644 --- a/detect_secrets/core/potential_secret.py +++ b/detect_secrets/core/potential_secret.py @@ -76,7 +76,7 @@ def set_secret(self, secret: str) -> None: @staticmethod def hash_secret(secret: str) -> str: """This offers a way to coherently test this class, without mocking self.secret_hash.""" - return hashlib.sha1(secret.encode('utf-8')).hexdigest() + return hashlib.sha1(secret.encode('utf-8')).hexdigest() # noqa: S324 @classmethod def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'PotentialSecret': diff --git a/detect_secrets/core/scan.py b/detect_secrets/core/scan.py index 3d451fb9..371079ec 100644 --- a/detect_secrets/core/scan.py +++ b/detect_secrets/core/scan.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import subprocess from functools import lru_cache @@ -8,6 +10,7 @@ from typing import List from typing import Set from typing import Tuple +from typing import TYPE_CHECKING from typing import Union from ..custom_types import NamedIO @@ -23,11 +26,13 @@ from ..util.inject import call_function_with_arguments from ..util.path import get_relative_path from .log import log -from .plugins import Plugin from .potential_secret import PotentialSecret from detect_secrets.util.filetype import determine_file_type from detect_secrets.util.filetype import FileType +if TYPE_CHECKING: + from detect_secrets.plugins.base import BasePlugin + MIN_LINE_LENGTH = int(os.getenv('CHECKOV_MIN_LINE_LENGTH', '5')) @@ -36,7 +41,7 @@ def read_raw_lines(file_name: str) -> List[str]: try: with open(file_name) as f: return f.readlines() - except IOError: + except OSError: log.debug(f"Can't open file {file_name}") return [] @@ -176,7 +181,7 @@ def scan_file(filename: str) -> Generator[PotentialSecret, None, None]: if has_secret: break - except IOError: + except OSError: log.warning(f'Unable to open file: {filename}') return @@ -222,7 +227,7 @@ def scan_for_allowlisted_secrets_in_file(filename: str) -> Generator[PotentialSe ] yield from _scan_for_allowlisted_secrets_in_lines(lines_list, filename) break - except IOError: + except OSError: log.warning(f'Unable to open file: {filename}') return @@ -306,7 +311,7 @@ def _get_lines_from_diff(diff: str) -> \ """ # Local imports, so that we don't need to require unidiff for versions of # detect-secrets that don't use it. - from unidiff import PatchSet # type:ignore[import] + from unidiff import PatchSet # type:ignore[import-untyped] patch_set = PatchSet.from_string(diff) for patch_file in patch_set: @@ -323,7 +328,7 @@ def _get_lines_from_diff(diff: str) -> \ ) for chunk in patch_file # target_lines refers to incoming (new) changes - for line in [line for line in chunk] + for line in list(chunk) if line.is_added or line.is_removed ], ) @@ -394,8 +399,8 @@ def _process_line_based_plugins( # different lines as 1. # Calculate actual line number in case of YAML multi-line string actual_line_number = line_number - for i, l in enumerate(raw_code_snippet_lines[actual_line_number - 1:]): - if secret.secret_value in l: + for i, cline in enumerate(raw_code_snippet_lines[actual_line_number - 1:]): + if secret.secret_value in cline: actual_line_number += i break secret.line_number = actual_line_number @@ -403,7 +408,7 @@ def _process_line_based_plugins( def _scan_line( - plugin: Plugin, + plugin: BasePlugin, filename: str, line: str, line_number: int, diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index cd477f32..e4962e4c 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -146,10 +146,7 @@ def trim( if not os.path.exists(filename) ] - if not filelist: - fileset = set() - else: - fileset = set(filelist) + fileset = set(filelist) if filelist else set() # Unfortunately, we can't merely do a set intersection since we want to update the line # numbers (if applicable). Therefore, this does it manually. @@ -255,21 +252,21 @@ def __eq__(self, other: Any, strict: bool = False) -> bool: if not strict: continue - for secretA in self_mapping.values(): - secretB = other_mapping[(secretA.secret_hash, secretA.type)] + for secret_a in self_mapping.values(): + secret_b = other_mapping[(secret_a.secret_hash, secret_a.type)] - valuesA = vars(secretA) - valuesA.pop('secret_value') - valuesB = vars(secretB) - valuesB.pop('secret_value') + values_a = vars(secret_a) + values_a.pop('secret_value') + values_b = vars(secret_b) + values_b.pop('secret_value') - if valuesA['line_number'] == 0 or valuesB['line_number'] == 0: + if values_a['line_number'] == 0 or values_b['line_number'] == 0: # If line numbers are not provided (for either one), then don't compare # line numbers. - valuesA.pop('line_number') - valuesB.pop('line_number') + values_a.pop('line_number') + values_b.pop('line_number') - if valuesA != valuesB: + if values_a != values_b: return False return True diff --git a/detect_secrets/core/usage/__init__.py b/detect_secrets/core/usage/__init__.py index 623de5b2..c7a13bc8 100644 --- a/detect_secrets/core/usage/__init__.py +++ b/detect_secrets/core/usage/__init__.py @@ -158,10 +158,9 @@ def parse_args(self, argv: Optional[List[str]] = None) -> argparse.Namespace: try: for processor in self._post_processors: processor(args) - except argparse.ArgumentTypeError as e: + except argparse.ArgumentTypeError: # TODO: Better help text? self._parser.print_usage(sys.stderr) - print(f'error: {str(e)}', file=sys.stderr) sys.exit(1) args.custom_root = args.custom_root[0] diff --git a/detect_secrets/core/usage/common.py b/detect_secrets/core/usage/common.py index f469f3d1..7cd5bf53 100644 --- a/detect_secrets/core/usage/common.py +++ b/detect_secrets/core/usage/common.py @@ -14,7 +14,7 @@ def valid_path(path: str) -> str: return path -def initialize_plugin_settings(args: argparse.Namespace) -> None: +def initialize_plugin_settings(args: argparse.Namespace) -> None: # noqa: ARG001 """ This is a stand-in function, which should be replaced if baseline options are used. This ensures that our global settings object is initialized to a minimal state diff --git a/detect_secrets/core/usage/plugins.py b/detect_secrets/core/usage/plugins.py index 3db4c483..c6fb0cc7 100644 --- a/detect_secrets/core/usage/plugins.py +++ b/detect_secrets/core/usage/plugins.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import os from typing import cast @@ -87,7 +89,7 @@ def minmax_type(string: str) -> float: def _add_disable_flag(parser: argparse._ArgumentGroup) -> None: def valid_plugin_name(string: str) -> str: - valid_plugin_names = { + valid_plugin_names: set[str] = { item.__name__ for item in get_mapping_from_secret_type_to_class().values() } @@ -109,7 +111,7 @@ def valid_plugin_name(string: str) -> str: def parse_args(args: argparse.Namespace) -> None: if args.disable_plugin: # Flatten entry for easier parsing. - args.disable_plugin = set([entry for item in args.disable_plugin for entry in item]) + args.disable_plugin = {entry for item in args.disable_plugin for entry in item} get_settings().disable_plugins(*args.disable_plugin) # By the time the code reaches here, the baseline logic will have populated an initial diff --git a/detect_secrets/exceptions.py b/detect_secrets/exceptions.py index 8731e587..ef2e0837 100644 --- a/detect_secrets/exceptions.py +++ b/detect_secrets/exceptions.py @@ -8,7 +8,7 @@ class InvalidBaselineError(ValueError): pass -class InvalidFile(ValueError): +class InvalidFile(ValueError): # noqa: N818 """Think of this as a 400, if FileNotFoundError was a 404 HTTPError code.""" pass diff --git a/detect_secrets/filters/common.py b/detect_secrets/filters/common.py index 18481af7..d47f3a44 100644 --- a/detect_secrets/filters/common.py +++ b/detect_secrets/filters/common.py @@ -1,16 +1,21 @@ +from __future__ import annotations + import os from functools import lru_cache from typing import cast +from typing import TYPE_CHECKING import requests from ..constants import VerifiedResult -from ..core.plugins import Plugin from ..settings import get_settings from ..util.code_snippet import CodeSnippet from ..util.inject import call_function_with_arguments from .util import get_caller_path +if TYPE_CHECKING: + from detect_secrets.plugins.base import BasePlugin + def is_invalid_file(filename: str) -> bool: return not os.path.isfile(filename) @@ -28,7 +33,7 @@ def _get_baseline_filename() -> str: def is_ignored_due_to_verification_policies( secret: str, - plugin: Plugin, + plugin: BasePlugin, context: CodeSnippet, ) -> bool: """ diff --git a/detect_secrets/filters/gibberish/__init__.py b/detect_secrets/filters/gibberish/__init__.py index d6812ff9..bd287cb1 100644 --- a/detect_secrets/filters/gibberish/__init__.py +++ b/detect_secrets/filters/gibberish/__init__.py @@ -1,16 +1,21 @@ +from __future__ import annotations + import os import string from functools import lru_cache from typing import Any from typing import Dict from typing import Optional +from typing import TYPE_CHECKING from typing import Union -from ...core.plugins import Plugin from ...plugins.private_key import PrivateKeyDetector from ...settings import get_settings from ..util import compute_file_hash +if TYPE_CHECKING: + from detect_secrets.plugins.base import BasePlugin + Model = Any @@ -36,8 +41,8 @@ def initialize(model_path: Optional[str] = None, limit: float = 3.7) -> None: model = get_model() - from gibberish_detector import serializer # type:ignore[import] - from gibberish_detector.exceptions import ParsingError # type:ignore[import] + from gibberish_detector import serializer # type:ignore[import-untyped] + from gibberish_detector.exceptions import ParsingError # type:ignore[import-untyped] with open(path) as f: try: model.update(serializer.deserialize(f.read())) @@ -55,7 +60,7 @@ def initialize(model_path: Optional[str] = None, limit: float = 3.7) -> None: get_settings().filters[path] = config -def should_exclude_secret(secret: str, plugin: Optional[Plugin] = None) -> bool: +def should_exclude_secret(secret: str, plugin: BasePlugin | None = None) -> bool: """ :param plugin: optional, for easier testing. The dependency injection system will populate its proper value on complete runs. @@ -74,7 +79,7 @@ def should_exclude_secret(secret: str, plugin: Optional[Plugin] = None) -> bool: if not get_model().data or not get_model().charset: raise AssertionError('Attempting to use uninitialized gibberish model.') - from gibberish_detector.detector import Detector # type:ignore[import] + from gibberish_detector.detector import Detector # type:ignore[import-untyped] detector = Detector( model=get_model(), threshold=get_settings().filters[f'{__name__}.should_exclude_secret']['limit'], @@ -90,6 +95,6 @@ def should_exclude_secret(secret: str, plugin: Optional[Plugin] = None) -> bool: @lru_cache(maxsize=1) -def get_model() -> 'Model': - from gibberish_detector.model import Model # type:ignore[import] +def get_model() -> Model: + from gibberish_detector.model import Model # type:ignore[import-untyped] return Model(charset='') diff --git a/detect_secrets/filters/heuristic.py b/detect_secrets/filters/heuristic.py index 5ae9f1b6..8f65edf4 100644 --- a/detect_secrets/filters/heuristic.py +++ b/detect_secrets/filters/heuristic.py @@ -42,11 +42,7 @@ def is_sequential_string(secret: str) -> bool: ) uppercase = secret.upper() - for sequential_string in sequences: - if uppercase in sequential_string: - return True - - return False + return any(uppercase in sequential_string for sequential_string in sequences) def is_potential_uuid(secret: str) -> bool: @@ -98,50 +94,48 @@ def is_non_text_file(filename: str) -> bool: # and look for "ASCII text", but that might be more expensive. # # Definitely something to look into, if this list gets unruly long. -IGNORED_FILE_EXTENSIONS = set( - ( - '.7z', - '.bin', - '.bmp', - '.bz2', - '.class', - '.css', - '.dmg', - '.doc', - '.eot', - '.exe', - '.gif', - '.gz', - '.ico', - '.iml', - '.ipr', - '.iws', - '.jar', - '.jpg', - '.jpeg', - '.lock', - '.map', - '.mo', - '.pdf', - '.png', - '.prefs', - '.psd', - '.rar', - '.realm', - '.s7z', - '.sum', - '.svg', - '.tar', - '.tif', - '.tiff', - '.ttf', - '.webp', - '.woff', - '.xls', - '.xlsx', - '.zip', - ), -) +IGNORED_FILE_EXTENSIONS = { + '.7z', + '.bin', + '.bmp', + '.bz2', + '.class', + '.css', + '.dmg', + '.doc', + '.eot', + '.exe', + '.gif', + '.gz', + '.ico', + '.iml', + '.ipr', + '.iws', + '.jar', + '.jpg', + '.jpeg', + '.lock', + '.map', + '.mo', + '.pdf', + '.png', + '.prefs', + '.psd', + '.rar', + '.realm', + '.s7z', + '.sum', + '.svg', + '.tar', + '.tif', + '.tiff', + '.ttf', + '.webp', + '.woff', + '.xls', + '.xlsx', + '.zip', +} def is_templated_secret(secret: str) -> bool: diff --git a/detect_secrets/filters/regex.py b/detect_secrets/filters/regex.py index e0fd11a2..c71ce8b4 100644 --- a/detect_secrets/filters/regex.py +++ b/detect_secrets/filters/regex.py @@ -9,10 +9,7 @@ def should_exclude_line(line: str) -> bool: regexes = _get_line_exclusion_regex() - for regex in regexes: - if regex.search(line): - return True - return False + return any(regex.search(line) for regex in regexes) @lru_cache(maxsize=1) @@ -23,10 +20,7 @@ def _get_line_exclusion_regex() -> List[Pattern]: def should_exclude_file(filename: str) -> bool: regexes = _get_file_exclusion_regex() - for regex in regexes: - if regex.search(filename): - return True - return False + return any(regex.search(filename) for regex in regexes) @lru_cache(maxsize=1) @@ -37,10 +31,7 @@ def _get_file_exclusion_regex() -> List[Pattern]: def should_exclude_secret(secret: str) -> bool: regexes = _get_secret_exclusion_regex() - for regex in regexes: - if regex.search(secret): - return True - return False + return any(regex.search(secret) for regex in regexes) @lru_cache(maxsize=1) diff --git a/detect_secrets/filters/util.py b/detect_secrets/filters/util.py index 54455c81..4166dfc0 100644 --- a/detect_secrets/filters/util.py +++ b/detect_secrets/filters/util.py @@ -40,7 +40,7 @@ def compute_file_hash(filename: str, buffer_size: int = 64 * 1024) -> str: This is akin to: $ sha1sum """ - sha1 = hashlib.sha1() + sha1 = hashlib.sha1() # noqa: S324 with open(filename, 'rb') as f: data = f.read(buffer_size) while data: diff --git a/detect_secrets/filters/wordlist.py b/detect_secrets/filters/wordlist.py index 9a47131a..0a879e39 100644 --- a/detect_secrets/filters/wordlist.py +++ b/detect_secrets/filters/wordlist.py @@ -22,7 +22,7 @@ def is_feature_enabled() -> bool: return False -def initialize(wordlist_filename: str, min_length: int = 3, file_hash: str = '') -> Automaton: +def initialize(wordlist_filename: str, min_length: int = 3, file_hash: str = '') -> Automaton: #noqa: ARG001 """ :param min_length: if words are too small, the automaton will flag too many words. As a result, our recall will decrease without a precision boost. @@ -68,5 +68,5 @@ def should_exclude_secret(secret: str) -> bool: @lru_cache(maxsize=1) def get_automaton() -> Automaton: - import ahocorasick # type:ignore[import] + import ahocorasick # type:ignore[import-not-found] return ahocorasick.Automaton() diff --git a/detect_secrets/main.py b/detect_secrets/main.py index 6f18311a..d9fd318e 100644 --- a/detect_secrets/main.py +++ b/detect_secrets/main.py @@ -146,9 +146,9 @@ def handle_audit_action(args: argparse.Namespace) -> None: # Starts interactive session. if args.diff: # Show changes - audit.compare_baselines(args.filename[0], args.filename[1]) + audit.compare.compare_baselines(args.filename[0], args.filename[1]) else: # Label secrets - audit.audit_baseline(args.filename[0]) + audit.audit.audit_baseline(args.filename[0]) except InvalidBaselineError: pass diff --git a/detect_secrets/plugins/aws.py b/detect_secrets/plugins/aws.py index ee822b6f..2d3e1675 100644 --- a/detect_secrets/plugins/aws.py +++ b/detect_secrets/plugins/aws.py @@ -39,7 +39,7 @@ class AWSKeyDetector(RegexBasedDetector): ), ) - def verify( # type: ignore[override] # noqa: F821 + def verify( # type: ignore[override] self, secret: str, context: CodeSnippet, @@ -99,12 +99,7 @@ def verify_aws_secret_access_key(key: str, secret: str) -> bool: # pragma: no c } # Step #1: Canonical Request - signed_headers = ';'.join( - map( - lambda x: x.lower(), - headers.keys(), - ), - ) + signed_headers = ';'.join(header.lower() for header in headers) canonical_request = textwrap.dedent(""" POST / diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index c04239f8..06be6e68 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -52,7 +52,7 @@ def analyze_line( line_number: int = 0, context: CodeSnippet | None = None, raw_context: CodeSnippet | None = None, - **kwargs: Any + **kwargs: Any, ) -> Set[PotentialSecret]: """This examines a line and finds all possible secret values in it.""" output = set() @@ -71,7 +71,7 @@ def analyze_line( context=context, raw_context=raw_context, ) - is_verified = True if verified_result == VerifiedResult.VERIFIED_TRUE else False + is_verified = bool(verified_result == VerifiedResult.VERIFIED_TRUE) except requests.exceptions.RequestException: is_verified = False @@ -173,7 +173,7 @@ def analyze_string(self, string: str) -> Generator[str, None, None]: for regex in self.denylist: for match in regex.findall(string): if isinstance(match, tuple): - for submatch in filter(bool, match): + for submatch in filter(bool, match): # noqa: UP028 # It might make sense to paste break after yielding yield submatch else: diff --git a/detect_secrets/plugins/cloudant.py b/detect_secrets/plugins/cloudant.py index c645b942..9760f1fc 100644 --- a/detect_secrets/plugins/cloudant.py +++ b/detect_secrets/plugins/cloudant.py @@ -62,7 +62,7 @@ class CloudantDetector(RegexBasedDetector): ), ] - def verify( # type: ignore[override] # noqa: F821 + def verify( # type: ignore[override] self, secret: str, context: CodeSnippet, diff --git a/detect_secrets/plugins/ibm_cloud_iam.py b/detect_secrets/plugins/ibm_cloud_iam.py index e2848445..401a84f7 100644 --- a/detect_secrets/plugins/ibm_cloud_iam.py +++ b/detect_secrets/plugins/ibm_cloud_iam.py @@ -49,22 +49,22 @@ def analyze_line( line_number: int = 0, context: CodeSnippet | None = None, raw_context: CodeSnippet | None = None, - **kwargs: Any + **kwargs: Any, ) -> Set[PotentialSecret]: potentials = super().analyze_line( - filename, line, line_number, context, raw_context, **kwargs + filename, line, line_number, context, raw_context, **kwargs, ) secrets = set() for p in potentials: if self.entropy_plugin.analyze_line( - filename, f'"{p.secret_value}"', line_number, context, raw_context, **kwargs + filename, f'"{p.secret_value}"', line_number, context, raw_context, **kwargs, ): secrets.add(p) return secrets def verify_cloud_iam_api_key(apikey: Union[str, bytes]) -> requests.Response: # pragma: no cover - if type(apikey) == bytes: + if isinstance(apikey, bytes): apikey = apikey.decode('UTF-8') headers = { diff --git a/detect_secrets/plugins/ibm_cos_hmac.py b/detect_secrets/plugins/ibm_cos_hmac.py index a377323d..833d18be 100644 --- a/detect_secrets/plugins/ibm_cos_hmac.py +++ b/detect_secrets/plugins/ibm_cos_hmac.py @@ -32,7 +32,7 @@ class IbmCosHmacDetector(RegexBasedDetector): ), ) - def verify( # type: ignore[override] # noqa: F821 + def verify( # type: ignore[override] self, secret: str, context: CodeSnippet, @@ -74,12 +74,12 @@ def hash(key: bytes, msg: str) -> bytes: return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest() -def createSignatureKey(key: str, datestamp: str, region: str, service: str) -> bytes: - keyDate = hash(('AWS4' + key).encode('utf-8'), datestamp) - keyString = hash(keyDate, region) - keyService = hash(keyString, service) - keySigning = hash(keyService, 'aws4_request') - return keySigning +def createSignatureKey(key: str, datestamp: str, region: str, service: str) -> bytes: # noqa: N802 + key_date = hash(('AWS4' + key).encode('utf-8'), datestamp) + key_string = hash(key_date, region) + key_service = hash(key_string, service) + key_signing = hash(key_service, 'aws4_request') + return key_signing def verify_ibm_cos_hmac_credentials( @@ -118,7 +118,7 @@ def query_ibm_cos_hmac( standardized_querystring = request_parameters standardized_headers = 'host:' + host + '\n' + 'x-amz-date:' + timestamp + '\n' signed_headers = 'host;x-amz-date' - payload_hash = hashlib.sha256(''.encode('utf-8')).hexdigest() + payload_hash = hashlib.sha256(b'').hexdigest() standardized_request = ( http_method + '\n' diff --git a/detect_secrets/plugins/jwt.py b/detect_secrets/plugins/jwt.py index 83763033..f091206f 100644 --- a/detect_secrets/plugins/jwt.py +++ b/detect_secrets/plugins/jwt.py @@ -34,9 +34,9 @@ def is_formally_valid(token: str) -> bool: if m == 1: raise TypeError('Incorrect padding') elif m == 2: - part += '=='.encode('utf-8') + part += b'==' elif m == 3: - part += '==='.encode('utf-8') + part += b'===' b64_decoded = base64.urlsafe_b64decode(part) if idx < 2: _ = json.loads(b64_decoded.decode('utf-8')) diff --git a/detect_secrets/plugins/keyword.py b/detect_secrets/plugins/keyword.py index f2a178f3..0ca38eed 100644 --- a/detect_secrets/plugins/keyword.py +++ b/detect_secrets/plugins/keyword.py @@ -138,7 +138,7 @@ # e.g. my_password = @"bar" # e.g. my_password[] = "bar"; # e.g. char my_password[25] = "bar"; - r'{denylist}({square_brackets})?{optional_whitespace}[!=]{{1,2}}{optional_whitespace}(@)?(")({secret})(\5)'.format( # noqa: E501 + r'{denylist}({square_brackets})?{optional_whitespace}[!=]{{1,2}}{optional_whitespace}(@)?(")({secret})(\5)'.format( denylist=DENYLIST_REGEX, square_brackets=SQUARE_BRACKETS, optional_whitespace=OPTIONAL_WHITESPACE, @@ -160,7 +160,7 @@ # or my_password !== "bar" # e.g. my_password == 'bar' or my_password != 'bar' or my_password === 'bar' # or my_password !== 'bar' - r'{denylist}({closing})?{whitespace}(={{1,3}}|!==?){whitespace}({quote}?)({secret})(\4)'.format( # noqa: E501 + r'{denylist}({closing})?{whitespace}(={{1,3}}|!==?){whitespace}({quote}?)({secret})(\4)'.format( denylist=DENYLIST_REGEX, closing=CLOSING, quote=QUOTE, @@ -175,7 +175,7 @@ # or my_password !== "bar" # e.g. my_password == 'bar' or my_password != 'bar' or my_password === 'bar' # or my_password !== 'bar' - r'{denylist}({closing})?{whitespace}(={{1,3}}|!==?){whitespace}({quote})({secret})(\4)'.format( # noqa: E501 + r'{denylist}({closing})?{whitespace}(={{1,3}}|!==?){whitespace}({quote})({secret})(\4)'.format( denylist=DENYLIST_REGEX, closing=CLOSING, quote=QUOTE, diff --git a/detect_secrets/plugins/private_key.py b/detect_secrets/plugins/private_key.py index 80329567..2d265dcf 100644 --- a/detect_secrets/plugins/private_key.py +++ b/detect_secrets/plugins/private_key.py @@ -75,14 +75,14 @@ def analyze_line( line_number: int = 0, context: Optional[CodeSnippet] = None, raw_context: Optional[CodeSnippet] = None, - **kwargs: Any + **kwargs: Any, ) -> Set[PotentialSecret]: output: Set[PotentialSecret] = set() output.update( super().analyze_line( filename=filename, line=line, line_number=line_number, - context=context, raw_context=raw_context, **kwargs + context=context, raw_context=raw_context, **kwargs, ), ) @@ -94,7 +94,7 @@ def analyze_line( output.update( super().analyze_line( filename=filename, line=file_content, line_number=1, - context=context, raw_context=raw_context, **kwargs + context=context, raw_context=raw_context, **kwargs, ), ) return output @@ -112,7 +112,7 @@ def analyze_string(self, string: str) -> Generator[str, None, None]: def read_file(self, file_path: str) -> str: try: - with open(file_path, 'r') as f: + with open(file_path) as f: file_content = f.read() return file_content except Exception: diff --git a/detect_secrets/plugins/softlayer.py b/detect_secrets/plugins/softlayer.py index 575b0e77..afb3502a 100644 --- a/detect_secrets/plugins/softlayer.py +++ b/detect_secrets/plugins/softlayer.py @@ -30,7 +30,7 @@ class SoftlayerDetector(RegexBasedDetector): ), ] - def verify( # type: ignore[override] # noqa: F821 + def verify( # type: ignore[override] self, secret: str, context: CodeSnippet, diff --git a/detect_secrets/settings.py b/detect_secrets/settings.py index e4bd9d8b..0d89e6f3 100644 --- a/detect_secrets/settings.py +++ b/detect_secrets/settings.py @@ -1,3 +1,4 @@ +import contextlib from contextlib import contextmanager from copy import deepcopy from functools import lru_cache @@ -91,7 +92,7 @@ def cache_bust() -> None: get_plugins.cache_clear() get_filters.cache_clear() - for path, config in get_settings().filters.items(): + for path in get_settings().filters: # Need to also clear the individual caches (e.g. cached regex patterns). parts = urlparse(path) if not parts.scheme: @@ -176,10 +177,8 @@ def configure_plugins(self, config: List[Dict[str, Any]]) -> 'Settings': def disable_plugins(self, *plugin_names: str) -> 'Settings': for name in plugin_names: - try: + with contextlib.suppress(KeyError): self.plugins.pop(name) - except KeyError: - pass get_plugins.cache_clear() return self @@ -275,7 +274,7 @@ def get_filters() -> List: from .util.inject import get_injectable_variables output = [] - for path, config in get_settings().filters.items(): + for path in get_settings().filters: parts = urlparse(path) if not parts.scheme: module_path, function_name = path.rsplit('.', 1) diff --git a/detect_secrets/transformers/__init__.py b/detect_secrets/transformers/__init__.py index c8849f10..e77174e0 100644 --- a/detect_secrets/transformers/__init__.py +++ b/detect_secrets/transformers/__init__.py @@ -5,7 +5,6 @@ from typing import Iterable from typing import List from typing import Optional -from typing import TypeVar from ..custom_types import NamedIO from ..util.importlib import import_types_from_package @@ -13,9 +12,6 @@ from .exceptions import ParsingError -Transformer = TypeVar('Transformer', bound=BaseTransformer) - - def get_transformed_file( file: NamedIO, use_eager_transformers: bool = False, @@ -38,7 +34,7 @@ def get_transformed_file( @lru_cache(maxsize=1) -def get_transformers() -> Iterable[Transformer]: +def get_transformers() -> Iterable[BaseTransformer]: return [ item() for item in import_types_from_package( diff --git a/detect_secrets/transformers/yaml.py b/detect_secrets/transformers/yaml.py index 574953a3..a060860b 100644 --- a/detect_secrets/transformers/yaml.py +++ b/detect_secrets/transformers/yaml.py @@ -247,7 +247,7 @@ def _parse_flow_mapping_key_shim( # KeyToken ('key:') is_inline_dictionary = ( first - and self.loader.marks[-1].line == self.loader.peek_token().start_mark.line + and self.loader.marks[-1].line == self.loader.peek_token().start_mark.line # type:ignore[no-untyped-call] or self._check_next_tokens_shim(FlowEntryToken, KeyToken) ) @@ -256,7 +256,10 @@ def _parse_flow_mapping_key_shim( else: self.is_inline_flow_mapping_key = False - return cast(yaml.nodes.Node, yaml.parser.Parser.parse_flow_mapping_key(self.loader, first)) + return cast( + 'yaml.nodes.Node', + yaml.parser.Parser.parse_flow_mapping_key(self.loader, first), # type:ignore[no-untyped-call] + ) def _check_next_tokens_shim( self, diff --git a/detect_secrets/util/code_snippet.py b/detect_secrets/util/code_snippet.py index aa865da5..ad11375e 100644 --- a/detect_secrets/util/code_snippet.py +++ b/detect_secrets/util/code_snippet.py @@ -65,10 +65,7 @@ def previous_line(self) -> str: def add_line_numbers(self) -> 'CodeSnippet': for index, line in enumerate(self.lines): - self.lines[index] = u'{}:{}'.format( - self.get_line_number(self.start_line + index + 1), - line, - ) + self.lines[index] = f'{self.get_line_number(self.start_line + index + 1)}:{line}' return self @@ -80,15 +77,15 @@ def highlight_line(self, payload: str) -> 'CodeSnippet': index_of_payload = self.target_line.lower().index(payload.lower()) end_of_payload = index_of_payload + len(payload) - self.target_line = u'{}{}{}'.format( + self.target_line = '{}{}{}'.format( self.target_line[:index_of_payload], self.apply_highlight(self.target_line[index_of_payload:end_of_payload]), self.target_line[end_of_payload:], ) return self - except ValueError: - raise SecretNotFoundOnSpecifiedLineError(self.target_index) + except ValueError as err: + raise SecretNotFoundOnSpecifiedLineError(self.target_index) from err def get_line_number(self, line_number: int) -> str: """Broken out, for custom colorization.""" diff --git a/detect_secrets/util/git.py b/detect_secrets/util/git.py index ff491153..7ef64918 100644 --- a/detect_secrets/util/git.py +++ b/detect_secrets/util/git.py @@ -15,7 +15,7 @@ def get_root_directory(path: str = '') -> str: command.extend(['-C', path]) command.extend(['rev-parse', '--show-toplevel']) - return subprocess.check_output(command).decode('utf-8').strip() + return subprocess.check_output(command).decode('utf-8').strip() # noqa: S603 def get_tracked_files(root: str) -> Set[str]: @@ -31,7 +31,7 @@ def get_tracked_files(root: str) -> Set[str]: output = set() try: files = subprocess.check_output( - ['git', '-C', root, 'ls-files'], + ['git', '-C', root, 'ls-files'], # noqa: S603,S607 stderr=subprocess.DEVNULL, ) @@ -50,7 +50,7 @@ def get_tracked_files(root: str) -> Set[str]: def get_changed_but_unstaged_files() -> Set[str]: try: - files = subprocess.check_output('git diff --name-only'.split()).decode().splitlines() + files = subprocess.check_output('git diff --name-only'.split()).decode().splitlines() # noqa: S603 except subprocess.CalledProcessError: # pragma: no cover # Since we don't pipe stderr, we get free logging through git. raise ValueError diff --git a/detect_secrets/util/importlib.py b/detect_secrets/util/importlib.py index f5e90005..81c10fdf 100644 --- a/detect_secrets/util/importlib.py +++ b/detect_secrets/util/importlib.py @@ -94,12 +94,12 @@ def import_file_as_module(filename: str, name: Optional[str] = None) -> ModuleTy # Source: https://stackoverflow.com/a/67692/13340678 spec = importlib.util.spec_from_file_location(name, filename) - if not spec: + if not spec or not spec.loader: raise InvalidFile module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore - module.__path__ = os.path.abspath(filename) # type: ignore + spec.loader.exec_module(module) + module.__path__ = os.path.abspath(filename) # type:ignore[assignment] return module diff --git a/detect_secrets/util/inject.py b/detect_secrets/util/inject.py index ae9a8e4e..3e16b708 100644 --- a/detect_secrets/util/inject.py +++ b/detect_secrets/util/inject.py @@ -16,10 +16,7 @@ def call_function_with_arguments( :raises: TypeError """ # First, we ensure that the function we're going to inject values into is self-aware. - if not isinstance(func, SelfAwareCallable): - function = make_function_self_aware(func) - else: - function = func + function = func if isinstance(func, SelfAwareCallable) else make_function_self_aware(func) # If `function` is derived from a method, we add the instance of the class by default. # However, if `function` is a method itself, it will already carry the reference of the diff --git a/mypy.ini b/mypy.ini index fe025cf2..ce0a70dd 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,17 +1,5 @@ [mypy] files = detect_secrets,testing -show_error_codes = True +strict = True -warn_return_any = True -warn_unused_configs = True - -# Functions should be typed -disallow_untyped_defs = True -disallow_incomplete_defs = True -disallow_untyped_decorators = True - -warn_redundant_casts = True -warn_unreachable = True -warn_unused_ignores = True - -show_column_numbers = True +disallow_any_generics = False diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..16d1e333 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[tool.ruff] +line-length = 100 + +select = [ + "A", + "ARG", + "B", + "C4", + "E", + "F", + "I", + "N", + "PGH", + "RUF", + "S", + "SIM", + "T10", + "T20", + "UP", + "W", + "YTT", +] +exclude = [ + "bumpity.py", + "scripts", + "test_data", + "test_diff", + "tests", + "testing", + "venv", +] +ignore = [ + "A001", + "A002", + "A003", + "ARG002", + "B904", # should be handled properly at some point + "I001", + "PGH003", # should be handled properly at some point + "RUF012", + "S113", # should be handled properly at some point + "UP006", + "UP007", + "UP032", # should be handled properly at some point +] + +target-version = "py38" + +[tool.ruff.per-file-ignores] +"detect_secrets/plugins/*" = ["S105"] + +"detect_secrets/audit/io.py" = ["T201"] +"detect_secrets/main.py" = ["T201"] +"detect_secrets/pre_commit_hook.py" = ["T201"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 7df13d22..230b7637 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,16 +1,15 @@ -coverage==7.2.7 -flake8==5.0.4 # 6.0.0+ needs Python 3.8 +coverage==7.3.2 gibberish-detector==0.1.1 monotonic==1.6 -mypy==1.4.1 -pre-commit==2.21.0 +mypy==1.7.1 +pre-commit==3.5.0 pyahocorasick==2.0.0 -pytest==7.4.0 +pytest==7.4.3 pyyaml==6.0.1 requests==2.31.0 -responses==0.23.3 -tox==3.28.0 # can't be updated to 4+ due to flake8@5.0.4 needing importlib-metadata<4.3 and tox needs importlib-metadata>=5.1 -types-pyyaml==6.0.12.11 -types-requests==2.31.0.2 -typing-extensions==4.7.1 +responses==0.24.1 +tox==4.11.3 +types-pyyaml==6.0.12.12 +types-requests==2.31.0.10 +typing-extensions==4.8.0 unidiff==0.7.5 diff --git a/setup.py b/setup.py index d8694d5d..96c71381 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def get_version(): setup( name='bc-detect-secrets', - python_requires='>=3.7', + python_requires='>=3.8', packages=find_packages(exclude=(['test*', 'tmp*'])), version=VERSION, description='Tool for detecting secrets in the codebase', @@ -57,7 +57,6 @@ def get_version(): }, classifiers=[ 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/testing/factories.py b/testing/factories.py index 9f1294cc..1d71baec 100644 --- a/testing/factories.py +++ b/testing/factories.py @@ -18,5 +18,5 @@ def potential_secret_factory( filename=filename, secret=secret, line_number=line_number, - **kwargs + **kwargs, ) diff --git a/testing/plugins.py b/testing/plugins.py index 5a5957fc..07f5b92c 100644 --- a/testing/plugins.py +++ b/testing/plugins.py @@ -1,16 +1,21 @@ +from __future__ import annotations + import re from contextlib import contextmanager from typing import Any from typing import Generator +from typing import TYPE_CHECKING -from detect_secrets.core.plugins import Plugin from detect_secrets.core.plugins.util import get_mapping_from_secret_type_to_class from detect_secrets.plugins.base import RegexBasedDetector +if TYPE_CHECKING: + from detect_secrets.plugins.base import BasePlugin + @contextmanager -def register_plugin(plugin: Plugin) -> Generator[None, None, None]: - def get_instance(*args: Any, **kwargs: Any) -> Plugin: +def register_plugin(plugin: BasePlugin) -> Generator[None, None, None]: + def get_instance(*args: Any, **kwargs: Any) -> BasePlugin: # NOTE: We need this, because the initialization process auto-fills in arguments # to the classname. However, we already have an instance, so it doesn't matter. return plugin diff --git a/tox.ini b/tox.ini index 28ae07f8..01cd649d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,11 @@ [tox] project = detect_secrets # These should match the ci python env list -envlist = py{37,38,39,310,311,312},mypy +envlist = py{38,39,310,311,312},mypy skip_missing_interpreters = true [testenv] passenv = SSH_AUTH_SOCK -# NO_PROXY is needed to call requests API within a forked process -# when using macOS and python version 3.6/3.7 -setenv = - NO_PROXY = '*' deps = -rrequirements-dev.txt whitelist_externals = coverage commands = @@ -28,7 +24,7 @@ commands = pre-commit install -f --install-hooks [testenv:pre-commit] -deps = pre-commit >= 1.16.1 +deps = pre-commit >= 3.0.0 commands = pre-commit {posargs} [pep8]