From a03b74f485c2717d450ae6e9906705151cbdfb0e Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 15:36:00 -0700 Subject: [PATCH 01/11] feat: repo specific exemptions for package ecosystems Signed-off-by: Zack Koppert --- .env-example | 1 + README.md | 1 + dependabot_file.py | 14 +++++- env.py | 32 +++++++++++++ evergreen.py | 7 ++- test_dependabot_file.py | 36 +++++++------- test_env.py | 103 ++++++++++++++++++++++++++++++++++++++-- 7 files changed, 170 insertions(+), 24 deletions(-) diff --git a/.env-example b/.env-example index 50e5156..b26865f 100644 --- a/.env-example +++ b/.env-example @@ -11,6 +11,7 @@ GH_TOKEN = "" GROUP_DEPENDENCIES = "" ORGANIZATION = "" PROJECT_ID = "" +REPO_SPECIFIC_EXEMPTIONS = "" REPOSITORY = "" TITLE = "" TYPE = "" diff --git a/README.md b/README.md index fad6909..9a450ac 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ This action can be configured to authenticate with GitHub App Installation or Pe | `BATCH_SIZE` | False | None | Set this to define the maximum amount of eligible repositories for every run. This is useful if you are targeting large organizations and you don't want to flood repositories with pull requests / issues. ex: if you want to target 20 repositories per time, set this to 20. | | `ENABLE_SECURITY_UPDATES` | False | true | If set to true, Evergreen will enable [Dependabot security updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates) on target repositories. Note that the GitHub token needs to have the `administration:write` permission on every repository in scope to successfully enable security updates. | | `EXEMPT_ECOSYSTEMS` | False | "" | A list of [package ecosystems](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem) to exempt from the generated dependabot configuration. To ignore ecosystems set this to one or more of `bundler`,`cargo`, `composer`, `pip`, `docker`, `npm`, `gomod`, `mix`, `nuget`, `github-actions` and `terraform`. ex: if you don't want Dependabot to update Dockerfiles and Github Actions you can set this to `docker,github-actions`. | +| `REPO_SPECIFIC_EXEMPTIONS | False | "" | A list of repositories that should be exempt from specific package ecosystems similar to EXEMPT_ECOSYSTEMS but those apply to all repositories. ex: `org1/repo1:docker,github-actions;org1/repo2:pip` would set exempt_ecosystems for `org1/repo1` to be `['docker', 'github-actions']`, and for `org1/repo2` it would be `['pip']`, while for every other repository evaluated, it would be set by the env variable `EXEMPT_ECOSYSTEMS`. NOTE: If you want specific exemptions to be added on top of the already specified global exemptions, you need to add the global exemptions to each repo specfic exemption. | ### Example workflows diff --git a/dependabot_file.py b/dependabot_file.py index 92b0939..27bf0a7 100644 --- a/dependabot_file.py +++ b/dependabot_file.py @@ -32,7 +32,11 @@ def make_dependabot_config(ecosystem, group_dependencies, indent) -> str: def build_dependabot_file( - repo, group_dependencies, exempt_ecosystems, existing_config + repo, + group_dependencies, + exempt_ecosystems, + repo_specfic_exemptions, + existing_config, ) -> str | None: """ Build the dependabot.yml file for a repo based on the repo contents @@ -41,6 +45,7 @@ def build_dependabot_file( repo: the repository to build the dependabot.yml file for group_dependencies: whether to group dependencies in the dependabot.yml file exempt_ecosystems: the list of ecosystems to ignore + repo_specfic_exemptions: the list of ecosystems to ignore for a specific repo existing_config: the existing dependabot configuration file or None if it doesn't exist Returns: @@ -83,6 +88,13 @@ def build_dependabot_file( add_existing_ecosystem_to_exempt_list(exempt_ecosystems, existing_config) + # If there are repository specific exemptions, + # overwrite the global exemptions for this repo only + if repo_specfic_exemptions and repo.full_name in repo_specfic_exemptions: + exempt_ecosystems = [] + for ecosystem in repo_specfic_exemptions[repo.full_name]: + exempt_ecosystems.append(ecosystem) + package_managers = { "bundler": ["Gemfile", "Gemfile.lock"], "npm": ["package.json", "package-lock.json", "yarn.lock"], diff --git a/env.py b/env.py index e36709c..fb84e2e 100644 --- a/env.py +++ b/env.py @@ -43,6 +43,30 @@ def get_int_env_var(env_var_name: str) -> int | None: return None +def parse_repo_specific_exemptions(repo_specific_exemptions_str: str) -> dict: + """Parse the REPO_SPECIFIC_EXEMPTIONS environment variable into a dictionary. + + Args: + repo_specific_exemptions_str: The REPO_SPECIFIC_EXEMPTIONS environment variable as a string. + + Returns: + A dictionary where keys are repository names and values are lists of exempt ecosystems. + """ + exemptions_dict = {} + if repo_specific_exemptions_str: + exemptions_list = repo_specific_exemptions_str.split(";") + for exemption in exemptions_list: + if ( + exemption == "" + ): # Account for final ; in the repo_specific_exemptions_str + continue + repo, ecosystems = exemption.split(":") + exemptions_dict[repo.strip()] = [ + ecosystem.strip() for ecosystem in ecosystems.split(",") + ] + return exemptions_dict + + def get_env_vars(test: bool = False) -> tuple[ str | None, list[str], @@ -65,6 +89,7 @@ def get_env_vars(test: bool = False) -> tuple[ bool | None, list[str], bool | None, + dict, ]: """ Get the environment variables for use in the action. @@ -93,6 +118,7 @@ def get_env_vars(test: bool = False) -> tuple[ enable_security_updates (bool): Whether to enable security updates in target repositories exempt_ecosystems_list (list[str]): A list of package ecosystems to exempt from the action update_existing (bool): Whether to update existing dependabot configuration files + repo_specific_exemptions (dict): A dictionary of per repository ecosystem exemptions """ if not test: @@ -234,6 +260,11 @@ def get_env_vars(test: bool = False) -> tuple[ update_existing = get_bool_env_var("UPDATE_EXISTING") + repo_specific_exemptions_str = os.getenv("REPO_SPECIFIC_EXEMPTIONS", "") + repo_specific_exemptions = parse_repo_specific_exemptions( + repo_specific_exemptions_str + ) + return ( organization, repositories_list, @@ -256,4 +287,5 @@ def get_env_vars(test: bool = False) -> tuple[ enable_security_updates_bool, exempt_ecosystems_list, update_existing, + repo_specific_exemptions, ) diff --git a/evergreen.py b/evergreen.py index 83a6dfb..a4c7dd0 100644 --- a/evergreen.py +++ b/evergreen.py @@ -36,6 +36,7 @@ def main(): # pragma: no cover enable_security_updates, exempt_ecosystems, update_existing, + repo_specfic_exemptions, ) = env.get_env_vars() # Auth to GitHub.com or GHE @@ -96,7 +97,11 @@ def main(): # pragma: no cover print("Checking " + repo.full_name + " for compatible package managers") # Try to detect package managers and build a dependabot file dependabot_file = build_dependabot_file( - repo, group_dependencies, exempt_ecosystems, existing_config + repo, + group_dependencies, + exempt_ecosystems, + repo_specfic_exemptions, + existing_config, ) if dependabot_file is None: diff --git a/test_dependabot_file.py b/test_dependabot_file.py index 61bdc94..c16cddd 100644 --- a/test_dependabot_file.py +++ b/test_dependabot_file.py @@ -20,7 +20,7 @@ def test_not_found_error(self): response.status_code = 404 repo.file_contents.side_effect = github3.exceptions.NotFoundError(resp=response) - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, None) def test_build_dependabot_file_with_bundler(self): @@ -38,7 +38,7 @@ def test_build_dependabot_file_with_bundler(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_existing_config_bundler_no_update(self): @@ -51,7 +51,7 @@ def test_build_dependabot_file_with_existing_config_bundler_no_update(self): existing_config = MagicMock() existing_config.decoded = b'---\nversion: 2\nupdates:\n - package-ecosystem: "bundler"\n\ directory: "/"\n schedule:\n interval: "weekly"\n commit-message:\n prefix: "chore(deps)"\n' - result = build_dependabot_file(repo, False, [], existing_config) + result = build_dependabot_file(repo, False, [], {}, existing_config) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_2_space_indent_existing_config_bundler_with_update( @@ -80,7 +80,7 @@ def test_build_dependabot_file_with_2_space_indent_existing_config_bundler_with_ existing_config = MagicMock() existing_config.decoded = b'---\nversion: 2\nupdates:\n - package-ecosystem: "pip"\n directory: "/"\n\ schedule:\n interval: "weekly"\n commit-message:\n prefix: "chore(deps)"\n' - result = build_dependabot_file(repo, False, [], existing_config) + result = build_dependabot_file(repo, False, [], {}, existing_config) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_weird_space_indent_existing_config_bundler_with_update( @@ -95,7 +95,7 @@ def test_build_dependabot_file_with_weird_space_indent_existing_config_bundler_w existing_config = MagicMock() existing_config.decoded = b'---\nversion: 2\nupdates:\n- package-ecosystem: "pip"\n directory: "/"\n\ schedule:\n interval: "weekly"\n commit-message:\n prefix: "chore(deps)"\n' - result = build_dependabot_file(repo, False, [], existing_config) + result = build_dependabot_file(repo, False, [], {}, existing_config) self.assertEqual(result, None) def test_build_dependabot_file_with_npm(self): @@ -113,7 +113,7 @@ def test_build_dependabot_file_with_npm(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_pip(self): @@ -137,7 +137,7 @@ def test_build_dependabot_file_with_pip(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_cargo(self): @@ -158,7 +158,7 @@ def test_build_dependabot_file_with_cargo(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_gomod(self): @@ -174,7 +174,7 @@ def test_build_dependabot_file_with_gomod(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_composer(self): @@ -195,7 +195,7 @@ def test_build_dependabot_file_with_composer(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_hex(self): @@ -216,7 +216,7 @@ def test_build_dependabot_file_with_hex(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_nuget(self): @@ -232,7 +232,7 @@ def test_build_dependabot_file_with_nuget(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_docker(self): @@ -248,7 +248,7 @@ def test_build_dependabot_file_with_docker(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_terraform_with_files(self): @@ -269,7 +269,7 @@ def test_build_dependabot_file_with_terraform_with_files(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_terraform_without_files(self): @@ -281,7 +281,7 @@ def test_build_dependabot_file_with_terraform_without_files(self): # Test absence of Terraform files repo.directory_contents.side_effect = lambda path: [] if path == "/" else [] - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertIsNone(result) # Test empty repository @@ -290,7 +290,7 @@ def test_build_dependabot_file_with_terraform_without_files(self): repo.directory_contents.side_effect = github3.exceptions.NotFoundError( resp=response ) - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], {}, None) self.assertIsNone(result) def test_build_dependabot_file_with_groups(self): @@ -311,7 +311,7 @@ def test_build_dependabot_file_with_groups(self): development-dependencies: dependency-type: 'development' """ - result = build_dependabot_file(repo, True, [], None) + result = build_dependabot_file(repo, True, [], {}, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_exempt_ecosystems(self): @@ -319,7 +319,7 @@ def test_build_dependabot_file_with_exempt_ecosystems(self): repo = MagicMock() repo.file_contents.side_effect = lambda filename: filename == "Dockerfile" - result = build_dependabot_file(repo, False, ["docker"], None) + result = build_dependabot_file(repo, False, ["docker"], {}, None) self.assertEqual(result, None) def test_add_existing_ecosystem_to_exempt_list(self): diff --git a/test_env.py b/test_env.py index facaf1e..4a7d07a 100644 --- a/test_env.py +++ b/test_env.py @@ -28,6 +28,7 @@ def setUp(self): "TITLE", "TYPE", "UPDATE_EXISTING", + "REPO_SPECIFIC_EXEMPTIONS", ] for key in env_keys: if key in os.environ: @@ -73,6 +74,56 @@ def test_get_env_vars_with_org(self): True, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions + ) + result = get_env_vars(True) + self.assertEqual(result, expected_result) + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "EXEMPT_REPOS": "repo4,repo5", + "GH_TOKEN": "my_token", + "TYPE": "issue", + "TITLE": "Dependabot Alert custom title", + "BODY": "Dependabot custom body", + "CREATED_AFTER_DATE": "2020-01-01", + "COMMIT_MESSAGE": "Create dependabot configuration", + "PROJECT_ID": "123", + "GROUP_DEPENDENCIES": "false", + "REPO_SPECIFIC_EXEMPTIONS": "repo1:gomod;repo2:docker,gomod;", + }, + clear=True, + ) + def test_get_env_vars_with_org_and_repo_specific_exemptions(self): + """Test that all environment variables are set correctly using an organization""" + expected_result = ( + "my_organization", + [], + None, + None, + b"", + "my_token", + "", + ["repo4", "repo5"], + "issue", + "Dependabot Alert custom title", + "Dependabot custom body", + "2020-01-01", + False, + "Create dependabot configuration", + "123", + False, + ["internal", "private", "public"], + None, # batch_size + True, # enable_security_updates + [], # exempt_ecosystems + False, # update_existing + { + "repo1": ["gomod"], + "repo2": ["docker", "gomod"], + }, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -91,6 +142,35 @@ def test_get_env_vars_with_org(self): "COMMIT_MESSAGE": "Create dependabot configuration", "PROJECT_ID": "123", "GROUP_DEPENDENCIES": "false", + "REPO_SPECIFIC_EXEMPTIONS": "org1/repo1-docker;org2/repo2", + }, + clear=True, + ) + def test_get_env_vars_repo_specific_exemptions_improper_format(self): + """Test that REPO_SPECIFIC_EXEMPTIONS is handled correctly when improperly formatted""" + with self.assertRaises(ValueError) as cm: + get_env_vars(True) + the_exception = cm.exception + self.assertEqual( + str(the_exception), + "not enough values to unpack (expected 2, got 1)", + ) + + @patch.dict( + os.environ, + { + "REPOSITORY": "org/repo1,org2/repo2", + "GH_TOKEN": "my_token", + "EXEMPT_REPOS": "repo4,repo5", + "TYPE": "pull", + "TITLE": "Dependabot Alert custom title", + "BODY": "Dependabot custom body", + "CREATED_AFTER_DATE": "2023-01-01", + "DRY_RUN": "true", + "COMMIT_MESSAGE": "Create dependabot configuration", + "PROJECT_ID": "123", + "GROUP_DEPENDENCIES": "false", + "REPO_SPECIFIC_EXEMPTIONS": "org1/repo1:docker;org2/repo2:gomod", }, clear=True, ) @@ -118,6 +198,10 @@ def test_get_env_vars_with_repos(self): True, # enable_security_updates [], # exempt_ecosystems False, # update_existing + { + "org1/repo1": ["docker"], + "org2/repo2": ["gomod"], + }, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -156,6 +240,7 @@ def test_get_env_vars_optional_values(self): True, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -195,6 +280,7 @@ def test_get_env_vars_with_update_existing(self): True, # enable_security_updates [], # exempt_ecosystems True, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -248,6 +334,7 @@ def test_get_env_vars_auth_with_github_app_installation(self): True, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -309,6 +396,7 @@ def test_get_env_vars_with_repos_no_dry_run(self): True, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -348,6 +436,7 @@ def test_get_env_vars_with_repos_disabled_security_updates(self): False, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -388,6 +477,7 @@ def test_get_env_vars_with_repos_filter_visibility_multiple_values(self): False, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -428,6 +518,7 @@ def test_get_env_vars_with_repos_filter_visibility_single_value(self): False, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -498,6 +589,7 @@ def test_get_env_vars_with_repos_filter_visibility_no_duplicates(self): False, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -509,12 +601,12 @@ def test_get_env_vars_with_repos_filter_visibility_no_duplicates(self): "GH_TOKEN": "my_token", "ENABLE_SECURITY_UPDATES": "false", "FILTER_VISIBILITY": "private,private,public", - "EXEMPT_ECOSYSTEMS": "gomod,DOCKER", + "EXEMPT_ECOSYSTEMS": "gomod,docker", }, clear=True, ) def test_get_env_vars_with_repos_exempt_ecosystems(self): - """Test that filter_visibility is set correctly when there are duplicate values""" + """Test that filter_visibility is set correctly when there are exempt ecosystems""" expected_result = ( "my_organization", [], @@ -539,6 +631,7 @@ def test_get_env_vars_with_repos_exempt_ecosystems(self): False, # enable_security_updates ["gomod", "docker"], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -554,7 +647,7 @@ def test_get_env_vars_with_repos_exempt_ecosystems(self): clear=True, ) def test_get_env_vars_with_no_batch_size(self): - """Test that filter_visibility is set correctly when there are duplicate values""" + """Test that filter_visibility is set correctly when there is no batch size provided""" expected_result = ( "my_organization", [], @@ -579,6 +672,7 @@ def test_get_env_vars_with_no_batch_size(self): False, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -595,7 +689,7 @@ def test_get_env_vars_with_no_batch_size(self): clear=True, ) def test_get_env_vars_with_batch_size(self): - """Test that filter_visibility is set correctly when there are duplicate values""" + """Test that filter_visibility is set correctly when there is a batch size""" expected_result = ( "my_organization", [], @@ -620,6 +714,7 @@ def test_get_env_vars_with_batch_size(self): False, # enable_security_updates [], # exempt_ecosystems False, # update_existing + {}, # repo_specific_exemptions ) result = get_env_vars(True) self.assertEqual(result, expected_result) From 9b7b78b5efce9cf8a0d2a6850d0be7c89245efce Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 15:42:03 -0700 Subject: [PATCH 02/11] chore: spelling fix Signed-off-by: Zack Koppert --- dependabot_file.py | 8 ++++---- evergreen.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dependabot_file.py b/dependabot_file.py index 27bf0a7..8e818f8 100644 --- a/dependabot_file.py +++ b/dependabot_file.py @@ -35,7 +35,7 @@ def build_dependabot_file( repo, group_dependencies, exempt_ecosystems, - repo_specfic_exemptions, + repo_specific_exemptions, existing_config, ) -> str | None: """ @@ -45,7 +45,7 @@ def build_dependabot_file( repo: the repository to build the dependabot.yml file for group_dependencies: whether to group dependencies in the dependabot.yml file exempt_ecosystems: the list of ecosystems to ignore - repo_specfic_exemptions: the list of ecosystems to ignore for a specific repo + repo_specific_exemptions: the list of ecosystems to ignore for a specific repo existing_config: the existing dependabot configuration file or None if it doesn't exist Returns: @@ -90,9 +90,9 @@ def build_dependabot_file( # If there are repository specific exemptions, # overwrite the global exemptions for this repo only - if repo_specfic_exemptions and repo.full_name in repo_specfic_exemptions: + if repo_specific_exemptions and repo.full_name in repo_specific_exemptions: exempt_ecosystems = [] - for ecosystem in repo_specfic_exemptions[repo.full_name]: + for ecosystem in repo_specific_exemptions[repo.full_name]: exempt_ecosystems.append(ecosystem) package_managers = { diff --git a/evergreen.py b/evergreen.py index a4c7dd0..2576bd0 100644 --- a/evergreen.py +++ b/evergreen.py @@ -36,7 +36,7 @@ def main(): # pragma: no cover enable_security_updates, exempt_ecosystems, update_existing, - repo_specfic_exemptions, + repo_specific_exemptions, ) = env.get_env_vars() # Auth to GitHub.com or GHE @@ -100,7 +100,7 @@ def main(): # pragma: no cover repo, group_dependencies, exempt_ecosystems, - repo_specfic_exemptions, + repo_specific_exemptions, existing_config, ) From 67e925f908d6e3773e365fbb79afed653a4a9ab8 Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 15:48:43 -0700 Subject: [PATCH 03/11] chore: better error handling for env var Signed-off-by: Zack Koppert --- env.py | 8 ++++++++ test_env.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/env.py b/env.py index fb84e2e..d3f2c69 100644 --- a/env.py +++ b/env.py @@ -54,6 +54,14 @@ def parse_repo_specific_exemptions(repo_specific_exemptions_str: str) -> dict: """ exemptions_dict = {} if repo_specific_exemptions_str: + # if repo_specific_exemptions_str doesn't have a ; and : character, it's not valid + if ( + repo_specific_exemptions_str.find(";") == -1 + or repo_specific_exemptions_str.find(":") == -1 + ): + raise ValueError( + "REPO_SPECIFIC_EXEMPTIONS environment variable not formatted correctly" + ) exemptions_list = repo_specific_exemptions_str.split(";") for exemption in exemptions_list: if ( diff --git a/test_env.py b/test_env.py index 4a7d07a..a8123ef 100644 --- a/test_env.py +++ b/test_env.py @@ -153,7 +153,7 @@ def test_get_env_vars_repo_specific_exemptions_improper_format(self): the_exception = cm.exception self.assertEqual( str(the_exception), - "not enough values to unpack (expected 2, got 1)", + "REPO_SPECIFIC_EXEMPTIONS environment variable not formatted correctly", ) @patch.dict( From eda592e288b7a8b3d557daf0559d98f5b36c5377 Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 15:54:43 -0700 Subject: [PATCH 04/11] test: add test case to cover repo specific exemption working effectively Signed-off-by: Zack Koppert --- test_dependabot_file.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test_dependabot_file.py b/test_dependabot_file.py index c16cddd..2555cd3 100644 --- a/test_dependabot_file.py +++ b/test_dependabot_file.py @@ -322,6 +322,15 @@ def test_build_dependabot_file_with_exempt_ecosystems(self): result = build_dependabot_file(repo, False, ["docker"], {}, None) self.assertEqual(result, None) + def test_build_dependabot_file_with_repo_specific_exempt_ecosystems(self): + """Test that the dependabot.yml file is built correctly with exempted ecosystems""" + repo = MagicMock() + repo.full_name = "test/test" + repo.file_contents.side_effect = lambda filename: filename == "Dockerfile" + + result = build_dependabot_file(repo, False, [], {"test/test": ["docker"]}, None) + self.assertEqual(result, None) + def test_add_existing_ecosystem_to_exempt_list(self): """Test that existing ecosystems are added to the exempt list""" exempt_ecosystems = ["npm", "pip", "github-actions"] From 3e2a497dcc83335203485fed0f7a6804178cf726 Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 15:56:06 -0700 Subject: [PATCH 05/11] docs: add missing ` Signed-off-by: Zack Koppert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a450ac..deb83de 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ This action can be configured to authenticate with GitHub App Installation or Pe | `BATCH_SIZE` | False | None | Set this to define the maximum amount of eligible repositories for every run. This is useful if you are targeting large organizations and you don't want to flood repositories with pull requests / issues. ex: if you want to target 20 repositories per time, set this to 20. | | `ENABLE_SECURITY_UPDATES` | False | true | If set to true, Evergreen will enable [Dependabot security updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates) on target repositories. Note that the GitHub token needs to have the `administration:write` permission on every repository in scope to successfully enable security updates. | | `EXEMPT_ECOSYSTEMS` | False | "" | A list of [package ecosystems](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem) to exempt from the generated dependabot configuration. To ignore ecosystems set this to one or more of `bundler`,`cargo`, `composer`, `pip`, `docker`, `npm`, `gomod`, `mix`, `nuget`, `github-actions` and `terraform`. ex: if you don't want Dependabot to update Dockerfiles and Github Actions you can set this to `docker,github-actions`. | -| `REPO_SPECIFIC_EXEMPTIONS | False | "" | A list of repositories that should be exempt from specific package ecosystems similar to EXEMPT_ECOSYSTEMS but those apply to all repositories. ex: `org1/repo1:docker,github-actions;org1/repo2:pip` would set exempt_ecosystems for `org1/repo1` to be `['docker', 'github-actions']`, and for `org1/repo2` it would be `['pip']`, while for every other repository evaluated, it would be set by the env variable `EXEMPT_ECOSYSTEMS`. NOTE: If you want specific exemptions to be added on top of the already specified global exemptions, you need to add the global exemptions to each repo specfic exemption. | +| `REPO_SPECIFIC_EXEMPTIONS` | False | "" | A list of repositories that should be exempt from specific package ecosystems similar to EXEMPT_ECOSYSTEMS but those apply to all repositories. ex: `org1/repo1:docker,github-actions;org1/repo2:pip` would set exempt_ecosystems for `org1/repo1` to be `['docker', 'github-actions']`, and for `org1/repo2` it would be `['pip']`, while for every other repository evaluated, it would be set by the env variable `EXEMPT_ECOSYSTEMS`. NOTE: If you want specific exemptions to be added on top of the already specified global exemptions, you need to add the global exemptions to each repo specfic exemption. | ### Example workflows From 8fcccd27ea9ed9ee2ce6f2b5d9e961ac3f768c39 Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 15:57:18 -0700 Subject: [PATCH 06/11] docs: spelling fix Signed-off-by: Zack Koppert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index deb83de..d2ae56b 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ This action can be configured to authenticate with GitHub App Installation or Pe | `BATCH_SIZE` | False | None | Set this to define the maximum amount of eligible repositories for every run. This is useful if you are targeting large organizations and you don't want to flood repositories with pull requests / issues. ex: if you want to target 20 repositories per time, set this to 20. | | `ENABLE_SECURITY_UPDATES` | False | true | If set to true, Evergreen will enable [Dependabot security updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates) on target repositories. Note that the GitHub token needs to have the `administration:write` permission on every repository in scope to successfully enable security updates. | | `EXEMPT_ECOSYSTEMS` | False | "" | A list of [package ecosystems](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem) to exempt from the generated dependabot configuration. To ignore ecosystems set this to one or more of `bundler`,`cargo`, `composer`, `pip`, `docker`, `npm`, `gomod`, `mix`, `nuget`, `github-actions` and `terraform`. ex: if you don't want Dependabot to update Dockerfiles and Github Actions you can set this to `docker,github-actions`. | -| `REPO_SPECIFIC_EXEMPTIONS` | False | "" | A list of repositories that should be exempt from specific package ecosystems similar to EXEMPT_ECOSYSTEMS but those apply to all repositories. ex: `org1/repo1:docker,github-actions;org1/repo2:pip` would set exempt_ecosystems for `org1/repo1` to be `['docker', 'github-actions']`, and for `org1/repo2` it would be `['pip']`, while for every other repository evaluated, it would be set by the env variable `EXEMPT_ECOSYSTEMS`. NOTE: If you want specific exemptions to be added on top of the already specified global exemptions, you need to add the global exemptions to each repo specfic exemption. | +| `REPO_SPECIFIC_EXEMPTIONS` | False | "" | A list of repositories that should be exempt from specific package ecosystems similar to EXEMPT_ECOSYSTEMS but those apply to all repositories. ex: `org1/repo1:docker,github-actions;org1/repo2:pip` would set exempt_ecosystems for `org1/repo1` to be `['docker', 'github-actions']`, and for `org1/repo2` it would be `['pip']`, while for every other repository evaluated, it would be set by the env variable `EXEMPT_ECOSYSTEMS`. NOTE: If you want specific exemptions to be added on top of the already specified global exemptions, you need to add the global exemptions to each repo specific exemption. | ### Example workflows From 92c8ef0dd1ce659c754aa91b25f2c91f6c3f1f8e Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 16:06:54 -0700 Subject: [PATCH 07/11] fix: error handling for unsupported package ecosystem and a test to cover it Signed-off-by: Zack Koppert --- env.py | 17 +++++++++++++++++ test_env.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/env.py b/env.py index d3f2c69..696aca2 100644 --- a/env.py +++ b/env.py @@ -69,6 +69,23 @@ def parse_repo_specific_exemptions(repo_specific_exemptions_str: str) -> dict: ): # Account for final ; in the repo_specific_exemptions_str continue repo, ecosystems = exemption.split(":") + for ecosystem in ecosystems.split(","): + if ecosystem not in [ + "bundler", + "npm", + "pip", + "cargo", + "docker", + "gomod", + "composer", + "mix", + "github-actions", + "nuget", + "terraform", + ]: + raise ValueError( + "REPO_SPECIFIC_EXEMPTIONS environment variable not formatted correctly. Unrecognized package-ecosystem." + ) exemptions_dict[repo.strip()] = [ ecosystem.strip() for ecosystem in ecosystems.split(",") ] diff --git a/test_env.py b/test_env.py index a8123ef..5a918b1 100644 --- a/test_env.py +++ b/test_env.py @@ -156,6 +156,34 @@ def test_get_env_vars_repo_specific_exemptions_improper_format(self): "REPO_SPECIFIC_EXEMPTIONS environment variable not formatted correctly", ) + @patch.dict( + os.environ, + { + "REPOSITORY": "org/repo1,org2/repo2", + "GH_TOKEN": "my_token", + "EXEMPT_REPOS": "repo4,repo5", + "TYPE": "pull", + "TITLE": "Dependabot Alert custom title", + "BODY": "Dependabot custom body", + "CREATED_AFTER_DATE": "2023-01-01", + "DRY_RUN": "true", + "COMMIT_MESSAGE": "Create dependabot configuration", + "PROJECT_ID": "123", + "GROUP_DEPENDENCIES": "false", + "REPO_SPECIFIC_EXEMPTIONS": "org1/repo1:snap;org2/repo2:docker", + }, + clear=True, + ) + def test_get_env_vars_repo_specific_exemptions_unsupported_ecosystem(self): + """Test that REPO_SPECIFIC_EXEMPTIONS is handled correctly when unsupported ecosystem is provided""" + with self.assertRaises(ValueError) as cm: + get_env_vars(True) + the_exception = cm.exception + self.assertEqual( + str(the_exception), + "REPO_SPECIFIC_EXEMPTIONS environment variable not formatted correctly. Unrecognized package-ecosystem.", + ) + @patch.dict( os.environ, { From 1996a9f463a74af9790051f81d837e20850ebf47 Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 20:01:19 -0700 Subject: [PATCH 08/11] extend separator list Co-authored-by: Jason Meridth --- env.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/env.py b/env.py index 696aca2..bd189c1 100644 --- a/env.py +++ b/env.py @@ -55,10 +55,8 @@ def parse_repo_specific_exemptions(repo_specific_exemptions_str: str) -> dict: exemptions_dict = {} if repo_specific_exemptions_str: # if repo_specific_exemptions_str doesn't have a ; and : character, it's not valid - if ( - repo_specific_exemptions_str.find(";") == -1 - or repo_specific_exemptions_str.find(":") == -1 - ): + separators = [';',':'] + if not any(sep in repo_specific_exemptions_str for sep in separators): raise ValueError( "REPO_SPECIFIC_EXEMPTIONS environment variable not formatted correctly" ) From dafc71dadc3aa5b1e4d1649b99d1dcca80ec895f Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 31 May 2024 20:02:47 -0700 Subject: [PATCH 09/11] chore: alphabetize Co-authored-by: Jason Meridth --- env.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/env.py b/env.py index bd189c1..5c5fa98 100644 --- a/env.py +++ b/env.py @@ -70,15 +70,15 @@ def parse_repo_specific_exemptions(repo_specific_exemptions_str: str) -> dict: for ecosystem in ecosystems.split(","): if ecosystem not in [ "bundler", - "npm", - "pip", "cargo", + "composer", "docker", + "github-actions", "gomod", - "composer", "mix", - "github-actions", + "npm", "nuget", + "pip", "terraform", ]: raise ValueError( From c47d959bbd813b1fc11e84eb40378cbb0d2abb3b Mon Sep 17 00:00:00 2001 From: jmeridth Date: Sat, 1 Jun 2024 09:08:36 -0500 Subject: [PATCH 10/11] fix: use all instead of any and lint fix :facepalm: needed to use all instead of any Signed-off-by: jmeridth --- env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/env.py b/env.py index 5c5fa98..d0484e5 100644 --- a/env.py +++ b/env.py @@ -55,8 +55,8 @@ def parse_repo_specific_exemptions(repo_specific_exemptions_str: str) -> dict: exemptions_dict = {} if repo_specific_exemptions_str: # if repo_specific_exemptions_str doesn't have a ; and : character, it's not valid - separators = [';',':'] - if not any(sep in repo_specific_exemptions_str for sep in separators): + separators = [";", ":"] + if not all(sep in repo_specific_exemptions_str for sep in separators): raise ValueError( "REPO_SPECIFIC_EXEMPTIONS environment variable not formatted correctly" ) From 6dbaedf43493d25dd0c6387e89496ec96c0104c1 Mon Sep 17 00:00:00 2001 From: jmeridth Date: Sat, 1 Jun 2024 09:16:26 -0500 Subject: [PATCH 11/11] fix: fix tests, missing arguments Signed-off-by: jmeridth --- test_dependabot_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_dependabot_file.py b/test_dependabot_file.py index ca1b35a..36b80b6 100644 --- a/test_dependabot_file.py +++ b/test_dependabot_file.py @@ -311,7 +311,7 @@ def test_build_dependabot_file_with_github_actions(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], None, None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_github_actions_without_files(self): @@ -324,7 +324,7 @@ def test_build_dependabot_file_with_github_actions_without_files(self): resp=response ) - result = build_dependabot_file(repo, False, [], None) + result = build_dependabot_file(repo, False, [], None, None) self.assertEqual(result, None) def test_build_dependabot_file_with_groups(self):