From 58532228d47f49efa88f49e7f7b382b0024b7e55 Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Fri, 24 May 2024 22:26:40 -0700 Subject: [PATCH] feat: Update existing configs with missing package-ecosystems Signed-off-by: Zack Koppert --- dependabot_file.py | 26 ++++++++++++++++++-- env.py | 7 ++++++ evergreen.py | 37 ++++++++++++++++++---------- requirements.txt | 2 ++ test_dependabot_file.py | 24 +++++++++--------- test_env.py | 54 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 27 deletions(-) diff --git a/dependabot_file.py b/dependabot_file.py index c6c800b..6a92bac 100644 --- a/dependabot_file.py +++ b/dependabot_file.py @@ -1,6 +1,7 @@ """This module contains the function to build the dependabot.yml file for a repo""" import github3 +import yaml def make_dependabot_config(ecosystem, group_dependencies) -> str: @@ -29,7 +30,9 @@ def make_dependabot_config(ecosystem, group_dependencies) -> str: return dependabot_config -def build_dependabot_file(repo, group_dependencies, exempt_ecosystems) -> str | None: +def build_dependabot_file( + repo, group_dependencies, exempt_ecosystems, existing_config +) -> str | None: """ Build the dependabot.yml file for a repo based on the repo contents @@ -37,6 +40,7 @@ def build_dependabot_file(repo, group_dependencies, exempt_ecosystems) -> str | repo: the repository to build the dependabot.yml file for group_dependencies: whether to group dependencies in the dependabot.yml file exempt_ecosystems: the list of ecosystems to ignore + existing_config: the existing dependabot configuration file or None if it doesn't exist Returns: str: the dependabot.yml file for the repo @@ -51,10 +55,16 @@ def build_dependabot_file(repo, group_dependencies, exempt_ecosystems) -> str | hex_found = False nuget_found = False docker_found = False - dependabot_file = """--- + if existing_config: + dependabot_file = existing_config.decoded.decode("utf-8") + else: + dependabot_file = """--- version: 2 updates: """ + + add_existing_ecosystem_to_exempt_list(exempt_ecosystems, existing_config) + try: if ( repo.file_contents("Gemfile") @@ -305,3 +315,15 @@ def build_dependabot_file(repo, group_dependencies, exempt_ecosystems) -> str | if compatible_package_manager_found: return dependabot_file return None + + +def add_existing_ecosystem_to_exempt_list(exempt_ecosystems, existing_config): + """ + Add the existing package ecosystems found in the dependabot.yml + to the exempt list so we don't get duplicate entries and maintain configuration settings + """ + if existing_config: + existing_config_obj = yaml.safe_load(existing_config.decoded) + if existing_config_obj: + for entry in existing_config_obj.get("updates", []): + exempt_ecosystems.append(entry["package-ecosystem"]) diff --git a/env.py b/env.py index 9f01899..5aee695 100644 --- a/env.py +++ b/env.py @@ -64,6 +64,7 @@ def get_env_vars(test: bool = False) -> tuple[ int | None, bool | None, list[str], + bool | None, ]: """ Get the environment variables for use in the action. @@ -91,7 +92,9 @@ def get_env_vars(test: bool = False) -> tuple[ batch_size (int): The max number of repositories in scope enable_security_updates (bool): Whether to enable security updates in target repositories exempt_ecosystems_list (list[str]): A list of package ecosystems to exempt from the action + update_existing (bool): Whether to update existing dependabot configuration files """ + if not test: # Load from .env file if it exists and not testing dotenv_path = join(dirname(__file__), ".env") @@ -228,6 +231,9 @@ def get_env_vars(test: bool = False) -> tuple[ project_id = os.getenv("PROJECT_ID") if project_id and not project_id.isnumeric(): raise ValueError("PROJECT_ID environment variable is not numeric") + + update_existing = get_bool_env_var("UPDATE_EXISTING") + return ( organization, repositories_list, @@ -249,4 +255,5 @@ def get_env_vars(test: bool = False) -> tuple[ batch_size, enable_security_updates_bool, exempt_ecosystems_list, + update_existing, ) diff --git a/evergreen.py b/evergreen.py index c94dc15..aaec5aa 100644 --- a/evergreen.py +++ b/evergreen.py @@ -35,6 +35,7 @@ def main(): # pragma: no cover batch_size, enable_security_updates, exempt_ecosystems, + update_existing, ) = env.get_env_vars() # Auth to GitHub.com or GHE @@ -47,7 +48,7 @@ def main(): # pragma: no cover gh_app_id, gh_app_private_key, gh_app_installation_id ) - # If Project ID is set lookup the global project ID + # If Project ID is set, lookup the global project ID if project_id: # Check Organization is set as it is required for linking to a project if not organization: @@ -61,6 +62,7 @@ def main(): # pragma: no cover # Iterate through the repositories and open an issue/PR if dependabot is not enabled count_eligible = 0 + existing_config = None for repo in repos: # if batch_size is defined, ensure we break if we exceed the number of eligible repos if batch_size and count_eligible >= batch_size: @@ -78,19 +80,27 @@ def main(): # pragma: no cover print("Skipping " + repo.full_name + " (visibility-filtered)") continue try: - if repo.file_contents(".github/dependabot.yml").size > 0: - print( - "Skipping " + repo.full_name + " (dependabot file already exists)" - ) - continue + existing_config = repo.file_contents(".github/dependabot.yml") + if existing_config.size > 0: + if not update_existing: + print( + "Skipping " + + repo.full_name + + " (dependabot file already exists)" + ) + continue except github3.exceptions.NotFoundError: pass try: - if repo.file_contents(".github/dependabot.yaml").size > 0: - print( - "Skipping " + repo.full_name + " (dependabot file already exists)" - ) - continue + existing_config = repo.file_contents(".github/dependabot.yaml") + if existing_config.size > 0: + if not update_existing: + print( + "Skipping " + + repo.full_name + + " (dependabot file already exists)" + ) + continue except github3.exceptions.NotFoundError: pass @@ -103,10 +113,11 @@ def main(): # pragma: no cover print("Checking " + repo.full_name + " for compatible package managers") # Try to detect package managers and build a dependabot file dependabot_file = build_dependabot_file( - repo, group_dependencies, exempt_ecosystems + repo, group_dependencies, exempt_ecosystems, existing_config ) + if dependabot_file is None: - print("\tNo compatible package manager found") + print("\tNo (new) compatible package manager found") continue # If dry_run is set, just print the dependabot file diff --git a/requirements.txt b/requirements.txt index 820c2d0..649c7d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ github3.py==4.0.1 +requests==2.31.0 python-dotenv==1.0.1 +PyYAML==6.0.1 diff --git a/test_dependabot_file.py b/test_dependabot_file.py index e90d124..ab2d6b1 100644 --- a/test_dependabot_file.py +++ b/test_dependabot_file.py @@ -19,7 +19,7 @@ def test_not_found_error(self): response.status_code = 404 repo.file_contents.side_effect = github3.exceptions.NotFoundError(resp=response) - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, None) def test_build_dependabot_file_with_bundler(self): @@ -37,7 +37,7 @@ def test_build_dependabot_file_with_bundler(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_npm(self): @@ -55,7 +55,7 @@ def test_build_dependabot_file_with_npm(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_pip(self): @@ -79,7 +79,7 @@ def test_build_dependabot_file_with_pip(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_cargo(self): @@ -100,7 +100,7 @@ def test_build_dependabot_file_with_cargo(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_gomod(self): @@ -116,7 +116,7 @@ def test_build_dependabot_file_with_gomod(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_composer(self): @@ -137,7 +137,7 @@ def test_build_dependabot_file_with_composer(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_hex(self): @@ -158,7 +158,7 @@ def test_build_dependabot_file_with_hex(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_nuget(self): @@ -174,7 +174,7 @@ def test_build_dependabot_file_with_nuget(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_docker(self): @@ -190,7 +190,7 @@ def test_build_dependabot_file_with_docker(self): schedule: interval: 'weekly' """ - result = build_dependabot_file(repo, False, []) + result = build_dependabot_file(repo, False, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_groups(self): @@ -211,7 +211,7 @@ def test_build_dependabot_file_with_groups(self): development-dependencies: dependency-type: 'development' """ - result = build_dependabot_file(repo, True, []) + result = build_dependabot_file(repo, True, [], None) self.assertEqual(result, expected_result) def test_build_dependabot_file_with_exempt_ecosystems(self): @@ -219,7 +219,7 @@ def test_build_dependabot_file_with_exempt_ecosystems(self): repo = MagicMock() repo.file_contents.side_effect = lambda filename: filename == "Dockerfile" - result = build_dependabot_file(repo, False, ["docker"]) + result = build_dependabot_file(repo, False, ["docker"], None) self.assertEqual(result, None) diff --git a/test_env.py b/test_env.py index 548a715..26f1922 100644 --- a/test_env.py +++ b/test_env.py @@ -27,6 +27,7 @@ def setUp(self): "PROJECT_ID", "TITLE", "TYPE", + "UPDATE_EXISTING", ] for key in env_keys: if key in os.environ: @@ -46,6 +47,7 @@ def setUp(self): "PROJECT_ID": "123", "GROUP_DEPENDENCIES": "false", }, + clear=True, ) def test_get_env_vars_with_org(self): """Test that all environment variables are set correctly using an organization""" @@ -70,6 +72,7 @@ def test_get_env_vars_with_org(self): None, # batch_size True, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -114,6 +117,7 @@ def test_get_env_vars_with_repos(self): None, # batch_size True, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -124,6 +128,7 @@ def test_get_env_vars_with_repos(self): "ORGANIZATION": "my_organization", "GH_TOKEN": "my_token", }, + clear=True, ) def test_get_env_vars_optional_values(self): """Test that optional values are set to their default values if not provided""" @@ -150,6 +155,46 @@ def test_get_env_vars_optional_values(self): None, # batch_size True, # enable_security_updates [], # exempt_ecosystems + False, # update_existing + ) + result = get_env_vars(True) + self.assertEqual(result, expected_result) + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "GH_TOKEN": "my_token", + "UPDATE_EXISTING": "true", + }, + clear=True, + ) + def test_get_env_vars_with_update_existing(self): + """Test that optional values are set to their default values if not provided""" + expected_result = ( + "my_organization", + [], + None, + None, + b"", + "my_token", + "", + [], + "pull", + "Enable Dependabot", + "Dependabot could be enabled for this repository. \ +Please enable it by merging this pull request so that \ +we can keep our dependencies up to date and secure.", + "", + False, + "Create dependabot.yaml", + None, + False, + ["internal", "private", "public"], + None, # batch_size + True, # enable_security_updates + [], # exempt_ecosystems + True, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -202,6 +247,7 @@ def test_get_env_vars_auth_with_github_app_installation(self): None, # batch_size True, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -262,6 +308,7 @@ def test_get_env_vars_with_repos_no_dry_run(self): None, # batch_size True, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -300,6 +347,7 @@ def test_get_env_vars_with_repos_disabled_security_updates(self): None, # batch_size False, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -339,6 +387,7 @@ def test_get_env_vars_with_repos_filter_visibility_multiple_values(self): None, # batch_size False, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -378,6 +427,7 @@ def test_get_env_vars_with_repos_filter_visibility_single_value(self): None, # batch_size False, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -447,6 +497,7 @@ def test_get_env_vars_with_repos_filter_visibility_no_duplicates(self): None, # batch_size False, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -487,6 +538,7 @@ def test_get_env_vars_with_repos_exempt_ecosystems(self): None, # batch_size False, # enable_security_updates ["gomod", "docker"], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -526,6 +578,7 @@ def test_get_env_vars_with_no_batch_size(self): None, # batch_size False, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result) @@ -566,6 +619,7 @@ def test_get_env_vars_with_batch_size(self): 5, # batch_size False, # enable_security_updates [], # exempt_ecosystems + False, # update_existing ) result = get_env_vars(True) self.assertEqual(result, expected_result)