From d684c2932aabb4a3007b3b37657404e15966bf5b Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Mon, 11 Sep 2023 16:59:51 +1000 Subject: [PATCH] fix: check if repository is available in provenance available check Signed-off-by: behnazh-w --- .../checks/provenance_available_check.py | 8 + .../checks/test_provenance_available_check.py | 153 +++++++++++------- 2 files changed, 104 insertions(+), 57 deletions(-) diff --git a/src/macaron/slsa_analyzer/checks/provenance_available_check.py b/src/macaron/slsa_analyzer/checks/provenance_available_check.py index 050f7f589..e3b8c6048 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_available_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_available_check.py @@ -450,6 +450,14 @@ def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResu # We look for the provenances in the package registries first, then CI services. # (Note the short-circuit evaluation with OR.) + # The current provenance discovery mechanism for package registries requires a + # repository to be available. Moreover, the repository path in Witness provenance + # contents are checked to match the target repository path. + # TODO: handle cases where a PURL string is provided for a software component but + # no repository is available. + if not ctx.component.repository: + check_result["justification"] = ["Unable to find provenances because no repository is available."] + return CheckResultType.FAILED try: provenance_assets = self.find_provenance_assets_on_package_registries( repo_fs_path=ctx.component.repository.fs_path, diff --git a/tests/slsa_analyzer/checks/test_provenance_available_check.py b/tests/slsa_analyzer/checks/test_provenance_available_check.py index 2dc182d14..d4a987c15 100644 --- a/tests/slsa_analyzer/checks/test_provenance_available_check.py +++ b/tests/slsa_analyzer/checks/test_provenance_available_check.py @@ -4,7 +4,12 @@ """This modules contains tests for the provenance available check.""" +from pathlib import Path + +import pytest + from macaron.code_analyzer.call_graph import BaseNode, CallGraph +from macaron.database.table_definitions import Repository from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType from macaron.slsa_analyzer.checks.provenance_available_check import ProvenanceAvailableCheck from macaron.slsa_analyzer.ci_service.circleci import CircleCI @@ -16,8 +21,6 @@ from macaron.slsa_analyzer.specs.ci_spec import CIInfo from tests.conftest import MockAnalyzeContext -from ...macaron_testcase import MacaronTestCase - class MockGitHubActions(GitHubActions): """Mock the GitHubActions class.""" @@ -47,58 +50,94 @@ def download_asset(self, url: str, download_path: str) -> bool: return False -class TestProvAvailableCheck(MacaronTestCase): - """Test the provenance available check.""" - - def test_provenance_available_check(self) -> None: - """Test the provenance available check.""" - check = ProvenanceAvailableCheck() - check_result = CheckResult(justification=[]) # type: ignore - github_actions = MockGitHubActions() - api_client = MockGhAPIClient({"headers": {}, "query": []}) - github_actions.api_client = api_client - github_actions.load_defaults() - jenkins = Jenkins() - jenkins.load_defaults() - travis = Travis() - travis.load_defaults() - circle_ci = CircleCI() - circle_ci.load_defaults() - gitlab_ci = GitLabCI() - gitlab_ci.load_defaults() - - ci_info = CIInfo( - service=github_actions, - bash_commands=[], - callgraph=CallGraph(BaseNode(), ""), - provenance_assets=[], - latest_release={}, - provenances=[], - ) - - # Repo has provenances. - ctx = MockAnalyzeContext(macaron_path=MacaronTestCase.macaron_path, output_dir="") - ctx.dynamic_data["ci_services"] = [ci_info] - assert check.run_check(ctx, check_result) == CheckResultType.PASSED - - # Repo doesn't have a provenance. - api_client.release = {"assets": [{"name": "attestation.intoto", "url": "URL", "size": 10}]} - assert check.run_check(ctx, check_result) == CheckResultType.FAILED - - api_client.release = {"assets": [{"name": "attestation.intoto.jsonl", "url": "URL", "size": 10}]} - - # Test Jenkins. - ci_info["service"] = jenkins - assert check.run_check(ctx, check_result) == CheckResultType.FAILED - - # Test Travis. - ci_info["service"] = travis - assert check.run_check(ctx, check_result) == CheckResultType.FAILED - - # Test Circle CI. - ci_info["service"] = circle_ci - assert check.run_check(ctx, check_result) == CheckResultType.FAILED - - # Test GitLab CI. - ci_info["service"] = gitlab_ci - assert check.run_check(ctx, check_result) == CheckResultType.FAILED +@pytest.mark.parametrize( + ("repository", "expected"), + [ + (None, CheckResultType.FAILED), + (Repository(complete_name="github.com/package-url/purl-spec", fs_path=""), CheckResultType.PASSED), + ], +) +def test_provenance_available_check_with_repos(macaron_path: Path, repository: Repository, expected: str) -> None: + """Test the provenance available check on different types of repositories.""" + check = ProvenanceAvailableCheck() + check_result = CheckResult(justification=[]) # type: ignore + github_actions = MockGitHubActions() + api_client = MockGhAPIClient({"headers": {}, "query": []}) + github_actions.api_client = api_client + github_actions.load_defaults() + jenkins = Jenkins() + jenkins.load_defaults() + travis = Travis() + travis.load_defaults() + circle_ci = CircleCI() + circle_ci.load_defaults() + gitlab_ci = GitLabCI() + gitlab_ci.load_defaults() + + ci_info = CIInfo( + service=github_actions, + bash_commands=[], + callgraph=CallGraph(BaseNode(), ""), + provenance_assets=[], + latest_release={}, + provenances=[], + ) + + # Set up the context object with provenances. + ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") + ctx.component.repository = repository + ctx.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(ctx, check_result) == expected + + +def test_provenance_available_check_on_ci(macaron_path: Path) -> None: + """Test the provenance available check on different types of CI services.""" + check = ProvenanceAvailableCheck() + check_result = CheckResult(justification=[]) # type: ignore + github_actions = MockGitHubActions() + api_client = MockGhAPIClient({"headers": {}, "query": []}) + github_actions.api_client = api_client + github_actions.load_defaults() + jenkins = Jenkins() + jenkins.load_defaults() + travis = Travis() + travis.load_defaults() + circle_ci = CircleCI() + circle_ci.load_defaults() + gitlab_ci = GitLabCI() + gitlab_ci.load_defaults() + + ci_info = CIInfo( + service=github_actions, + bash_commands=[], + callgraph=CallGraph(BaseNode(), ""), + provenance_assets=[], + latest_release={}, + provenances=[], + ) + + # Set up the context object with provenances. + ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") + ctx.dynamic_data["ci_services"] = [ci_info] + + # Repo doesn't have a provenance. + api_client.release = {"assets": [{"name": "attestation.intoto", "url": "URL", "size": 10}]} + assert check.run_check(ctx, check_result) == CheckResultType.FAILED + + api_client.release = {"assets": [{"name": "attestation.intoto.jsonl", "url": "URL", "size": 10}]} + + # Test Jenkins. + ci_info["service"] = jenkins + assert check.run_check(ctx, check_result) == CheckResultType.FAILED + + # Test Travis. + ci_info["service"] = travis + assert check.run_check(ctx, check_result) == CheckResultType.FAILED + + # Test Circle CI. + ci_info["service"] = circle_ci + assert check.run_check(ctx, check_result) == CheckResultType.FAILED + + # Test GitLab CI. + ci_info["service"] = gitlab_ci + assert check.run_check(ctx, check_result) == CheckResultType.FAILED