diff --git a/src/macaron/config/defaults.ini b/src/macaron/config/defaults.ini index abca6969e..42a1d3909 100644 --- a/src/macaron/config/defaults.ini +++ b/src/macaron/config/defaults.ini @@ -392,3 +392,5 @@ hostname = search.maven.org # The search REST API. See https://central.sonatype.org/search/rest-api-guide/ search_endpoint = solrsearch/select request_timeout = 20 +[check.two_person] +required_reviewers = 1 diff --git a/src/macaron/slsa_analyzer/checks/two_person_reviewed_check.py b/src/macaron/slsa_analyzer/checks/two_person_reviewed_check.py new file mode 100644 index 000000000..561c9e44c --- /dev/null +++ b/src/macaron/slsa_analyzer/checks/two_person_reviewed_check.py @@ -0,0 +1,149 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the TwoPersonReviewedCheck class.""" + +import logging +import os + +import requests +from sqlalchemy import ForeignKey +from sqlalchemy.orm import Mapped, mapped_column + +from macaron.config.defaults import defaults +from macaron.database.database_manager import ORMBase +from macaron.database.table_definitions import CheckFacts +from macaron.slsa_analyzer.analyze_context import AnalyzeContext +from macaron.slsa_analyzer.checks.base_check import BaseCheck +from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType +from macaron.slsa_analyzer.registry import registry +from macaron.slsa_analyzer.slsa_req import ReqName + +logger: logging.Logger = logging.getLogger(__name__) + + +class TwoPersonReviewedTable(CheckFacts, ORMBase): + """Check result table for two-person_reviewed.""" + + __tablename__ = "_two_person_reviewed_check" + # The primary key. + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + __mapper_args__ = { + "polymorphic_identity": "_two_person_reviewed_check", + } + + +class TwoPersonReviewedCheck(BaseCheck): + """This Check checks whether the target submitted code has been reviewed by two people.""" + + def __init__(self) -> None: + """Initiate the BuildScriptCheck instance.""" + check_id = "mcn_two_person_reviewed_1" + description = "Check whether the merged pull requests has been reviewd and approved by at least one reviewer." + depends_on: list[tuple[str, CheckResultType]] = [] + eval_reqs = [ReqName.TWO_PERSON_REVIEWED] + super().__init__( + check_id=check_id, + description=description, + depends_on=depends_on, + eval_reqs=eval_reqs, + # result_on_skip=CheckResultType.FAILED, + ) + + def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ + check_result["result_tables"] = [TwoPersonReviewedTable()] + required_reviewers = defaults.get_list("check.two_person", "required_reviewers", fallback=[]) + logger.info("Reviewers number required: %s", {required_reviewers[0]}) + # Your GitHub personal access token (replace with your own token) + token = os.getenv("GITHUB_TOKEN") + # GitHub GraphQL API endpoint + url = "https://api.github.com/graphql" + # Define the GraphQL query + query = """ + query ($owner: String!, $name: String!, $cursor: String, $branch_name: String) { + repository(owner: $owner, name: $name) { + pullRequests(first: 100, states: MERGED, after: $cursor, baseRefName: $branch_name) { + totalCount + pageInfo { + hasNextPage + endCursor + } + edges { + node { + reviewDecision + } + } + } + } + } + """ + # Set up the HTTP headers with your token + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + # Send the GraphQL query to GitHub API. + # 100 data per query. + approved_pr_num = 0 + merged_pr_num = 0 + has_next_page = True + end_cursor = None + + while has_next_page: + variables = { + "owner": ctx.component.repository.owner, + "name": ctx.component.repository.name, + "cursor": end_cursor, + "branch_name": ctx.component.repository.branch_name, + } + response = requests.post( + url, + timeout=defaults.getint("requests", "timeout", fallback=10), + json={"query": query, "variables": variables}, + headers=headers, + ) # nosec B113:request_without_timeout + if response.status_code == 200: + data = response.json() + merged_pr_num = data["data"]["repository"]["pullRequests"]["totalCount"] + for edge in data["data"]["repository"]["pullRequests"]["edges"]: + review_decision = edge["node"]["reviewDecision"] + if review_decision == "APPROVED": + approved_pr_num += 1 + has_next_page = data["data"]["repository"]["pullRequests"]["pageInfo"]["hasNextPage"] + end_cursor = data["data"]["repository"]["pullRequests"]["pageInfo"]["endCursor"] + else: + logger.error("%s, %s", response.status_code, response.text) + + logger.info( + "%d pull requests have been reviewed by at least two person, and the pass rate is %d / %d", + approved_pr_num, + approved_pr_num, + merged_pr_num, + ) + check_result["justification"].extend( + [ + f"{str(approved_pr_num)} pull requests have been reviewed by at least two person.", + f"The pass rate is {str(approved_pr_num)} / {str(merged_pr_num)}", + ] + ) + # print(f"[*] Two-person Reviewed in {duration} seconds") + if approved_pr_num == merged_pr_num: + return CheckResultType.PASSED + return CheckResultType.FAILED + + +registry.register(TwoPersonReviewedCheck()) diff --git a/tests/e2e/expected_results/docker_test/docker_test.json b/tests/e2e/expected_results/docker_test/docker_test.json index 6940d8964..fb1ea3cbb 100644 --- a/tests/e2e/expected_results/docker_test/docker_test.json +++ b/tests/e2e/expected_results/docker_test/docker_test.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:09:42" + "timestamps": "2023-09-18 10:49:07" }, "target": { "info": { @@ -65,7 +65,7 @@ "summary": { "DISABLED": 0, "FAILED": 6, - "PASSED": 4, + "PASSED": 5, "SKIPPED": 0, "UNKNOWN": 0 }, @@ -108,6 +108,18 @@ ], "result_type": "PASSED" }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "0 pull requests have been reviewed by at least two person.", + "The pass rate is 0 / 0" + ], + "result_type": "PASSED" + }, { "check_id": "mcn_version_control_system_1", "check_description": "Check whether the target repo uses a version control system.", @@ -142,7 +154,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -207,19 +219,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -227,23 +243,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/jackson-databind/jackson-databind.json b/tests/e2e/expected_results/jackson-databind/jackson-databind.json index 4d2999187..d50e573b5 100644 --- a/tests/e2e/expected_results/jackson-databind/jackson-databind.json +++ b/tests/e2e/expected_results/jackson-databind/jackson-databind.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:24:16" + "timestamps": "2023-09-18 10:55:51" }, "target": { "info": { @@ -110,7 +110,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 6, + "FAILED": 7, "PASSED": 4, "SKIPPED": 0, "UNKNOWN": 0 @@ -188,7 +188,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -244,6 +244,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "14 pull requests have been reviewed by at least two person.", + "The pass rate is 14 / 62" + ], + "result_type": "FAILED" } ] } @@ -253,19 +265,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -273,23 +289,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/maven/guava.json b/tests/e2e/expected_results/maven/guava.json index a608b9803..2dc016b62 100644 --- a/tests/e2e/expected_results/maven/guava.json +++ b/tests/e2e/expected_results/maven/guava.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:28:04" + "timestamps": "2023-09-18 10:57:34" }, "target": { "info": { @@ -65,7 +65,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 6, + "FAILED": 7, "PASSED": 4, "SKIPPED": 0, "UNKNOWN": 0 @@ -143,7 +143,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -199,6 +199,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "0 pull requests have been reviewed by at least two person.", + "The pass rate is 0 / 370" + ], + "result_type": "FAILED" } ] } @@ -208,19 +220,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -228,23 +244,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/maven/maven.json b/tests/e2e/expected_results/maven/maven.json index d33a87a2f..ea21976d7 100644 --- a/tests/e2e/expected_results/maven/maven.json +++ b/tests/e2e/expected_results/maven/maven.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:28:08" + "timestamps": "2023-09-18 10:57:40" }, "target": { "info": { @@ -110,7 +110,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 6, + "FAILED": 7, "PASSED": 4, "SKIPPED": 0, "UNKNOWN": 0 @@ -183,7 +183,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -239,6 +239,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "283 pull requests have been reviewed by at least two person.", + "The pass rate is 283 / 374" + ], + "result_type": "FAILED" } ] } @@ -248,19 +260,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -268,23 +284,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/maven/mockito.json b/tests/e2e/expected_results/maven/mockito.json index ee4791623..e104c5f0c 100644 --- a/tests/e2e/expected_results/maven/mockito.json +++ b/tests/e2e/expected_results/maven/mockito.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:28:04" + "timestamps": "2023-09-18 10:57:34" }, "target": { "info": { @@ -65,7 +65,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 6, + "FAILED": 7, "PASSED": 4, "SKIPPED": 0, "UNKNOWN": 0 @@ -143,7 +143,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -199,6 +199,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "128 pull requests have been reviewed by at least two person.", + "The pass rate is 128 / 410" + ], + "result_type": "FAILED" } ] } @@ -208,19 +220,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -228,23 +244,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/micronaut-core/caffeine.json b/tests/e2e/expected_results/micronaut-core/caffeine.json index 88ac3fe9a..ed800395a 100644 --- a/tests/e2e/expected_results/micronaut-core/caffeine.json +++ b/tests/e2e/expected_results/micronaut-core/caffeine.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 22:55:15" + "timestamps": "2023-09-18 10:53:52" }, "target": { "info": { @@ -21,7 +21,7 @@ "predicateType": "https://slsa.dev/provenance/v0.2", "predicate": { "builder": { - "id": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/build.yml" + "id": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/release.yml" }, "buildType": "Custom github_actions", "invocation": { @@ -30,14 +30,14 @@ "digest": { "sha1": "05a040c2478341bab8a58a02b3dc1fe14d626d72" }, - "entryPoint": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/build.yml" + "entryPoint": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/release.yml" }, "parameters": {}, "environment": {} }, "buildConfig": { - "jobID": "build", - "stepID": "Publish Snapshot" + "jobID": "release", + "stepID": "Releasing" }, "metadata": { "buildInvocationId": "", @@ -111,7 +111,7 @@ "summary": { "DISABLED": 0, "FAILED": 6, - "PASSED": 4, + "PASSED": 5, "SKIPPED": 0, "UNKNOWN": 0 }, @@ -124,10 +124,10 @@ ], "justification": [ { - "The target repository uses build tool gradle to deploy": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/build.yml", - "The build is triggered by": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/build.yml" + "The target repository uses build tool gradle to deploy": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/release.yml", + "The build is triggered by": "https://github.com/ben-manes/caffeine/blob/05a040c2478341bab8a58a02b3dc1fe14d626d72/.github/workflows/release.yml" }, - "Deploy command: ['./gradlew', 'publishToSonatype']", + "Deploy command: ['./gradlew', 'publishToSonatype', 'closeAndReleaseSonatypeStagingRepository', '-Prelease']", "However, could not find a passing workflow run.", "The target repository does not use maven to deploy." ], @@ -155,6 +155,18 @@ ], "result_type": "PASSED" }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "0 pull requests have been reviewed by at least two person.", + "The pass rate is 0 / 0" + ], + "result_type": "PASSED" + }, { "check_id": "mcn_version_control_system_1", "check_description": "Check whether the target repo uses a version control system.", @@ -189,7 +201,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -254,19 +266,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -274,23 +290,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/micronaut-core/micronaut-core.json b/tests/e2e/expected_results/micronaut-core/micronaut-core.json index 6994537a9..fc5d2163e 100644 --- a/tests/e2e/expected_results/micronaut-core/micronaut-core.json +++ b/tests/e2e/expected_results/micronaut-core/micronaut-core.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-13 10:04:00" + "timestamps": "2023-09-18 10:53:52" }, "target": { "info": { @@ -753,7 +753,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 3, + "FAILED": 4, "PASSED": 6, "SKIPPED": 0, "UNKNOWN": 1 @@ -778,10 +778,10 @@ ], "justification": [ { - "The target repository uses build tool gradle to deploy": "https://github.com/micronaut-projects/micronaut-core/blob/68f9bb0a78fa930865d37fca39252b9ec66e4a43/.github/workflows/gradle.yml", - "The build is triggered by": "https://github.com/micronaut-projects/micronaut-core/blob/68f9bb0a78fa930865d37fca39252b9ec66e4a43/.github/workflows/gradle.yml" + "The target repository uses build tool gradle to deploy": "https://github.com/micronaut-projects/micronaut-core/blob/68f9bb0a78fa930865d37fca39252b9ec66e4a43/.github/workflows/release.yml", + "The build is triggered by": "https://github.com/micronaut-projects/micronaut-core/blob/68f9bb0a78fa930865d37fca39252b9ec66e4a43/.github/workflows/release.yml" }, - "Deploy command: ['./gradlew', 'publishToSonatype', 'docs', '--no-daemon']", + "Deploy command: ['./gradlew', 'publishAllPublicationsToBuildRepository', 'publishToSonatype', 'closeAndReleaseSonatypeStagingRepository']", "However, could not find a passing workflow run." ], "result_type": "PASSED" @@ -889,6 +889,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "88 pull requests have been reviewed by at least two person.", + "The pass rate is 88 / 159" + ], + "result_type": "FAILED" } ] } @@ -898,19 +910,23 @@ "unique_dep_repos": 2, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", - "num_deps_pass": 0 + "check_id": "mcn_build_script_1", + "num_deps_pass": 2 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 2 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -918,24 +934,24 @@ "num_deps_pass": 1 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 2 }, { - "check_id": "mcn_trusted_builder_level_three_1", - "num_deps_pass": 0 + "check_id": "mcn_version_control_system_1", + "num_deps_pass": 2 }, { - "check_id": "mcn_build_script_1", - "num_deps_pass": 2 + "check_id": "mcn_infer_artifact_pipeline_1", + "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", - "num_deps_pass": 2 + "check_id": "mcn_provenance_available_1", + "num_deps_pass": 0 } ], "dep_status": [ diff --git a/tests/e2e/expected_results/micronaut-core/slf4j.json b/tests/e2e/expected_results/micronaut-core/slf4j.json index 1de0884e2..99487d751 100644 --- a/tests/e2e/expected_results/micronaut-core/slf4j.json +++ b/tests/e2e/expected_results/micronaut-core/slf4j.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 22:55:15" + "timestamps": "2023-09-18 10:53:52" }, "target": { "info": { @@ -66,7 +66,7 @@ "summary": { "DISABLED": 0, "FAILED": 7, - "PASSED": 3, + "PASSED": 4, "SKIPPED": 0, "UNKNOWN": 0 }, @@ -93,6 +93,18 @@ ], "result_type": "PASSED" }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "0 pull requests have been reviewed by at least two person.", + "The pass rate is 0 / 0" + ], + "result_type": "PASSED" + }, { "check_id": "mcn_version_control_system_1", "check_description": "Check whether the target repo uses a version control system.", @@ -138,7 +150,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -203,19 +215,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -223,23 +239,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/multibuild_test/multibuild_test.json b/tests/e2e/expected_results/multibuild_test/multibuild_test.json index 4ce642a20..506d84eab 100644 --- a/tests/e2e/expected_results/multibuild_test/multibuild_test.json +++ b/tests/e2e/expected_results/multibuild_test/multibuild_test.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:09:38" + "timestamps": "2023-09-18 10:49:05" }, "target": { "info": { @@ -66,7 +66,7 @@ "summary": { "DISABLED": 0, "FAILED": 6, - "PASSED": 4, + "PASSED": 5, "SKIPPED": 0, "UNKNOWN": 0 }, @@ -110,6 +110,18 @@ ], "result_type": "PASSED" }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "0 pull requests have been reviewed by at least two person.", + "The pass rate is 0 / 0" + ], + "result_type": "PASSED" + }, { "check_id": "mcn_version_control_system_1", "check_description": "Check whether the target repo uses a version control system.", @@ -144,7 +156,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -209,19 +221,23 @@ "unique_dep_repos": 2, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", - "num_deps_pass": 0 + "check_id": "mcn_build_script_1", + "num_deps_pass": 2 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 1 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -229,24 +245,24 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 2 }, { - "check_id": "mcn_trusted_builder_level_three_1", - "num_deps_pass": 0 + "check_id": "mcn_version_control_system_1", + "num_deps_pass": 2 }, { - "check_id": "mcn_build_script_1", - "num_deps_pass": 2 + "check_id": "mcn_infer_artifact_pipeline_1", + "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", - "num_deps_pass": 2 + "check_id": "mcn_provenance_available_1", + "num_deps_pass": 0 } ], "dep_status": [ diff --git a/tests/e2e/expected_results/plot-plugin/plot-plugin.json b/tests/e2e/expected_results/plot-plugin/plot-plugin.json index faa5f2e95..227c363fb 100644 --- a/tests/e2e/expected_results/plot-plugin/plot-plugin.json +++ b/tests/e2e/expected_results/plot-plugin/plot-plugin.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:07:15" + "timestamps": "2023-09-18 10:46:56" }, "target": { "info": { @@ -110,7 +110,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 7, + "FAILED": 8, "PASSED": 3, "SKIPPED": 0, "UNKNOWN": 0 @@ -183,7 +183,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -239,6 +239,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "53 pull requests have been reviewed by at least two person.", + "The pass rate is 53 / 107" + ], + "result_type": "FAILED" } ] } @@ -248,19 +260,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -268,23 +284,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/purl/com_google_guava/guava/guava.json b/tests/e2e/expected_results/purl/com_google_guava/guava/guava.json index d28951558..6d57d0a1c 100644 --- a/tests/e2e/expected_results/purl/com_google_guava/guava/guava.json +++ b/tests/e2e/expected_results/purl/com_google_guava/guava/guava.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 16:52:10" + "timestamps": "2023-09-18 10:55:59" }, "target": { "info": { @@ -65,7 +65,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 5, + "FAILED": 6, "PASSED": 5, "SKIPPED": 0, "UNKNOWN": 0 @@ -145,7 +145,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -201,6 +201,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "0 pull requests have been reviewed by at least two person.", + "The pass rate is 0 / 370" + ], + "result_type": "FAILED" } ] } @@ -210,19 +222,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -230,23 +246,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/purl/maven/maven.json b/tests/e2e/expected_results/purl/maven/maven.json index 833cb74bb..2a27d1b9d 100644 --- a/tests/e2e/expected_results/purl/maven/maven.json +++ b/tests/e2e/expected_results/purl/maven/maven.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:10:37" + "timestamps": "2023-09-18 10:49:14" }, "target": { "info": { @@ -110,7 +110,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 6, + "FAILED": 7, "PASSED": 4, "SKIPPED": 0, "UNKNOWN": 0 @@ -183,7 +183,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -239,6 +239,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "283 pull requests have been reviewed by at least two person.", + "The pass rate is 283 / 374" + ], + "result_type": "FAILED" } ] } @@ -248,19 +260,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -268,23 +284,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json b/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json index d32078220..5d7d6cf8f 100644 --- a/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json +++ b/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:36:00" + "timestamps": "2023-09-18 10:58:38" }, "target": { "info": { @@ -1683,7 +1683,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 2, + "FAILED": 3, "PASSED": 8, "SKIPPED": 0, "UNKNOWN": 0 @@ -1822,6 +1822,18 @@ "Failed to discover any witness provenance." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "359 pull requests have been reviewed by at least two person.", + "The pass rate is 359 / 371" + ], + "result_type": "FAILED" } ] } @@ -1831,19 +1843,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -1851,23 +1867,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.json b/tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.json index bbe1ada0c..4fa137923 100644 --- a/tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.json +++ b/tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:07:11" + "timestamps": "2023-09-18 10:46:53" }, "target": { "info": { @@ -66,7 +66,7 @@ "summary": { "DISABLED": 0, "FAILED": 8, - "PASSED": 2, + "PASSED": 3, "SKIPPED": 0, "UNKNOWN": 0 }, @@ -83,6 +83,18 @@ ], "result_type": "PASSED" }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "0 pull requests have been reviewed by at least two person.", + "The pass rate is 0 / 0" + ], + "result_type": "PASSED" + }, { "check_id": "mcn_version_control_system_1", "check_description": "Check whether the target repo uses a version control system.", @@ -142,7 +154,7 @@ "Provenance content - Identifies builder - SLSA Level 1" ], "justification": [ - "Could not find any SLSA provenances." + "Could not find any SLSA or Witness provenances." ], "result_type": "FAILED" }, @@ -207,19 +219,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -227,23 +243,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/urllib3/urllib3.json b/tests/e2e/expected_results/urllib3/urllib3.json index 1a24d23ef..ca0492d17 100644 --- a/tests/e2e/expected_results/urllib3/urllib3.json +++ b/tests/e2e/expected_results/urllib3/urllib3.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 20:10:10" + "timestamps": "2023-09-22 16:05:00" }, "target": { "info": { @@ -20,51 +20,51 @@ "predicateType": "https://slsa.dev/provenance/v0.2", "subject": [ { - "name": "urllib3-2.0.4-py3-none-any.whl", + "name": "urllib3-2.0.5-py3-none-any.whl", "digest": { - "sha256": "de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4" + "sha256": "ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e" } }, { - "name": "urllib3-2.0.4.tar.gz", + "name": "urllib3-2.0.5.tar.gz", "digest": { - "sha256": "8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11" + "sha256": "13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594" } } ], "predicate": { "builder": { - "id": "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.7.0" + "id": "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.9.0" }, "buildType": "https://github.com/slsa-framework/slsa-github-generator/generic@v1", "invocation": { "configSource": { - "uri": "git+https://github.com/urllib3/urllib3@refs/tags/2.0.4", + "uri": "git+https://github.com/urllib3/urllib3@refs/tags/v2.0.5", "digest": { - "sha1": "c9fa144545eedb5dc4a2cc3f255e95602a1d7db0" + "sha1": "d9f85a749488188c286cd50606d159874db94d5f" }, "entryPoint": ".github/workflows/publish.yml" }, "parameters": {}, "environment": { - "github_actor": "illia-v", - "github_actor_id": "17710133", + "github_actor": "pquentin", + "github_actor_id": "42327", "github_base_ref": "", "github_event_name": "push", "github_event_payload": { - "after": "d267c99f7e890ff22e136c34d29be802d9c2e773", + "after": "d4e424a471c79f245740a3734567e7402c036620", "base_ref": null, "before": "0000000000000000000000000000000000000000", "commits": [], - "compare": "https://github.com/urllib3/urllib3/compare/2.0.4", + "compare": "https://github.com/urllib3/urllib3/compare/v2.0.5", "created": true, "deleted": false, "forced": false, "head_commit": { "author": { - "email": "64815328+Eutropios@users.noreply.github.com", - "name": "Noah Jenner", - "username": "Eutropios" + "email": "quentin.pradet@gmail.com", + "name": "Quentin Pradet", + "username": "pquentin" }, "committer": { "email": "noreply@github.com", @@ -72,11 +72,11 @@ "username": "web-flow" }, "distinct": true, - "id": "c9fa144545eedb5dc4a2cc3f255e95602a1d7db0", - "message": "Release version 2.0.4 (#3084)\n\nCo-authored-by: Illia Volochii ", - "timestamp": "2023-07-19T17:46:02+03:00", - "tree_id": "e61f50347e7bb803a0c8942ba63fe917c8424f77", - "url": "https://github.com/urllib3/urllib3/commit/c9fa144545eedb5dc4a2cc3f255e95602a1d7db0" + "id": "d9f85a749488188c286cd50606d159874db94d5f", + "message": "Release 2.0.5", + "timestamp": "2023-09-20T08:59:31+02:00", + "tree_id": "842632ede50bc641555c90779642ab521e0c1401", + "url": "https://github.com/urllib3/urllib3/commit/d9f85a749488188c286cd50606d159874db94d5f" }, "organization": { "avatar_url": "https://avatars.githubusercontent.com/u/26825299?v=4", @@ -93,10 +93,10 @@ "url": "https://api.github.com/orgs/urllib3" }, "pusher": { - "email": "illia.volochii@gmail.com", - "name": "illia-v" + "email": "quentin.pradet@gmail.com", + "name": "pquentin" }, - "ref": "refs/tags/2.0.4", + "ref": "refs/tags/v2.0.5", "repository": { "allow_forking": true, "archive_url": "https://api.github.com/repos/urllib3/urllib3/{archive_format}{/ref}", @@ -119,8 +119,8 @@ "downloads_url": "https://api.github.com/repos/urllib3/urllib3/downloads", "events_url": "https://api.github.com/repos/urllib3/urllib3/events", "fork": false, - "forks": 1078, - "forks_count": 1078, + "forks": 1092, + "forks_count": 1092, "forks_url": "https://api.github.com/repos/urllib3/urllib3/forks", "full_name": "urllib3/urllib3", "git_commits_url": "https://api.github.com/repos/urllib3/urllib3/git/commits{/sha}", @@ -159,8 +159,8 @@ "name": "urllib3", "node_id": "MDEwOlJlcG9zaXRvcnkyNDEwNjc2", "notifications_url": "https://api.github.com/repos/urllib3/urllib3/notifications{?since,all,participating}", - "open_issues": 125, - "open_issues_count": 125, + "open_issues": 130, + "open_issues_count": 130, "organization": "urllib3", "owner": { "avatar_url": "https://avatars.githubusercontent.com/u/26825299?v=4", @@ -186,12 +186,12 @@ }, "private": false, "pulls_url": "https://api.github.com/repos/urllib3/urllib3/pulls{/number}", - "pushed_at": 1689779927, + "pushed_at": 1695193235, "releases_url": "https://api.github.com/repos/urllib3/urllib3/releases{/id}", - "size": 7242, + "size": 6839, "ssh_url": "git@github.com:urllib3/urllib3.git", - "stargazers": 3452, - "stargazers_count": 3452, + "stargazers": 3485, + "stargazers_count": 3485, "stargazers_url": "https://api.github.com/repos/urllib3/urllib3/stargazers", "statuses_url": "https://api.github.com/repos/urllib3/urllib3/statuses/{sha}", "subscribers_url": "https://api.github.com/repos/urllib3/urllib3/subscribers", @@ -206,48 +206,48 @@ "urllib3" ], "trees_url": "https://api.github.com/repos/urllib3/urllib3/git/trees{/sha}", - "updated_at": "2023-07-19T02:19:14Z", + "updated_at": "2023-09-19T20:11:25Z", "url": "https://github.com/urllib3/urllib3", "visibility": "public", - "watchers": 3452, - "watchers_count": 3452, + "watchers": 3485, + "watchers_count": 3485, "web_commit_signoff_required": false }, "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/17710133?v=4", - "events_url": "https://api.github.com/users/illia-v/events{/privacy}", - "followers_url": "https://api.github.com/users/illia-v/followers", - "following_url": "https://api.github.com/users/illia-v/following{/other_user}", - "gists_url": "https://api.github.com/users/illia-v/gists{/gist_id}", + "avatar_url": "https://avatars.githubusercontent.com/u/42327?v=4", + "events_url": "https://api.github.com/users/pquentin/events{/privacy}", + "followers_url": "https://api.github.com/users/pquentin/followers", + "following_url": "https://api.github.com/users/pquentin/following{/other_user}", + "gists_url": "https://api.github.com/users/pquentin/gists{/gist_id}", "gravatar_id": "", - "html_url": "https://github.com/illia-v", - "id": 17710133, - "login": "illia-v", - "node_id": "MDQ6VXNlcjE3NzEwMTMz", - "organizations_url": "https://api.github.com/users/illia-v/orgs", - "received_events_url": "https://api.github.com/users/illia-v/received_events", - "repos_url": "https://api.github.com/users/illia-v/repos", + "html_url": "https://github.com/pquentin", + "id": 42327, + "login": "pquentin", + "node_id": "MDQ6VXNlcjQyMzI3", + "organizations_url": "https://api.github.com/users/pquentin/orgs", + "received_events_url": "https://api.github.com/users/pquentin/received_events", + "repos_url": "https://api.github.com/users/pquentin/repos", "site_admin": false, - "starred_url": "https://api.github.com/users/illia-v/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/illia-v/subscriptions", + "starred_url": "https://api.github.com/users/pquentin/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/pquentin/subscriptions", "type": "User", - "url": "https://api.github.com/users/illia-v" + "url": "https://api.github.com/users/pquentin" } }, "github_head_ref": "", - "github_ref": "refs/tags/2.0.4", + "github_ref": "refs/tags/v2.0.5", "github_ref_type": "tag", "github_repository_id": "2410676", "github_repository_owner": "urllib3", "github_repository_owner_id": "26825299", "github_run_attempt": "1", - "github_run_id": "5600993171", - "github_run_number": "21", - "github_sha1": "c9fa144545eedb5dc4a2cc3f255e95602a1d7db0" + "github_run_id": "6245105149", + "github_run_number": "22", + "github_sha1": "d9f85a749488188c286cd50606d159874db94d5f" } }, "metadata": { - "buildInvocationID": "5600993171-1", + "buildInvocationID": "6245105149-1", "completeness": { "parameters": true, "environment": false, @@ -257,9 +257,9 @@ }, "materials": [ { - "uri": "git+https://github.com/urllib3/urllib3@refs/tags/2.0.4", + "uri": "git+https://github.com/urllib3/urllib3@refs/tags/v2.0.5", "digest": { - "sha1": "c9fa144545eedb5dc4a2cc3f255e95602a1d7db0" + "sha1": "d9f85a749488188c286cd50606d159874db94d5f" } } ] @@ -271,7 +271,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 3, + "FAILED": 4, "PASSED": 7, "SKIPPED": 0, "UNKNOWN": 0 @@ -352,7 +352,7 @@ ], "justification": [ "Successfully verified level 3: ", - "verify passed : urllib3-2.0.4-py3-none-any.whl,verify passed : urllib3-2.0.4.tar.gz" + "verify passed : urllib3-2.0.5-py3-none-any.whl,verify passed : urllib3-2.0.5.tar.gz" ], "result_type": "PASSED" }, @@ -407,6 +407,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "425 pull requests have been reviewed by at least two person.", + "The pass rate is 425 / 475" + ], + "result_type": "FAILED" } ] } @@ -416,19 +428,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -436,23 +452,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/e2e/expected_results/urllib3/urllib3_cue_invalid.json b/tests/e2e/expected_results/urllib3/urllib3_cue_invalid.json index b41112913..f19c44364 100644 --- a/tests/e2e/expected_results/urllib3/urllib3_cue_invalid.json +++ b/tests/e2e/expected_results/urllib3/urllib3_cue_invalid.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-09-12 17:36:12" + "timestamps": "2023-09-18 10:59:04" }, "target": { "info": { @@ -271,7 +271,7 @@ "checks": { "summary": { "DISABLED": 0, - "FAILED": 3, + "FAILED": 4, "PASSED": 6, "SKIPPED": 0, "UNKNOWN": 1 @@ -407,6 +407,18 @@ "Could not find a trusted level 3 builder as a GitHub Actions workflow." ], "result_type": "FAILED" + }, + { + "check_id": "mcn_two_person_reviewed_1", + "check_description": "Check whether the merged pull requests has been reviewd and approved by at least one reviewer.", + "slsa_requirements": [ + "Two-person reviewed - SLSA Level 4" + ], + "justification": [ + "422 pull requests have been reviewed by at least two person.", + "The pass rate is 422 / 472" + ], + "result_type": "FAILED" } ] } @@ -416,19 +428,23 @@ "unique_dep_repos": 0, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", + "check_id": "mcn_provenance_witness_level_one_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_witness_level_one_1", + "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_available_1", + "check_id": "mcn_build_script_1", "num_deps_pass": 0 }, { - "check_id": "mcn_infer_artifact_pipeline_1", + "check_id": "mcn_two_person_reviewed_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", "num_deps_pass": 0 }, { @@ -436,23 +452,23 @@ "num_deps_pass": 0 }, { - "check_id": "mcn_version_control_system_1", + "check_id": "mcn_build_service_1", "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_version_control_system_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", + "check_id": "mcn_infer_artifact_pipeline_1", "num_deps_pass": 0 }, { - "check_id": "mcn_provenance_level_three_1", + "check_id": "mcn_trusted_builder_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 } ], diff --git a/tests/policy_engine/expected_results/policy_report.json b/tests/policy_engine/expected_results/policy_report.json index 5108356cc..1e2e7b395 100644 --- a/tests/policy_engine/expected_results/policy_report.json +++ b/tests/policy_engine/expected_results/policy_report.json @@ -1,16 +1,16 @@ { - "passed_policies": [ + "component_satisfies_policy": [ [ + "101", + "pkg:github.com/slsa-framework/slsa-verifier@fc50b662fcfeeeb0e97243554b47d9b20b14efac", "auth-provenance" ] ], - "failed_policies": [], - "component_violates_policy": [], - "component_satisfies_policy": [ + "passed_policies": [ [ - "121", - "pkg:github.com/slsa-framework/slsa-verifier@fc50b662fcfeeeb0e97243554b47d9b20b14efac", "auth-provenance" ] - ] + ], + "component_violates_policy": [], + "failed_policies": [] } diff --git a/tests/slsa_analyzer/checks/test_two_person_reviewed_check.py b/tests/slsa_analyzer/checks/test_two_person_reviewed_check.py new file mode 100644 index 000000000..c6df4c397 --- /dev/null +++ b/tests/slsa_analyzer/checks/test_two_person_reviewed_check.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the tests for the TwoPersonReviewed Check.""" +import logging +import os + +import requests + +from macaron.config.defaults import defaults + +logger: logging.Logger = logging.getLogger(__name__) + + +class TestTwoPersonReviewedCheck: + """ + Provide three test cases here + """ + + def test_two_person_reviewed_check(self) -> None: + """This is a function check two-person reviewed.""" + # Change request merged pull request + assert self.check_a_review("micronaut-projects", "micronaut-core", 593) == "CHANGES_REQUESTED" + # Approved merged pull request + assert self.check_a_review("micronaut-projects", "micronaut-core", 9875) == "APPROVED" + + def check_a_review(self, owner: str, name: str, pr_number: int) -> str: + """ + Implement the function to fetch a review and checks for two-person review completion. + + Parameters + ---------- + owner (String): The name of the owner. + name (String): The name of the repo. + pr_number (String): Identify which pull request. + + Returns + ------- + CheckResultType: Result of the test. + """ + # Your GitHub personal access token (replace with your own token) + token = os.getenv("GITHUB_TOKEN") + # GitHub GraphQL API endpoint + url = "https://api.github.com/graphql" + # Define the GraphQL query + query = """ + query ($owner: String!, $name: String!, $number: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviewDecision + } + } + } + """ + # Set up the HTTP headers with your token + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + # Send the GraphQL query to GitHub API. + variables = { + "owner": owner, + "name": name, + "number": pr_number, + } + response = requests.post( + url, + timeout=defaults.getint("requests", "timeout", fallback=10), + json={"query": query, "variables": variables}, + headers=headers, + ) # nosec B113:request_without_timeout + review_decision = "" + if response.status_code == 200: + data = response.json() + review_decision = data["data"]["repository"]["pullRequest"]["reviewDecision"] + else: + logger.error("%s, %s", response.status_code, response.text) + + return review_decision