Skip to content

Commit

Permalink
chore: add the Maven Central timestamp fetching back
Browse files Browse the repository at this point in the history
Signed-off-by: behnazh-w <[email protected]>
  • Loading branch information
behnazh-w committed Oct 2, 2024
1 parent ab4ef5c commit 7eac146
Show file tree
Hide file tree
Showing 14 changed files with 228 additions and 53 deletions.
23 changes: 10 additions & 13 deletions src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from macaron.slsa_analyzer.checks.base_check import BaseCheck
from macaron.slsa_analyzer.checks.check_result import CheckResultData, CheckResultType, Confidence, JustificationType
from macaron.slsa_analyzer.ci_service.base_ci_service import NoneCIService
from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry
from macaron.slsa_analyzer.registry import registry
from macaron.slsa_analyzer.slsa_req import ReqName

Expand Down Expand Up @@ -119,10 +118,15 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData:

# Look for the artifact in the corresponding registry and find the publish timestamp.
artifact_published_date = None
try:
artifact_published_date = PackageRegistry.find_publish_timestamp(ctx.component.purl)
except InvalidHTTPResponseError as error:
logger.debug(error)
for registry_info in ctx.dynamic_data["package_registries"]:
if registry_info.build_tool.purl_type == ctx.component.type:
try:
artifact_published_date = registry_info.package_registry.find_publish_timestamp(ctx.component.purl)
break
except InvalidHTTPResponseError as error:
logger.debug(error)
except NotImplementedError:
continue

# This check requires the timestamps of published artifact and its source-code commit to proceed.
# If the timestamps are not found, we return with a fail result.
Expand Down Expand Up @@ -304,14 +308,7 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData:
# We should reach here when the analysis has failed to detect any successful deploy step in a
# CI run. In this case the check fails with a medium confidence.
return CheckResultData(
result_tables=[
ArtifactPipelineFacts(
from_provenance=False,
run_deleted=False,
published_before_commit=False,
confidence=Confidence.MEDIUM,
)
],
result_tables=[],
result_type=CheckResultType.FAILED,
)

Expand Down
33 changes: 33 additions & 0 deletions src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import json
import logging
from datetime import datetime
from typing import NamedTuple
from urllib.parse import SplitResult, urlunsplit

Expand Down Expand Up @@ -851,3 +852,35 @@ def download_asset(self, url: str, dest: str) -> bool:
return False

return True

def find_publish_timestamp(self, purl: str, registry_url: str | None = None) -> datetime:
"""Make a search request to Maven Central to find the publishing timestamp of an artifact.
The reason for directly fetching timestamps from Maven Central is that deps.dev occasionally
misses timestamps for Maven artifacts, making it unreliable for this purpose.
To see the search API syntax see: https://central.sonatype.org/search/rest-api-guide/
Parameters
----------
purl: str
The Package URL (purl) of the package whose publication timestamp is to be retrieved.
This should conform to the PURL specification.
registry_url: str | None
The registry URL that can be set for testing.
Returns
-------
datetime
A timezone-aware datetime object representing the publication timestamp
of the specified package.
Raises
------
InvalidHTTPResponseError
If the URL construction fails, the HTTP response is invalid, or if the response
cannot be parsed correctly, or if the expected timestamp is missing or invalid.
NotImplementedError
If not implemented for a registry.
"""
raise NotImplementedError("Fetching timestamps for artifacts on JFrog is not currently supported.")
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,18 @@

import logging
import urllib.parse
from datetime import datetime, timezone

import requests
from packageurl import PackageURL

from macaron.config.defaults import defaults
from macaron.errors import ConfigurationError
from macaron.errors import ConfigurationError, InvalidHTTPResponseError
from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool
from macaron.slsa_analyzer.build_tool.gradle import Gradle
from macaron.slsa_analyzer.build_tool.maven import Maven
from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry
from macaron.util import send_get_http_raw

logger: logging.Logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -127,35 +132,39 @@ def is_detected(self, build_tool: BaseBuildTool) -> bool:
compatible_build_tool_classes = [Maven, Gradle]
return any(isinstance(build_tool, build_tool_class) for build_tool_class in compatible_build_tool_classes)

def find_publish_timestamp(self, group_id: str, artifact_id: str, version: str | None = None) -> datetime:
def find_publish_timestamp(self, purl: str, registry_url: str | None = None) -> datetime:
"""Make a search request to Maven Central to find the publishing timestamp of an artifact.
If version is not provided, the timestamp of the latest version will be returned.
The reason for directly fetching timestamps from Maven Central is that deps.dev occasionally
misses timestamps for Maven artifacts, making it unreliable for this purpose.
To see the search API syntax see: https://central.sonatype.org/search/rest-api-guide/
Parameters
----------
group_id : str
The group id of the artifact.
artifact_id: str
The artifact id of the artifact.
version: str | None
The version of the artifact.
purl: str
The Package URL (purl) of the package whose publication timestamp is to be retrieved.
This should conform to the PURL specification.
registry_url: str | None
The registry URL that can be set for testing.
Returns
-------
datetime
The artifact publish timestamp as a timezone-aware datetime object.
A timezone-aware datetime object representing the publication timestamp
of the specified package.
Raises
------
InvalidHTTPResponseError
If the HTTP response is invalid or unexpected.
If the URL construction fails, the HTTP response is invalid, or if the response
cannot be parsed correctly, or if the expected timestamp is missing or invalid.
"""
query_params = [f"q=g:{group_id}", f"a:{artifact_id}"]
if version:
query_params.append(f"v:{version}")
try:
purl_object = PackageURL.from_string(purl)
except ValueError as error:
logger.debug("Could not parse PURL: %s", error)
query_params = [f"q=g:{purl_object.namespace}", f"a:{purl_object.name}", f"v:{purl_object.version}"]

try:
url = urllib.parse.urlunsplit(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,8 @@ def is_detected(self, build_tool: BaseBuildTool) -> bool:
based on the given build tool.
"""

@staticmethod
def find_publish_timestamp(purl: str, registry_url: str | None = None) -> datetime:
"""Retrieve the publication timestamp for a package specified by its purl from the deps.dev repository.
def find_publish_timestamp(self, purl: str, registry_url: str | None = None) -> datetime:
"""Retrieve the publication timestamp for a package specified by its purl from the deps.dev repository by default.
This method constructs a request URL based on the provided purl, sends an HTTP GET
request to fetch metadata about the package, and extracts the publication timestamp
Expand Down Expand Up @@ -80,6 +79,8 @@ def find_publish_timestamp(purl: str, registry_url: str | None = None) -> dateti
InvalidHTTPResponseError
If the URL construction fails, the HTTP response is invalid, or if the response
cannot be parsed correctly, or if the expected timestamp is missing or invalid.
NotImplementedError
If not implemented for a registry.
"""
# TODO: To reduce redundant calls to deps.dev, store relevant parts of the response
# in the AnalyzeContext object retrieved by the Repo Finder. This step should be
Expand Down
6 changes: 5 additions & 1 deletion tests/integration/cases/semver/policy.dl
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ Policy("test_policy", component_id, "") :-
check_passed(component_id, "mcn_provenance_verified_1"),
provenance_verified_check(_, build_level, _),
build_level = 2,
check_passed(component_id, "mcn_find_artifact_pipeline_1"),
// The build_as_code check is reporting the integration_release.yaml workflow
// which is not the same as the workflow in the provenance. Therefore, the
// mcn_find_artifact_pipeline_1 check fails, which is a false negative.
// TODO: improve the build_as_code check analysis.
check_failed(component_id, "mcn_find_artifact_pipeline_1"),
check_failed(component_id, "mcn_provenance_level_three_1"),
check_failed(component_id, "mcn_provenance_witness_level_one_1"),
check_failed(component_id, "mcn_trusted_builder_level_three_1"),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"version":{"versionKey":{"system":"MAVEN","name":"org.apache.logging.log4j:log4j-core","version":"3.0.0-beta2"},"purl":"pkg:maven/org.apache.logging.log4j/[email protected]","isDefault":true,"isDeprecated":false,"licenses":["Apache-2.0"],"licenseDetails":[{"license":"Apache-2.0","spdx":"Apache-2.0"}],"advisoryKeys":[],"links":[{"label":"SOURCE_REPO","url":"https://github.com/apache/logging-log4j2"},{"label":"ISSUE_TRACKER","url":"https://github.com/apache/logging-log4j2/issues"},{"label":"HOMEPAGE","url":"https://logging.apache.org/log4j/3.x/"}],"slsaProvenances":[],"registries":["https://repo.maven.apache.org/maven2/"],"relatedProjects":[{"projectKey":{"id":"github.com/apache/logging-log4j2"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"SOURCE_REPO"},{"projectKey":{"id":"github.com/apache/logging-log4j2"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"ISSUE_TRACKER"}],"upstreamIdentifiers":[{"packageName":"org.apache.logging.log4j:log4j-core","versionString":"3.0.0-beta2","source":"MAVEN_POM_FILE"}]}}
{"responseHeader":{"status":0,"QTime":4,"params":{"q":"g:org.apache.logging.log4j AND a:log4j-core AND v:3.0.0-beta2","core":"gav","indent":"off","fl":"id,g,a,v,p,ec,timestamp,tags","start":"","sort":"score desc,timestamp desc,g asc,a asc,v desc","rows":"1","wt":"json","version":"2.2"}},"response":{"numFound":1,"start":0,"docs":[]}}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"version":{"versionKey":{"system":"MAVEN","name":"com.fasterxml.jackson.core:jackson-annotations","version":"2.16.1"},"purl":"pkg:maven/com.fasterxml.jackson.core/[email protected]","publishedAt":"2023-12-24T04:02:35Z","isDefault":false,"isDeprecated":false,"licenses":["Apache-2.0"],"licenseDetails":[{"license":"The Apache Software License, Version 2.0","spdx":"Apache-2.0"}],"advisoryKeys":[],"links":[{"label":"SOURCE_REPO","url":"https://github.com/FasterXML/jackson-annotations"},{"label":"HOMEPAGE","url":"https://github.com/FasterXML/jackson"}],"slsaProvenances":[],"registries":["https://repo.maven.apache.org/maven2/"],"relatedProjects":[{"projectKey":{"id":"github.com/fasterxml/jackson-annotations"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"SOURCE_REPO"}],"upstreamIdentifiers":[{"packageName":"com.fasterxml.jackson.core:jackson-annotations","versionString":"2.16.1","source":"MAVEN_POM_FILE"}]}}
{"responseHeader":{"status":0,"QTime":2,"params":{"q":"g:com.fasterxml.jackson.core AND a:jackson-annotations AND v:2.16.1","core":"gav","indent":"off","fl":"id,g,a,v,p,ec,timestamp,tags","start":"","sort":"score desc,timestamp desc,g asc,a asc,v desc","rows":"1","wt":"json","version":"2.2"}},"response":{"numFound":1,"start":0,"docs":[{"id":"com.fasterxml.jackson.core:jackson-annotations:2.16.1","g":"com.fasterxml.jackson.core","a":"jackson-annotations","v":"2.16.1","p":"jar","timestamp":1703390559843,"ec":["-sources.jar",".module",".pom","-javadoc.jar",".jar"],"tags":["core","types","jackson","package","data","annotations","binding","used","value"]}]}}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"version":{"versionKey":{"system":"MAVEN","name":"org.apache.logging.log4j:log4j-core","version":"3.0.0-beta2"},"purl":"pkg:maven/org.apache.logging.log4j/[email protected]","publishedAt":"2024-02-17T18:50:10Z","isDefault":true,"isDeprecated":false,"licenses":["Apache-2.0"],"licenseDetails":[{"license":"Apache-2.0","spdx":"Apache-2.0"}],"advisoryKeys":[],"links":[{"label":"SOURCE_REPO","url":"https://github.com/apache/logging-log4j2"},{"label":"ISSUE_TRACKER","url":"https://github.com/apache/logging-log4j2/issues"},{"label":"HOMEPAGE","url":"https://logging.apache.org/log4j/3.x/"}],"slsaProvenances":[],"registries":["https://repo.maven.apache.org/maven2/"],"relatedProjects":[{"projectKey":{"id":"github.com/apache/logging-log4j2"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"SOURCE_REPO"},{"projectKey":{"id":"github.com/apache/logging-log4j2"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"ISSUE_TRACKER"}],"upstreamIdentifiers":[{"packageName":"org.apache.logging.log4j:log4j-core","versionString":"3.0.0-beta2","source":"MAVEN_POM_FILE"}]}}
{"responseHeader":{"status":0,"QTime":4,"params":{"q":"g:org.apache.logging.log4j AND a:log4j-core AND v:3.0.0-beta2","core":"gav","indent":"off","fl":"id,g,a,v,p,ec,timestamp,tags","start":"","sort":"score desc,timestamp desc,g asc,a asc,v desc","rows":"1","wt":"json","version":"2.2"}},"response":{"numFound":1,"start":0,"docs":[{"id":"org.apache.logging.log4j:log4j-core:3.0.0-beta2","g":"org.apache.logging.log4j","a":"log4j-core","v":"3.0.0-beta2","p":"jar","timestamp":1708195809000,"ec":["-sources.jar","-cyclonedx.xml",".pom",".jar"],"tags":["apache","implementation","log4j"]}]}}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"version":{"versionKey":{"system":"NPM","name":"@sigstore/mock","version":"0.7.5"},"purl":"pkg:npm/%40sigstore/[email protected]","publishedAt":"2024-06-11T23:49:17Z","isDefault":true,"isDeprecated":false,"licenses":["Apache-2.0"],"licenseDetails":[{"license":"Apache-2.0","spdx":"Apache-2.0"}],"advisoryKeys":[],"links":[{"label":"HOMEPAGE","url":"https://github.com/sigstore/sigstore-js/tree/main/packages/mock#readme"},{"label":"ISSUE_TRACKER","url":"https://github.com/sigstore/sigstore-js/issues"},{"label":"ATTESTATION","url":"https://registry.npmjs.org/-/npm/v1/attestations/@sigstore%[email protected]"},{"label":"ORIGIN","url":"https://registry.npmjs.org/@sigstore%2Fmock/0.7.5"},{"label":"SOURCE_REPO","url":"git+https://github.com/sigstore/sigstore-js.git"}],"slsaProvenances":[{"sourceRepository":"https://github.com/sigstore/sigstore-js","commit":"426540e2142edc2aa438e5390b64bdeb3c8f507d","url":"https://registry.npmjs.org/-/npm/v1/attestations/@sigstore%[email protected]","verified":true}],"registries":["https://registry.npmjs.org/"],"relatedProjects":[{"projectKey":{"id":"github.com/sigstore/sigstore-js"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"ISSUE_TRACKER"},{"projectKey":{"id":"github.com/sigstore/sigstore-js"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"SOURCE_REPO"},{"projectKey":{"id":"github.com/sigstore/sigstore-js"},"relationProvenance":"SLSA_ATTESTATION","relationType":"SOURCE_REPO"}],"upstreamIdentifiers":[{"packageName":"@sigstore/mock","versionString":"0.7.5","source":"NPM_NPMJS_ORG"}]}}
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"version":{"versionKey":{"system":"NPM","name":"@sigstore/mock","version":"0.7.5"},"purl":"pkg:npm/%40sigstore/[email protected]","isDefault":true,"isDeprecated":false,"licenses":["Apache-2.0"],"licenseDetails":[{"license":"Apache-2.0","spdx":"Apache-2.0"}],"advisoryKeys":[],"links":[{"label":"HOMEPAGE","url":"https://github.com/sigstore/sigstore-js/tree/main/packages/mock#readme"},{"label":"ISSUE_TRACKER","url":"https://github.com/sigstore/sigstore-js/issues"},{"label":"ATTESTATION","url":"https://registry.npmjs.org/-/npm/v1/attestations/@sigstore%[email protected]"},{"label":"ORIGIN","url":"https://registry.npmjs.org/@sigstore%2Fmock/0.7.5"},{"label":"SOURCE_REPO","url":"git+https://github.com/sigstore/sigstore-js.git"}],"slsaProvenances":[{"sourceRepository":"https://github.com/sigstore/sigstore-js","commit":"426540e2142edc2aa438e5390b64bdeb3c8f507d","url":"https://registry.npmjs.org/-/npm/v1/attestations/@sigstore%[email protected]","verified":true}],"registries":["https://registry.npmjs.org/"],"relatedProjects":[{"projectKey":{"id":"github.com/sigstore/sigstore-js"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"ISSUE_TRACKER"},{"projectKey":{"id":"github.com/sigstore/sigstore-js"},"relationProvenance":"UNVERIFIED_METADATA","relationType":"SOURCE_REPO"},{"projectKey":{"id":"github.com/sigstore/sigstore-js"},"relationProvenance":"SLSA_ATTESTATION","relationType":"SOURCE_REPO"}],"upstreamIdentifiers":[{"packageName":"@sigstore/mock","versionString":"0.7.5","source":"NPM_NPMJS_ORG"}]}}
Loading

0 comments on commit 7eac146

Please sign in to comment.