Skip to content

Commit fbacec6

Browse files
sfoslundsfoslund
authored
feat(spdx): add default-off SPDX 3.0 document detection (#1903)
* feat(spdx): add default-off SPDX 3 document detector Detect SPDX 3.0.0 and 3.0.1 compact JSON-LD documents as SBOM references, including root elements and creator metadata. Register the detector and add focused tests and a verification fixture. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 * fix(spdx): restrict SPDX 3 discovery to *.spdx.json Update detector tests and rename the verification fixture to match the sole supported search pattern. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 * refactor(spdx): scope the new detector to SPDX 3.0 Rename the detector, its ID, tests, and verification fixture to reserve distinct identities for future SPDX minor versions. Continue accepting SPDX 3.0.0 and 3.0.1 only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 * refactor(spdx): share common detector file processing Extract shared SPDX metadata, SHA1 hashing, JSON loading, and component registration into SpdxComponentDetector. Keep format-specific parsing and exception handling in the concrete detectors, with regression coverage for their differing error policies. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 * Revert "refactor(spdx): share common detector file processing" This reverts commit b7caee8, restoring independent SPDX detector implementations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 * fix(spdx): handle version mismatches and oversized documents Log detector version mismatches at debug level and accept stable SPDX 3.0 patch versions. Warn and skip SPDX 3.0 files that exhaust memory or overflow size limits, without stopping subsequent files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e4f54f6-111a-4ceb-85fb-5b08c49779af * refactor(spdx): replace version regex with explicit validation Preserve stable 3.0.x matching, including multi-digit patches, while rejecting leading zeros and non-ASCII digits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e4f54f6-111a-4ceb-85fb-5b08c49779af --------- Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 Copilot-Session: 3e4f54f6-111a-4ceb-85fb-5b08c49779af
1 parent 5f2f136 commit fbacec6

6 files changed

Lines changed: 582 additions & 1 deletion

File tree

‎src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
6767
}
6868
else
6969
{
70-
this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location);
70+
this.Logger.LogDebug("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location);
7171
}
7272
}
7373
catch (JsonException)

‎src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
namespace Microsoft.ComponentDetection.Detectors.Spdx;
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using System.Linq;
7+
using System.Security.Cryptography;
8+
using System.Text.Json;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
using Microsoft.ComponentDetection.Contracts;
12+
using Microsoft.ComponentDetection.Contracts.Internal;
13+
using Microsoft.ComponentDetection.Contracts.TypedComponent;
14+
using Microsoft.Extensions.Logging;
15+
16+
public class Spdx30ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector
17+
{
18+
public Spdx30ComponentDetector(
19+
IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
20+
IObservableDirectoryWalkerFactory walkerFactory,
21+
ILogger<Spdx30ComponentDetector> logger)
22+
{
23+
this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
24+
this.Scanner = walkerFactory;
25+
this.Logger = logger;
26+
}
27+
28+
public override string Id => "SPDX30SBOM";
29+
30+
public override IEnumerable<string> Categories => [nameof(DetectorClass.Spdx)];
31+
32+
public override IEnumerable<ComponentType> SupportedComponentTypes => [ComponentType.Spdx];
33+
34+
public override int Version => 1;
35+
36+
public override IList<string> SearchPatterns => ["*.spdx.json"];
37+
38+
protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
39+
{
40+
var file = processRequest.ComponentStream;
41+
this.Logger.LogDebug("Discovered SPDX 3.0 manifest file at: {ManifestLocation}", file.Location);
42+
43+
try
44+
{
45+
// Keep the SHA1 document fingerprint used by SpdxComponent and existing SBOM reference consumers.
46+
#pragma warning disable CA5350, CA1308 // Existing SPDX document references require lowercase SHA1 checksums.
47+
var checksum = Convert.ToHexString(await SHA1.HashDataAsync(file.Stream, cancellationToken)).ToLowerInvariant();
48+
#pragma warning restore CA5350, CA1308
49+
file.Stream.Seek(0, SeekOrigin.Begin);
50+
51+
using var json = await JsonDocument.ParseAsync(file.Stream, cancellationToken: cancellationToken);
52+
var component = this.ReadDocument(json.RootElement, checksum, file.Location);
53+
if (component != null)
54+
{
55+
processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(component));
56+
}
57+
}
58+
catch (JsonException e)
59+
{
60+
this.Logger.LogWarning(e, "Unable to parse SPDX 3.0 file at {ManifestLocation}, skipping", file.Location);
61+
}
62+
catch (IOException e)
63+
{
64+
this.Logger.LogError(e, "Error while reading SPDX 3.0 file at {ManifestLocation}", file.Location);
65+
}
66+
catch (Exception e) when (e is OutOfMemoryException or OverflowException)
67+
{
68+
this.Logger.LogWarning(e, "SPDX 3.0 file at {ManifestLocation} exceeds available memory or supported size, skipping", file.Location);
69+
}
70+
}
71+
72+
private static bool IsSupportedVersion(string version)
73+
{
74+
const string prefix = "3.0.";
75+
if (!version.StartsWith(prefix, StringComparison.Ordinal) || version.Length == prefix.Length)
76+
{
77+
return false;
78+
}
79+
80+
var patchVersion = version[prefix.Length..];
81+
return (patchVersion.Length == 1 || patchVersion[0] != '0')
82+
&& patchVersion.All(character => character is >= '0' and <= '9');
83+
}
84+
85+
private static string? ReadString(JsonElement element, string property)
86+
{
87+
if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value))
88+
{
89+
return null;
90+
}
91+
92+
if (value.ValueKind != JsonValueKind.String)
93+
{
94+
throw new JsonException($"Expected a string for {property}.");
95+
}
96+
97+
return value.GetString();
98+
}
99+
100+
private static bool HasType(JsonElement element, string type)
101+
{
102+
if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty("type", out var value))
103+
{
104+
return false;
105+
}
106+
107+
return value.ValueKind == JsonValueKind.String
108+
? value.GetString() == type
109+
: value.ValueKind == JsonValueKind.Array && value.EnumerateArray().Any(item => item.ValueKind == JsonValueKind.String && item.GetString() == type);
110+
}
111+
112+
private static string? ReadId(JsonElement element) =>
113+
element.ValueKind == JsonValueKind.String ? element.GetString() : ReadString(element, "spdxId") ?? ReadString(element, "@id");
114+
115+
private static JsonElement Resolve(JsonElement element, IDictionary<string, JsonElement> elements)
116+
{
117+
var id = ReadId(element);
118+
return id != null && elements.TryGetValue(id, out var resolved) ? resolved : element;
119+
}
120+
121+
private static IEnumerable<JsonElement> ReadValues(JsonElement element, string property)
122+
{
123+
if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value))
124+
{
125+
return [];
126+
}
127+
128+
return value.ValueKind == JsonValueKind.Array ? value.EnumerateArray().ToArray() : [value];
129+
}
130+
131+
private static string? ReadCreator(JsonElement creationInfo, string property, string type, IDictionary<string, JsonElement> elements) =>
132+
ReadValues(creationInfo, property)
133+
.Select(element => Resolve(element, elements))
134+
.Where(element => HasType(element, type))
135+
.Select(element => ReadString(element, "name"))
136+
.FirstOrDefault(name => !string.IsNullOrWhiteSpace(name));
137+
138+
private SpdxComponent? ReadDocument(JsonElement root, string checksum, string path)
139+
{
140+
if (root.ValueKind != JsonValueKind.Object
141+
|| !root.TryGetProperty("@graph", out var graph)
142+
|| graph.ValueKind != JsonValueKind.Array)
143+
{
144+
this.Logger.LogDebug("Discovered file at {ManifestLocation} is not an SPDX 3.0 JSON-LD graph, skipping", path);
145+
return null;
146+
}
147+
148+
var documents = graph.EnumerateArray().Where(element => HasType(element, "SpdxDocument")).ToList();
149+
if (documents.Count != 1)
150+
{
151+
this.Logger.LogWarning("SPDX 3.0 file at {ManifestLocation} must contain exactly one SpdxDocument, skipping", path);
152+
return null;
153+
}
154+
155+
var elements = new Dictionary<string, JsonElement>(StringComparer.Ordinal);
156+
foreach (var element in graph.EnumerateArray())
157+
{
158+
var id = ReadId(element);
159+
if (id != null && !elements.TryAdd(id, element))
160+
{
161+
throw new JsonException($"Duplicate SPDX element identifier: {id}.");
162+
}
163+
}
164+
165+
var document = documents[0];
166+
if (!document.TryGetProperty("creationInfo", out var creationInfo))
167+
{
168+
throw new JsonException("SpdxDocument is missing creationInfo.");
169+
}
170+
171+
creationInfo = Resolve(creationInfo, elements);
172+
var version = ReadString(creationInfo, "specVersion")
173+
?? throw new JsonException("SpdxDocument creationInfo is missing specVersion.");
174+
175+
if (!IsSupportedVersion(version))
176+
{
177+
this.Logger.LogDebug("SPDX file at {ManifestLocation} has unsupported specVersion {SpecVersion}, skipping", path, version);
178+
return null;
179+
}
180+
181+
var documentId = ReadId(document);
182+
if (string.IsNullOrWhiteSpace(documentId) || !Uri.TryCreate(documentId, UriKind.Absolute, out var documentUri))
183+
{
184+
throw new JsonException("SpdxDocument must have an absolute spdxId.");
185+
}
186+
187+
var roots = ReadValues(document, "rootElement").Select(ReadId).ToList();
188+
if (roots.Any(id => string.IsNullOrWhiteSpace(id)))
189+
{
190+
throw new JsonException("SpdxDocument contains an invalid rootElement.");
191+
}
192+
193+
if (roots.Count > 1)
194+
{
195+
this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one rootElement, first will be selected as root element.", path);
196+
}
197+
else if (roots.Count == 0)
198+
{
199+
this.Logger.LogWarning("SPDX file at {ManifestLocation} has no rootElement, considering the SpdxDocument as the root element.", path);
200+
}
201+
202+
// SPDX 3 has no documentNamespace, and name is optional; the document IRI identifies the document.
203+
var name = ReadString(document, "name");
204+
return new SpdxComponent(
205+
$"SPDX-{version}",
206+
documentUri,
207+
string.IsNullOrWhiteSpace(name) ? documentId : name,
208+
checksum,
209+
roots.FirstOrDefault() ?? documentId,
210+
path)
211+
{
212+
CreatorTool = ReadCreator(creationInfo, "createdUsing", "Tool", elements),
213+
CreatorOrganization = ReadCreator(creationInfo, "createdBy", "Organization", elements),
214+
};
215+
}
216+
}

‎src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s
160160

161161
// SPDX
162162
services.AddSingleton<IComponentDetector, Spdx22ComponentDetector>();
163+
services.AddSingleton<IComponentDetector, Spdx30ComponentDetector>();
163164

164165
// VCPKG
165166
services.AddSingleton<IComponentDetector, VcpkgComponentDetector>();

‎test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@ namespace Microsoft.ComponentDetection.Detectors.Tests;
1414
using Microsoft.ComponentDetection.Contracts.TypedComponent;
1515
using Microsoft.ComponentDetection.Detectors.Spdx;
1616
using Microsoft.ComponentDetection.TestsUtilities;
17+
using Microsoft.Extensions.Logging;
1718
using Microsoft.VisualStudio.TestTools.UnitTesting;
19+
using Moq;
1820

1921
[TestClass]
2022
[TestCategory("Governance/All")]
@@ -132,6 +134,37 @@ public async Task TestSbomDetector_SimpleSbomAsync()
132134
sbomComponent.CreatorOrganization.Should().Be("Microsoft");
133135
}
134136

137+
[TestMethod]
138+
[DataRow("{\"@context\":\"https://spdx.org/rdf/3.0.1/spdx-context.jsonld\",\"@graph\":[]}")]
139+
[DataRow("{\"spdxVersion\":\"SPDX-2.3\"}")]
140+
public async Task TestSbomDetector_OtherVersionsLogAtDebugAsync(string content)
141+
{
142+
var logger = new Mock<ILogger<Spdx22ComponentDetector>>();
143+
var (result, recorder) = await this.detectorTestUtility
144+
.AddServiceMock(logger)
145+
.WithFile("manifest.spdx.json", content)
146+
.ExecuteDetectorAsync();
147+
148+
result.ResultCode.Should().Be(ProcessingResultCode.Success);
149+
recorder.GetDetectedComponents().Should().BeEmpty();
150+
logger.Verify(
151+
log => log.Log(
152+
LogLevel.Debug,
153+
It.IsAny<EventId>(),
154+
It.Is<It.IsAnyType>((value, _) => value.ToString().Contains("not SPDX-2.2")),
155+
It.IsAny<Exception>(),
156+
It.IsAny<Func<It.IsAnyType, Exception, string>>()),
157+
Times.Once);
158+
logger.Verify(
159+
log => log.Log(
160+
It.Is<LogLevel>(level => level >= LogLevel.Warning),
161+
It.IsAny<EventId>(),
162+
It.IsAny<It.IsAnyType>(),
163+
It.IsAny<Exception>(),
164+
It.IsAny<Func<It.IsAnyType, Exception, string>>()),
165+
Times.Never);
166+
}
167+
135168
[TestMethod]
136169
public async Task TestSbomDetector_BlankJsonAsync()
137170
{

0 commit comments

Comments
 (0)