Skip to content

Commit b68ce98

Browse files
michaelnebelmichaelnebel
authored
Merge pull request #22494 from michaelnebel/csharp/replaces-base
C#: Support `replaces-base` via the DependabotProxy.
2 parents 401a516 + 93aa3a3 commit b68ce98

13 files changed

Lines changed: 396 additions & 54 deletions

File tree

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs‎

Lines changed: 58 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections.Immutable;
23
using System.Collections.Generic;
34
using System.IO;
45
using System.Security.Cryptography.X509Certificates;
@@ -14,13 +15,51 @@ public class DependabotProxy : IDependabotProxy
1415
/// <summary>
1516
/// Represents configurations for package registries.
1617
/// </summary>
17-
/// <param name="Type">The type of package registry.</param>
18-
/// <param name="URL">The URL of the package registry.</param>
19-
public record class RegistryConfig(string Type, string URL);
18+
public class RegistryConfig
19+
{
20+
/// <summary>
21+
/// The type of the package registry.
22+
/// </summary>
23+
public string? Type { get; init; }
24+
25+
/// <summary>
26+
/// The URL of the package registry.
27+
/// </summary>
28+
public string? Url { get; init; }
29+
30+
/// <summary>
31+
/// A boolean indicating whether this registry replaces the base registry.
32+
/// </summary>
33+
[JsonProperty("replaces-base")]
34+
public bool ReplacesBase { get; init; } = false;
35+
};
2036

2137
public string Address { get; }
2238

23-
public HashSet<string> RegistryURLs { get; } = [];
39+
/// <summary>
40+
/// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry.
41+
/// </summary>
42+
private readonly Dictionary<string, bool> registryMapping = [];
43+
44+
private ImmutableHashSet<string>? registryURLs;
45+
/// <summary>
46+
/// Gets the set of registry URLs that have been configured as part of the organization-level
47+
/// private registry configuration. This includes all registries, regardless of whether they replace
48+
/// the default feeds.
49+
/// </summary>
50+
public ImmutableHashSet<string> RegistryURLs =>
51+
registryURLs ??= registryMapping.Keys.ToImmutableHashSet();
52+
53+
private ImmutableHashSet<string>? registryBaseURLs;
54+
/// <summary>
55+
/// Gets the set of registry URLs that have been configured as part of the organization-level
56+
/// private registry configuration and that replace the default registry. This is a subset of
57+
/// <see cref="RegistryURLs"/>.
58+
/// If non-empty, the set should be used as a replacement for the default registry during
59+
/// package resolution.
60+
/// </summary>
61+
public ImmutableHashSet<string> RegistryBaseURLs =>
62+
registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet();
2463

2564
public string? CertificatePath { get; private set; }
2665

@@ -56,16 +95,28 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te
5695
{
5796
foreach (RegistryConfig registry in array)
5897
{
98+
if (string.IsNullOrWhiteSpace(registry.Url))
99+
{
100+
logger.LogError("Ignoring registry with empty URL.");
101+
continue;
102+
}
103+
104+
if (string.IsNullOrWhiteSpace(registry.Type))
105+
{
106+
logger.LogError($"Ignoring registry at '{registry.Url}' since it has no type.");
107+
continue;
108+
}
109+
59110
// The array contains all configured private registries, not just ones for C#.
60111
// We ignore the non-C# ones here.
61112
if (!registry.Type.Equals("nuget_feed"))
62113
{
63-
logger.LogDebug($"Ignoring registry at '{registry.URL}' since it is not of type 'nuget_feed'.");
114+
logger.LogDebug($"Ignoring registry at '{registry.Url}' since it is not of type 'nuget_feed'.");
64115
continue;
65116
}
66117

67-
logger.LogInfo($"Found private registry at '{registry.URL}'");
68-
RegistryURLs.Add(registry.URL);
118+
logger.LogInfo($"Found private registry at '{registry.Url}'");
119+
registryMapping.AddOrUpdateToLatest(registry.Url, registry.ReplacesBase);
69120
}
70121
}
71122
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ public bool Exec(List<string> execArgs)
137137

138138
private static readonly IReadOnlyList<string> nugetListSourceCommandArgs = ["nuget", "list", "source", "--format", "Short"];
139139

140-
public IList<string> GetNugetFeeds(string nugetConfig)
140+
public IList<string> GetNugetFeedsFromConfig(string nugetConfig)
141141
{
142142
logger.LogInfo($"Getting NuGet feeds from '{nugetConfig}'...");
143143
return GetResultList([.. nugetListSourceCommandArgs, "--configfile", nugetConfig]);

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ internal static class EnvironmentVariableNames
5656

5757
/// <summary>
5858
/// Specifies the NuGet feeds to use for fallback NuGet dependency fetching. The value is a space-separated list of feed URLs.
59-
/// The default value is `https://api.nuget.org/v3/index.json`.
6059
/// </summary>
6160
public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
6261

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs‎

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
1010
{
1111
internal sealed partial class FeedManager : IDisposable
1212
{
13-
internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json";
13+
private const string PublicNugetOrg = "nuget.org";
14+
private const string PublicDotNugetOrg = $".{PublicNugetOrg}";
15+
internal const string PublicApiNugetOrgFeed = $"https://api{PublicDotNugetOrg}/v3/index.json";
1416

1517
private readonly ILogger logger;
1618
private readonly IDotNet dotnet;
1719
private readonly IFileProvider fileProvider;
1820
private readonly DependencyDirectory emptyPackageDirectory;
1921
private readonly ImmutableHashSet<string> privateRegistryFeeds;
22+
private readonly bool hasPrivateRegistryBaseFeeds;
23+
private readonly ImmutableHashSet<string> privateRegistryBaseFeeds;
2024
private readonly IFeedManagerIO feedManagerIo;
2125

2226
/// <summary>
@@ -72,14 +76,33 @@ internal sealed partial class FeedManager : IDisposable
7276
/// </summary>
7377
public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
7478

79+
private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds;
80+
81+
/// <summary>
82+
/// Gets the list of default NuGet feeds that are configured in the environment.
83+
/// This is either the public NuGet feed or a set of feeds specified by the environment.
84+
/// </summary>
85+
public ImmutableHashSet<string> DefaultFeeds { get; init; }
86+
87+
/// <summary>
88+
/// Gets the list of reachable default NuGet feeds.
89+
/// </summary>
90+
public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value;
91+
7592
public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
7693
{
7794
this.logger = logger;
7895
this.dotnet = dotnet;
7996
this.fileProvider = fileProvider;
8097
this.feedManagerIo = feedManagerIo;
81-
privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
98+
privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? [];
8299
HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
100+
privateRegistryBaseFeeds = dependabotProxy?.RegistryBaseURLs ?? [];
101+
hasPrivateRegistryBaseFeeds = privateRegistryBaseFeeds.Count > 0;
102+
103+
DefaultFeeds = hasPrivateRegistryBaseFeeds
104+
? privateRegistryBaseFeeds
105+
: [PublicApiNugetOrgFeed];
83106
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
84107

85108
lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
@@ -96,13 +119,28 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
96119
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
97120
return reachableFallbackFeeds.ToImmutableHashSet();
98121
});
122+
lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(DefaultFeeds));
99123
}
100124

101125
public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
102126
: this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy))
103127
{
104128
}
105129

130+
private bool IsNugetOrgFeed(string url)
131+
{
132+
try
133+
{
134+
var uri = new Uri(url);
135+
return uri.Host.EndsWith(PublicDotNugetOrg, StringComparison.InvariantCultureIgnoreCase) ||
136+
string.Equals(uri.Host, PublicNugetOrg, StringComparison.InvariantCultureIgnoreCase);
137+
}
138+
catch (UriFormatException)
139+
{
140+
return false;
141+
}
142+
}
143+
106144
private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
107145
{
108146
var results = getNugetFeeds();
@@ -124,18 +162,26 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
124162
continue;
125163
}
126164

127-
if (!string.IsNullOrWhiteSpace(url))
165+
if (hasPrivateRegistryBaseFeeds && IsNugetOrgFeed(url))
128166
{
129-
yield return url;
167+
// Use private registry base feeds.
168+
foreach (var feed in privateRegistryBaseFeeds)
169+
{
170+
logger.LogDebug($"Using private registry base feed '{feed}'.");
171+
yield return feed;
172+
}
173+
continue;
130174
}
175+
176+
yield return url;
131177
}
132178
}
133179

134180
private IEnumerable<string> GetFeedsFromFolder(string folderPath) =>
135181
GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folderPath));
136182

137183
private IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
138-
GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath));
184+
GetFeeds(() => dotnet.GetNugetFeedsFromConfig(nugetConfigPath));
139185

140186
/// <summary>
141187
/// Constructs the NuGet sources argument for the restore command based on the given feeds.
@@ -266,22 +312,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe
266312
return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
267313
}
268314

269-
/// <summary>
270-
/// Return true if the default NuGet feed is reachable, false otherwise.
271-
/// If the reachability check is disabled, this method will always return true.
272-
/// </summary>
273-
/// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
274-
public bool IsDefaultFeedReachable()
275-
{
276-
if (CheckNugetFeedResponsiveness)
277-
{
278-
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
279-
return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
280-
}
281-
282-
return true;
283-
}
284-
285315
/// <summary>
286316
/// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
287317
/// </summary>
@@ -315,8 +345,8 @@ private List<string> GetReachableFallbackNugetFeeds()
315345
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
316346
if (fallbackFeeds.Count == 0)
317347
{
318-
fallbackFeeds.Add(PublicNugetOrgFeed);
319-
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
348+
fallbackFeeds.UnionWith(DefaultFeeds);
349+
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", DefaultFeeds.OrderBy(f => f))}");
320350

321351
var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
322352
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
@@ -329,6 +359,10 @@ private List<string> GetReachableFallbackNugetFeeds()
329359
logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", ExplicitFeeds.OrderBy(f => f))}");
330360
}
331361
}
362+
else
363+
{
364+
logger.LogInfo($"Using fallback NuGet feeds from environment variable '{EnvironmentVariableNames.FallbackNugetFeeds}'.");
365+
}
332366

333367
return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true);
334368
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
using System;
2-
using System.Collections.Generic;
2+
using System.Collections.Immutable;
33
using System.Security.Cryptography.X509Certificates;
44

55
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable
1414
/// <summary>
1515
/// The URLs of package registries that are configured for the proxy.
1616
/// </summary>
17-
HashSet<string> RegistryURLs { get; }
17+
ImmutableHashSet<string> RegistryURLs { get; }
18+
19+
/// <summary>
20+
/// The URLs of package registries that replace the base registry.
21+
/// </summary>
22+
ImmutableHashSet<string> RegistryBaseURLs { get; }
1823

1924
/// <summary>
2025
/// The path to the temporary file where the certificate is stored.

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ public interface IDotNet
1313
IList<string> GetListedRuntimes();
1414
IList<string> GetListedSdks();
1515
bool Exec(List<string> execArgs);
16-
IList<string> GetNugetFeeds(string nugetConfig);
16+
IList<string> GetNugetFeedsFromConfig(string nugetConfig);
1717
IList<string> GetNugetFeedsFromFolder(string folderPath);
1818
}
1919

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,7 @@ private bool TryRestorePackageManually(string package, List<string> nugetSources
460460
return true;
461461
}
462462

463-
if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources.Count > 0)
463+
if (!feedManager.CheckNugetFeedResponsiveness && !feedManager.HasPrivateRegistryFeeds && res.HasNugetPackageSourceError && nugetSources.Count > 0)
464464
{
465465
logger.LogDebug($"Trying to restore '{package}' without explicitly providing NuGet sources.");
466466
// Restore could not be completed because the listed source is unavailable. Try without an explicit restore source argument.

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore
6767

6868
private bool IsWindows => SystemBuildActions.Instance.IsWindows();
6969

70-
private bool? isDefaultFeedReachable;
71-
private bool IsDefaultFeedReachable =>
72-
isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable();
73-
7470
/// <summary>
7571
/// Create the package manager for a specified source tree.
7672
/// </summary>
@@ -169,15 +165,18 @@ private bool TryRestoreNugetPackage(string packagesConfig)
169165

170166
List<string> sourcesArgument = [];
171167
var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList();
172-
var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable;
168+
var defaultFeeds = feedManager.CheckNugetFeedResponsiveness
169+
? feedManager.ReachableDefaultFeeds
170+
: feedManager.DefaultFeeds;
171+
var useDefaultFeeds = feedsToUse.Count == 0 && defaultFeeds.Count > 0;
173172

174173
// Explicitly construct the sources to be used for the restore command when checking feed
175-
// responsiveness, using private registries, or falling back to nuget.org.
176-
if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed)
174+
// responsiveness, using private registries, or falling back to default feeds.
175+
if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds)
177176
{
178-
if (useDefaultFeed)
177+
if (useDefaultFeeds)
179178
{
180-
feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
179+
feedsToUse.AddRange(defaultFeeds);
181180
}
182181
var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
183182
sourcesArgument = restoreFeeds.SelectMany<string, string>(feed => ["-Source", feed]).ToList();

0 commit comments

Comments
 (0)