From dc15077e618b284d43a2d9b0c2418c5fea4f2aea Mon Sep 17 00:00:00 2001
From: Daniel Cazzulino <daniel@cazzulino.com>
Date: Tue, 1 Oct 2024 12:52:57 -0300
Subject: [PATCH] Add rate limit pausing when collecting stats

Since we run for many ours, it might be we run out of requests.
---
 src/Commands/CliGraphQueryClient.cs |  1 +
 src/Commands/NuGetStatsCommand.cs   | 26 ++++++++++++++++++++++++++
 src/Web/Program.cs                  |  1 -
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/Commands/CliGraphQueryClient.cs b/src/Commands/CliGraphQueryClient.cs
index bef73034..376cbc7a 100644
--- a/src/Commands/CliGraphQueryClient.cs
+++ b/src/Commands/CliGraphQueryClient.cs
@@ -27,6 +27,7 @@ public class CliGraphQueryClient : IGraphQueryClient
             // Legacy queries won't have anything in the "query" (a URL endpoint is expected), but can still paginate in the CLI.
             // NOTE: this is an inconsistency with the HttpGraphQueryClient, which doesn't do pagination for legacy queries.
             // TODO: perhaps we should implement that, but it's not needed right now.
+            // In Particular, we ARE levarging this inconsistency in our RepositoryContributors query.
             (query.IsLegacy ||
             (query.Query.Contains("$endCursor") && query.Query.Contains("first:")));
 
diff --git a/src/Commands/NuGetStatsCommand.cs b/src/Commands/NuGetStatsCommand.cs
index e05b363f..75783941 100644
--- a/src/Commands/NuGetStatsCommand.cs
+++ b/src/Commands/NuGetStatsCommand.cs
@@ -97,6 +97,10 @@ public override async Task<int> ExecuteAsync(CommandContext context, NuGetStatsS
 
         using var withToken = GitHub.WithToken(token);
         if (!string.IsNullOrEmpty(token) && withToken is null)
+        {
+            AnsiConsole.MarkupLine(":cross_mark: [yellow]Invalid GitHub token provided[/]");
+            return -1;
+        }
 
         var repository = new SourceRepository(new PackageSource("https://api.nuget.org/v3/index.json"), Repository.Provider.GetCoreV3());
         var resource = await repository.GetResourceAsync<PackageMetadataResource>();
@@ -376,11 +380,33 @@ await Parallel.ForEachAsync(tasks, paralell, async (source, cancellation) =>
                             if (!model.Repositories.ContainsKey(ownerRepo))
                             {
                                 var contribs = await graph.QueryAsync(GraphQueries.RepositoryContributors(ownerRepo));
+                                if (contribs?.Length == 0)
+                                {
+                                    // Make sure we haven't exhausted the GH API rate limit
+                                    var rate = await graph.QueryAsync(GraphQueries.RateLimits);
+                                    if (rate is not { })
+                                        throw new InvalidOperationException("Failed to get rate limits for current token.");
+
+                                    if (rate.General.Remaining < 10 || rate.GraphQL.Remaining < 10)
+                                    {
+                                        var reset = DateTimeOffset.FromUnixTimeSeconds(Math.Min(rate.General.Reset, rate.GraphQL.Reset));
+                                        var wait = reset - DateTimeOffset.UtcNow;
+                                        task.Description = $":hourglass_not_done: [yellow]Rate limit exhausted, waiting {wait.Humanize()} until reset[/]";
+                                        await Task.Delay(wait, cancellation);
+                                        contribs = await graph.QueryAsync(GraphQueries.RepositoryContributors(ownerRepo));
+                                    }
+                                }
+
                                 if (contribs != null)
+                                {
                                     model.Repositories.TryAdd(ownerRepo, new(contribs));
+                                }
                                 else
+                                {
                                     // Might not be a GH repo at all, or perhaps it's just empty?
                                     model.Repositories.TryAdd(ownerRepo, []);
+                                    AnsiConsole.MarkupLine($":warning: [yellow]{link}[/]: no contributors found for [white]{ownerRepo}[/]");
+                                }
                             }
 
                             foreach (var author in model.Repositories[ownerRepo])
diff --git a/src/Web/Program.cs b/src/Web/Program.cs
index b52d72c3..9e93c121 100644
--- a/src/Web/Program.cs
+++ b/src/Web/Program.cs
@@ -1,5 +1,4 @@
 using System.Net.Http.Headers;
-using System.Net.Http.Json;
 using System.Security.Cryptography;
 using System.Text.Json;
 using Devlooped;