From d4c596de87583afd72f59f01e7ac81f7da5b9585 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Tue, 19 Sep 2023 17:17:43 +0300 Subject: [PATCH] Make use of IPeriodicTask in place of BackgroundService so as to separate concerns --- .../MissedTriggerCheckerTaskTests.cs | 156 +++++++++++++++++ .../PeriodicTasks/SynchronizationTaskTests.cs | 69 ++++++++ .../UpdateJobsCleanerTaskTests.cs} | 104 ++--------- .../PeriodicTasks/MissedTriggerCheckerTask.cs | 78 +++++++++ .../PeriodicTasks/SynchronizationTask.cs | 27 +++ .../PeriodicTasks/UpdateJobsCleanerTask.cs | 61 +++++++ server/Tingle.Dependabot/Program.cs | 9 +- .../Workflow/WorkflowBackgroundService.cs | 165 ------------------ 8 files changed, 412 insertions(+), 257 deletions(-) create mode 100644 server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs create mode 100644 server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs rename server/Tingle.Dependabot.Tests/{Workflow/WorkflowBackgroundServiceTests.cs => PeriodicTasks/UpdateJobsCleanerTaskTests.cs} (60%) create mode 100644 server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs create mode 100644 server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs create mode 100644 server/Tingle.Dependabot/PeriodicTasks/UpdateJobsCleanerTask.cs delete mode 100644 server/Tingle.Dependabot/Workflow/WorkflowBackgroundService.cs diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs new file mode 100644 index 00000000..d4810ec7 --- /dev/null +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs @@ -0,0 +1,156 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; +using Tingle.Dependabot.Models.Dependabot; +using Tingle.Dependabot.Models.Management; +using Tingle.Dependabot.PeriodicTasks; +using Tingle.EventBus; +using Tingle.EventBus.Transports.InMemory; +using Xunit; +using Xunit.Abstractions; + +namespace Tingle.Dependabot.Tests.PeriodicTasks; + +public class MissedTriggerCheckerTaskTests +{ + private const string RepositoryId = "repo_1234567890"; + private const int UpdateId1 = 1; + + private readonly ITestOutputHelper outputHelper; + + public MissedTriggerCheckerTaskTests(ITestOutputHelper outputHelper) + { + this.outputHelper = outputHelper ?? throw new ArgumentNullException(nameof(outputHelper)); + } + + [Fact] + public async Task CheckAsync_MissedScheduleIsDetected() + { + var referencePoint = DateTimeOffset.Parse("2023-01-24T05:00:00+00:00"); + var lastUpdate0 = DateTimeOffset.Parse("2023-01-24T03:45:00+00:00"); + var lastUpdate1 = DateTimeOffset.Parse("2023-01-23T03:30:00+00:00"); + await TestAsync(lastUpdate0, lastUpdate1, async (harness, context, pt) => + { + await pt.CheckAsync(referencePoint); + + // Ensure the message was published + var evt_context = Assert.IsType>(Assert.Single(await harness.PublishedAsync())); + var inner = evt_context.Event; + Assert.NotNull(inner); + Assert.Equal(RepositoryId, inner.RepositoryId); + Assert.Equal(UpdateId1, inner.RepositoryUpdateId); + Assert.Equal(UpdateJobTrigger.MissedSchedule, inner.Trigger); + }); + } + + [Fact] + public async Task CheckAsync_MissedScheduleIsDetected_NotRun_Before() + { + var referencePoint = DateTimeOffset.Parse("2023-01-24T05:00:00+00:00"); + var lastUpdate0 = DateTimeOffset.Parse("2023-01-24T03:45:00+00:00"); + var lastUpdate1 = (DateTimeOffset?)null; + await TestAsync(lastUpdate0, lastUpdate1, async (harness, context, pt) => + { + await pt.CheckAsync(referencePoint); + + // Ensure the message was published + var evt_context = Assert.IsType>(Assert.Single(await harness.PublishedAsync())); + var inner = evt_context.Event; + Assert.NotNull(inner); + Assert.Equal(RepositoryId, inner.RepositoryId); + Assert.Equal(UpdateId1, inner.RepositoryUpdateId); + Assert.Equal(UpdateJobTrigger.MissedSchedule, inner.Trigger); + }); + } + + [Fact] + public async Task CheckAsync_NoMissedSchedule() + { + var referencePoint = DateTimeOffset.Parse("2023-01-24T05:00:00+00:00"); + var lastUpdate0 = DateTimeOffset.Parse("2023-01-24T03:45:00+00:00"); + var lastUpdate1 = DateTimeOffset.Parse("2023-01-24T03:30:00+00:00"); + await TestAsync(lastUpdate0, lastUpdate1, async (harness, context, pt) => + { + await pt.CheckAsync(referencePoint); + + // Ensure nothing was published + Assert.Empty(await harness.PublishedAsync()); + }); + } + + private async Task TestAsync(DateTimeOffset? lastUpdate0, DateTimeOffset? lastUpdate1, Func executeAndVerify) + { + var host = Host.CreateDefaultBuilder() + .ConfigureLogging(builder => builder.AddXUnit(outputHelper)) + .ConfigureServices((context, services) => + { + var dbName = Guid.NewGuid().ToString(); + services.AddDbContext(options => + { + options.UseInMemoryDatabase(dbName, o => o.EnableNullChecks()); + options.EnableDetailedErrors(); + }); + services.AddEventBus(builder => builder.AddInMemoryTransport().AddInMemoryTestHarness()); + }) + .Build(); + + using var scope = host.Services.CreateScope(); + var provider = scope.ServiceProvider; + + var context = provider.GetRequiredService(); + await context.Database.EnsureCreatedAsync(); + + await context.Repositories.AddAsync(new Repository + { + Id = RepositoryId, + Name = "test-repo", + ConfigFileContents = "", + Updates = new List + { + new RepositoryUpdate + { + PackageEcosystem = "npm", + Directory = "/", + Schedule = new DependabotUpdateSchedule + { + Interval = DependabotScheduleInterval.Daily, + Time = new(3, 45), + }, + LatestUpdate = lastUpdate0, + }, + new RepositoryUpdate + { + PackageEcosystem = "npm", + Directory = "/legacy", + Schedule = new DependabotUpdateSchedule + { + Interval = DependabotScheduleInterval.Daily, + Time = new(3, 30), + }, + LatestUpdate = lastUpdate1, + }, + }, + }); + await context.SaveChangesAsync(); + + var harness = provider.GetRequiredService(); + await harness.StartAsync(); + + try + { + var pt = ActivatorUtilities.GetServiceOrCreateInstance(provider); + + await executeAndVerify(harness, context, pt); + + // Ensure there were no publish failures + Assert.Empty(await harness.FailedAsync()); + } + finally + { + await harness.StopAsync(); + } + } +} diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs new file mode 100644 index 00000000..37a256e3 --- /dev/null +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Tingle.Dependabot.Events; +using Tingle.Dependabot.Workflow; +using Tingle.EventBus; +using Tingle.EventBus.Transports.InMemory; +using Xunit; +using Xunit.Abstractions; + +namespace Tingle.Dependabot.Tests.PeriodicTasks; + +public class SynchronizationTaskTests +{ + private readonly ITestOutputHelper outputHelper; + + public SynchronizationTaskTests(ITestOutputHelper outputHelper) + { + this.outputHelper = outputHelper ?? throw new ArgumentNullException(nameof(outputHelper)); + } + + [Fact] + public async Task SynchronizationInnerAsync_Works() + { + await TestAsync(async (harness, pt) => + { + await pt.SyncAsync(); + + // Ensure the message was published + var evt_context = Assert.IsType>(Assert.Single(await harness.PublishedAsync())); + var inner = evt_context.Event; + Assert.NotNull(inner); + Assert.Null(inner.RepositoryId); + Assert.Null(inner.RepositoryProviderId); + Assert.False(inner.Trigger); + }); + } + + private async Task TestAsync(Func executeAndVerify) + { + var host = Host.CreateDefaultBuilder() + .ConfigureLogging(builder => builder.AddXUnit(outputHelper)) + .ConfigureServices((context, services) => + { + services.AddEventBus(builder => builder.AddInMemoryTransport().AddInMemoryTestHarness()); + }) + .Build(); + + using var scope = host.Services.CreateScope(); + var provider = scope.ServiceProvider; + + var harness = provider.GetRequiredService(); + await harness.StartAsync(); + + try + { + var pt = ActivatorUtilities.GetServiceOrCreateInstance(provider); + + await executeAndVerify(harness, pt); + + // Ensure there were no publish failures + Assert.Empty(await harness.FailedAsync()); + } + finally + { + await harness.StopAsync(); + } + } +} diff --git a/server/Tingle.Dependabot.Tests/Workflow/WorkflowBackgroundServiceTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs similarity index 60% rename from server/Tingle.Dependabot.Tests/Workflow/WorkflowBackgroundServiceTests.cs rename to server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs index 03ebb9e1..997dafcf 100644 --- a/server/Tingle.Dependabot.Tests/Workflow/WorkflowBackgroundServiceTests.cs +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs @@ -6,103 +6,29 @@ using Tingle.Dependabot.Models; using Tingle.Dependabot.Models.Dependabot; using Tingle.Dependabot.Models.Management; -using Tingle.Dependabot.Workflow; +using Tingle.Dependabot.PeriodicTasks; using Tingle.EventBus; using Tingle.EventBus.Transports.InMemory; using Xunit; using Xunit.Abstractions; -namespace Tingle.Dependabot.Tests.Workflow; +namespace Tingle.Dependabot.Tests.PeriodicTasks; -public class WorkflowBackgroundServiceTests +public class UpdateJobsCleanerTaskTests { private const string RepositoryId = "repo_1234567890"; - private const int UpdateId1 = 1; private readonly ITestOutputHelper outputHelper; - public WorkflowBackgroundServiceTests(ITestOutputHelper outputHelper) + public UpdateJobsCleanerTaskTests(ITestOutputHelper outputHelper) { this.outputHelper = outputHelper ?? throw new ArgumentNullException(nameof(outputHelper)); } [Fact] - public async Task SynchronizationInnerAsync_Works() + public async Task CleanupAsync_ResolvesJobs() { - await TestAsync(async (harness, context, service) => - { - await service.SynchronizationInnerAsync(); - - // Ensure the message was published - var evt_context = Assert.IsType>(Assert.Single(await harness.PublishedAsync())); - var inner = evt_context.Event; - Assert.NotNull(inner); - Assert.Null(inner.RepositoryId); - Assert.Null(inner.RepositoryProviderId); - Assert.False(inner.Trigger); - }); - } - - [Fact] - public async Task CheckMissedTriggerInnerAsync_MissedScheduleIsDetected() - { - var referencePoint = DateTimeOffset.Parse("2023-01-24T05:00:00+00:00"); - var lastUpdate0 = DateTimeOffset.Parse("2023-01-24T03:45:00+00:00"); - var lastUpdate1 = DateTimeOffset.Parse("2023-01-23T03:30:00+00:00"); - await TestAsync(lastUpdate0, lastUpdate1, async (harness, context, service) => - { - await service.CheckMissedTriggerInnerAsync(referencePoint); - - // Ensure the message was published - var evt_context = Assert.IsType>(Assert.Single(await harness.PublishedAsync())); - var inner = evt_context.Event; - Assert.NotNull(inner); - Assert.Equal(RepositoryId, inner.RepositoryId); - Assert.Equal(UpdateId1, inner.RepositoryUpdateId); - Assert.Equal(UpdateJobTrigger.MissedSchedule, inner.Trigger); - }); - } - - [Fact] - public async Task CheckMissedTriggerInnerAsync_MissedScheduleIsDetected_NotRun_Before() - { - var referencePoint = DateTimeOffset.Parse("2023-01-24T05:00:00+00:00"); - var lastUpdate0 = DateTimeOffset.Parse("2023-01-24T03:45:00+00:00"); - var lastUpdate1 = (DateTimeOffset?)null; - await TestAsync(lastUpdate0, lastUpdate1, async (harness, context, service) => - { - await service.CheckMissedTriggerInnerAsync(referencePoint); - - // Ensure the message was published - var evt_context = Assert.IsType>(Assert.Single(await harness.PublishedAsync())); - var inner = evt_context.Event; - Assert.NotNull(inner); - Assert.Equal(RepositoryId, inner.RepositoryId); - Assert.Equal(UpdateId1, inner.RepositoryUpdateId); - Assert.Equal(UpdateJobTrigger.MissedSchedule, inner.Trigger); - }); - } - - [Fact] - public async Task CheckMissedTriggerInnerAsync_NoMissedSchedule() - { - var referencePoint = DateTimeOffset.Parse("2023-01-24T05:00:00+00:00"); - var lastUpdate0 = DateTimeOffset.Parse("2023-01-24T03:45:00+00:00"); - var lastUpdate1 = DateTimeOffset.Parse("2023-01-24T03:30:00+00:00"); - await TestAsync(lastUpdate0, lastUpdate1, async (harness, context, service) => - { - await service.CheckMissedTriggerInnerAsync(referencePoint); - - // Ensure nothing was published - Assert.Empty(await harness.PublishedAsync()); - }); - } - - - [Fact] - public async Task CleanupInnerAsync_ResolvesJobs() - { - await TestAsync(async (harness, context, job) => + await TestAsync(async (harness, context, pt) => { var targetId = Guid.NewGuid().ToString(); await context.UpdateJobs.AddAsync(new UpdateJob @@ -143,7 +69,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob }); await context.SaveChangesAsync(); - await job.CleanupInnerAsync(); + await pt.CleanupAsync(); // Ensure the message was published var evt_context = Assert.IsType>(Assert.Single(await harness.PublishedAsync())); @@ -154,9 +80,9 @@ await context.UpdateJobs.AddAsync(new UpdateJob } [Fact] - public async Task CleanupInnerAsync_DeletesOldJobsAsync() + public async Task CleanupAsync_DeletesOldJobsAsync() { - await TestAsync(async (harness, context, job) => + await TestAsync(async (harness, context, pt) => { await context.UpdateJobs.AddAsync(new UpdateJob { @@ -193,14 +119,12 @@ await context.UpdateJobs.AddAsync(new UpdateJob }); await context.SaveChangesAsync(); - await job.CleanupInnerAsync(); + await pt.CleanupAsync(); Assert.Equal(1, await context.UpdateJobs.CountAsync()); }); } - private Task TestAsync(Func executeAndVerify) => TestAsync(null, null, executeAndVerify); - - private async Task TestAsync(DateTimeOffset? lastUpdate0, DateTimeOffset? lastUpdate1, Func executeAndVerify) + private async Task TestAsync(Func executeAndVerify) { var host = Host.CreateDefaultBuilder() .ConfigureLogging(builder => builder.AddXUnit(outputHelper)) @@ -238,7 +162,6 @@ await context.Repositories.AddAsync(new Repository Interval = DependabotScheduleInterval.Daily, Time = new(3, 45), }, - LatestUpdate = lastUpdate0, }, new RepositoryUpdate { @@ -249,7 +172,6 @@ await context.Repositories.AddAsync(new Repository Interval = DependabotScheduleInterval.Daily, Time = new(3, 30), }, - LatestUpdate = lastUpdate1, }, }, }); @@ -260,9 +182,9 @@ await context.Repositories.AddAsync(new Repository try { - var service = ActivatorUtilities.GetServiceOrCreateInstance(provider); + var pt = ActivatorUtilities.GetServiceOrCreateInstance(provider); - await executeAndVerify(harness, context, service); + await executeAndVerify(harness, context, pt); // Ensure there were no publish failures Assert.Empty(await harness.FailedAsync()); diff --git a/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs b/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs new file mode 100644 index 00000000..d258c43e --- /dev/null +++ b/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs @@ -0,0 +1,78 @@ +using Microsoft.EntityFrameworkCore; +using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; +using Tingle.Dependabot.Models.Dependabot; +using Tingle.Dependabot.Models.Management; +using Tingle.EventBus; +using Tingle.PeriodicTasks; + +namespace Tingle.Dependabot.PeriodicTasks; + +internal class MissedTriggerCheckerTask : IPeriodicTask +{ + private readonly MainDbContext dbContext; + private readonly IEventPublisher publisher; + private readonly ILogger logger; + + public MissedTriggerCheckerTask(MainDbContext dbContext, IEventPublisher publisher, ILogger logger) + { + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); + this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(PeriodicTaskExecutionContext context, CancellationToken cancellationToken) + { + await CheckAsync(DateTimeOffset.UtcNow, cancellationToken); + } + + internal virtual async Task CheckAsync(DateTimeOffset referencePoint, CancellationToken cancellationToken = default) + { + var repositories = await dbContext.Repositories.ToListAsync(cancellationToken); + + foreach (var repository in repositories) + { + foreach (var update in repository.Updates) + { + var schedule = (CronSchedule)update.Schedule!.GenerateCron(); + var timezone = TimeZoneInfo.FindSystemTimeZoneById(update.Schedule.Timezone); + + // check if we missed an execution + var latestUpdate = update.LatestUpdate; + var missed = latestUpdate is null; // when null, it was missed + if (latestUpdate != null) + { + var nextFromLast = schedule.GetNextOccurrence(latestUpdate.Value, timezone); + if (nextFromLast is null) continue; + + var nextFromReference = schedule.GetNextOccurrence(referencePoint, timezone); + if (nextFromReference is null) continue; + + missed = nextFromLast.Value <= referencePoint; // when next is in the past, it was missed + + // for daily schedules, only check if the next is more than 12 hours away + if (missed && update.Schedule.Interval is DependabotScheduleInterval.Daily) + { + missed = (nextFromReference.Value - referencePoint).Hours > 12; + } + } + + // if we missed an execution, trigger one + if (missed) + { + logger.LogWarning("Schedule was missed for {RepositoryId}({UpdateId}). Triggering now", repository.Id, repository.Updates.IndexOf(update)); + + // publish event for the job to be run + var evt = new TriggerUpdateJobsEvent + { + RepositoryId = repository.Id, + RepositoryUpdateId = repository.Updates.IndexOf(update), + Trigger = UpdateJobTrigger.MissedSchedule, + }; + + await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + } + } + } + } +} diff --git a/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs b/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs new file mode 100644 index 00000000..6ccff61d --- /dev/null +++ b/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs @@ -0,0 +1,27 @@ +using Tingle.Dependabot.Events; +using Tingle.EventBus; +using Tingle.PeriodicTasks; + +namespace Tingle.Dependabot.Workflow; + +internal class SynchronizationTask : IPeriodicTask +{ + private readonly IEventPublisher publisher; + + public SynchronizationTask(IEventPublisher publisher) + { + this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + } + + public async Task ExecuteAsync(PeriodicTaskExecutionContext context, CancellationToken cancellationToken) + { + await SyncAsync(cancellationToken); + } + + internal virtual async Task SyncAsync(CancellationToken cancellationToken = default) + { + // request synchronization of the whole project via events + var evt = new ProcessSynchronization(false); /* database sync should not trigger, just in case it's too many */ + await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + } +} diff --git a/server/Tingle.Dependabot/PeriodicTasks/UpdateJobsCleanerTask.cs b/server/Tingle.Dependabot/PeriodicTasks/UpdateJobsCleanerTask.cs new file mode 100644 index 00000000..af0896da --- /dev/null +++ b/server/Tingle.Dependabot/PeriodicTasks/UpdateJobsCleanerTask.cs @@ -0,0 +1,61 @@ +using Microsoft.EntityFrameworkCore; +using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; +using Tingle.Dependabot.Models.Management; +using Tingle.EventBus; +using Tingle.PeriodicTasks; + +namespace Tingle.Dependabot.PeriodicTasks; + +internal class UpdateJobsCleanerTask : IPeriodicTask +{ + private readonly MainDbContext dbContext; + private readonly IEventPublisher publisher; + private readonly ILogger logger; + + public UpdateJobsCleanerTask(MainDbContext dbContext, IEventPublisher publisher, ILogger logger) + { + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); + this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(PeriodicTaskExecutionContext context, CancellationToken cancellationToken) + { + await CleanupAsync(cancellationToken); + } + + internal virtual async Task CleanupAsync(CancellationToken cancellationToken = default) + { + // resolve pending jobs + + // Change this to 3 hours once we have figured out how to get events from Azure + var oldest = DateTimeOffset.UtcNow.AddMinutes(-10); // older than 10 minutes + var jobs = await (from j in dbContext.UpdateJobs + where j.Created <= oldest + where j.Status == UpdateJobStatus.Scheduled || j.Status == UpdateJobStatus.Running + orderby j.Created ascending + select j).Take(100).ToListAsync(cancellationToken); + + if (jobs.Count > 0) + { + logger.LogInformation("Found {Count} jobs that are still pending for more than 10 min. Requesting manual resolution ...", jobs.Count); + + var events = jobs.Select(j => new UpdateJobCheckStateEvent { JobId = j.Id, }).ToList(); + await publisher.PublishAsync(events, cancellationToken: cancellationToken); + } + + // delete old jobs + var cutoff = DateTimeOffset.UtcNow.AddDays(-90); + jobs = await (from j in dbContext.UpdateJobs + where j.Created <= cutoff + orderby j.Created ascending + select j).Take(100).ToListAsync(cancellationToken); + if (jobs.Count > 0) + { + dbContext.UpdateJobs.RemoveRange(jobs); + await dbContext.SaveChangesAsync(cancellationToken); + logger.LogInformation("Removed {Count} jobs that older than {Cutoff}", jobs.Count, cutoff); + } + } +} diff --git a/server/Tingle.Dependabot/Program.cs b/server/Tingle.Dependabot/Program.cs index faf0243a..2fa171bf 100644 --- a/server/Tingle.Dependabot/Program.cs +++ b/server/Tingle.Dependabot/Program.cs @@ -7,6 +7,7 @@ using Tingle.Dependabot; using Tingle.Dependabot.Consumers; using Tingle.Dependabot.Models; +using Tingle.Dependabot.PeriodicTasks; using Tingle.Dependabot.Workflow; var builder = WebApplication.CreateBuilder(args); @@ -75,7 +76,6 @@ builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); -builder.Services.AddHostedService(); // Add event bus var selectedTransport = builder.Configuration.GetValue("EventBus:SelectedTransport"); @@ -100,6 +100,13 @@ } }); +builder.Services.AddPeriodicTasks(builder => +{ + builder.AddTask(schedule: "0 * * * *"); // every hour + builder.AddTask(schedule: "*/15 * * * *"); // every 15 minutes + builder.AddTask(schedule: "23 */6 * * *"); // every 6 hours at minute 23 +}); + // Add health checks builder.Services.AddHealthChecks() .AddDbContextCheck(); diff --git a/server/Tingle.Dependabot/Workflow/WorkflowBackgroundService.cs b/server/Tingle.Dependabot/Workflow/WorkflowBackgroundService.cs deleted file mode 100644 index 36c5bd2c..00000000 --- a/server/Tingle.Dependabot/Workflow/WorkflowBackgroundService.cs +++ /dev/null @@ -1,165 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using Tingle.Dependabot.Events; -using Tingle.Dependabot.Models; -using Tingle.Dependabot.Models.Dependabot; -using Tingle.Dependabot.Models.Management; -using Tingle.EventBus; -using Tingle.PeriodicTasks; - -namespace Tingle.Dependabot.Workflow; - -internal class WorkflowBackgroundService : BackgroundService -{ - private readonly IServiceProvider serviceProvider; - private readonly ILogger logger; - - public WorkflowBackgroundService(IServiceProvider serviceProvider, ILogger logger) - { - this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - var t_synch = SynchronizationAsync(stoppingToken); - var t_missed = CheckMissedTriggerAsync(stoppingToken); - var t_cleanup = CleanupAsync(stoppingToken); - - await Task.WhenAll(t_synch, t_missed, t_cleanup); - } - - internal virtual async Task SynchronizationAsync(CancellationToken cancellationToken = default) - { - var timer = new PeriodicTimer(TimeSpan.FromHours(6)); - - while (!cancellationToken.IsCancellationRequested) - { - await timer.WaitForNextTickAsync(cancellationToken); - await SynchronizationInnerAsync(cancellationToken); - } - } - - internal virtual async Task SynchronizationInnerAsync(CancellationToken cancellationToken = default) - { - // request synchronization of the whole project via events - using var scope = serviceProvider.CreateScope(); - var provider = scope.ServiceProvider; - var publisher = provider.GetRequiredService(); - var evt = new ProcessSynchronization(false); /* database sync should not trigger, just in case it's too many */ - await publisher.PublishAsync(evt, cancellationToken: cancellationToken); - } - - internal virtual Task CheckMissedTriggerAsync(CancellationToken cancellationToken = default) => CheckMissedTriggerAsync(DateTimeOffset.UtcNow, cancellationToken); - internal virtual async Task CheckMissedTriggerAsync(DateTimeOffset referencePoint, CancellationToken cancellationToken = default) - { - var timer = new PeriodicTimer(TimeSpan.FromHours(1)); - - while (!cancellationToken.IsCancellationRequested) - { - await timer.WaitForNextTickAsync(cancellationToken); - await CheckMissedTriggerInnerAsync(referencePoint, cancellationToken); - } - } - internal virtual async Task CheckMissedTriggerInnerAsync(DateTimeOffset referencePoint, CancellationToken cancellationToken = default) - { - using var scope = serviceProvider.CreateScope(); - var provider = scope.ServiceProvider; - var dbContext = provider.GetRequiredService(); - var publisher = provider.GetRequiredService(); - var repositories = await dbContext.Repositories.ToListAsync(cancellationToken); - - foreach (var repository in repositories) - { - foreach (var update in repository.Updates) - { - var schedule = (CronSchedule)update.Schedule!.GenerateCron(); - var timezone = TimeZoneInfo.FindSystemTimeZoneById(update.Schedule.Timezone); - - // check if we missed an execution - var latestUpdate = update.LatestUpdate; - var missed = latestUpdate is null; // when null, it was missed - if (latestUpdate != null) - { - var nextFromLast = schedule.GetNextOccurrence(latestUpdate.Value, timezone); - if (nextFromLast is null) continue; - - var nextFromReference = schedule.GetNextOccurrence(referencePoint, timezone); - if (nextFromReference is null) continue; - - missed = nextFromLast.Value <= referencePoint; // when next is in the past, it was missed - - // for daily schedules, only check if the next is more than 12 hours away - if (missed && update.Schedule.Interval is DependabotScheduleInterval.Daily) - { - missed = (nextFromReference.Value - referencePoint).Hours > 12; - } - } - - // if we missed an execution, trigger one - if (missed) - { - logger.LogWarning("Schedule was missed for {RepositoryId}({UpdateId}). Triggering now", repository.Id, repository.Updates.IndexOf(update)); - - // publish event for the job to be run - var evt = new TriggerUpdateJobsEvent - { - RepositoryId = repository.Id, - RepositoryUpdateId = repository.Updates.IndexOf(update), - Trigger = UpdateJobTrigger.MissedSchedule, - }; - - await publisher.PublishAsync(evt, cancellationToken: cancellationToken); - } - } - } - } - - internal virtual async Task CleanupAsync(CancellationToken cancellationToken = default) - { - var timer = new PeriodicTimer(TimeSpan.FromMinutes(15)); // change to once per hour once we move to jobs in Azure ContainerApps - - while (!cancellationToken.IsCancellationRequested) - { - await timer.WaitForNextTickAsync(cancellationToken); - await CleanupInnerAsync(cancellationToken); - } - } - internal virtual async Task CleanupInnerAsync(CancellationToken cancellationToken = default) - { - using var scope = serviceProvider.CreateScope(); - var provider = scope.ServiceProvider; - var dbContext = provider.GetRequiredService(); - var publisher = provider.GetRequiredService(); - - // resolve pending jobs - - // Change this to 3 hours once we have figured out how to get events from Azure - var oldest = DateTimeOffset.UtcNow.AddMinutes(-10); // older than 10 minutes - var jobs = await (from j in dbContext.UpdateJobs - where j.Created <= oldest - where j.Status == UpdateJobStatus.Scheduled || j.Status == UpdateJobStatus.Running - orderby j.Created ascending - select j).Take(100).ToListAsync(cancellationToken); - - if (jobs.Count > 0) - { - logger.LogInformation("Found {Count} jobs that are still pending for more than 10 min. Requesting manual resolution ...", jobs.Count); - - var events = jobs.Select(j => new UpdateJobCheckStateEvent { JobId = j.Id, }).ToList(); - await publisher.PublishAsync(events, cancellationToken: cancellationToken); - } - - // delete old jobs - var cutoff = DateTimeOffset.UtcNow.AddDays(-90); - jobs = await (from j in dbContext.UpdateJobs - where j.Created <= cutoff - orderby j.Created ascending - select j).Take(100).ToListAsync(cancellationToken); - if (jobs.Count > 0) - { - dbContext.UpdateJobs.RemoveRange(jobs); - await dbContext.SaveChangesAsync(cancellationToken); - logger.LogInformation("Removed {Count} jobs that older than {Cutoff}", jobs.Count, cutoff); - } - } -}