diff --git a/docs/server.md b/docs/server.md index 616acc77..7916a2b8 100644 --- a/docs/server.md +++ b/docs/server.md @@ -51,18 +51,10 @@ The deployment exposes the following parameters that can be tuned to suit the se |Parameter Name|Remarks|Required|Default| |--|--|--|--| -|projectUrl|The URL of the Azure DevOps project or collection. For example `https://dev.azure.com/fabrikam/DefaultCollection`. This URL must be accessible from the network that the deployment is done in. You can modify the deployment to be done in an private network but you are on your own there.|Yes|**none**| -|projectToken|Personal Access Token (PAT) for accessing the Azure DevOps project. The required permissions are:
- Code (Full)
- Pull Requests Threads (Read & Write).
- Notifications (Read, Write & Manage).
See the [documentation](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#create-a-pat) to know more about creating a Personal Access Token|Yes|**none**| |location|Location to deploy the resources.|No|<resource-group-location>| |name|The name of all resources.|No|`dependabot`| -|synchronizeOnStartup|Whether to synchronize repositories on startup. This option is useful for initial deployments since the server synchronizes every 6 hours. Leaving it on has no harm, it actually helps you find out if the token works based on the logs.|No|false| -|createOrUpdateWebhooksOnStartup|Whether to create or update Azure DevOps subscriptions on startup. This is required if you want configuration files to be picked up automatically and other event driven functionality.
When this is set to `true`, ensure the value provided for `projectToken` has permissions for service hooks and the owner is a Project Administrator. Leaving this on has no harm because the server will only create new subscription if there are no existing ones based on the URL.|No|false| +|projectSetups|A JSON array string representing the projects to be setup on startup. This is useful when running your own setup. Example: `[{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]`| |githubToken|Access token for authenticating requests to GitHub. Required for vulnerability checks and to avoid rate limiting on free requests|No|<empty>| -|autoComplete|Whether to set auto complete on created pull requests.|No|true| -|autoCompleteIgnoreConfigs|Identifiers of configs to be ignored in auto complete. E.g 3,4,10|No|<empty>| -|autoCompleteMergeStrategy|Merge strategy to use when setting auto complete on created pull requests. Allowed values: `NoFastForward`, `Rebase`, `RebaseMerge`, or `Squash`|No|`Squash`| -|autoApprove|Whether to automatically approve created pull requests.|No|false| -|notificationsPassword|The password used to authenticate incoming requests from Azure DevOps|No|<auto-generated>| |imageTag|The image tag to use when pulling the docker containers. A tag also defines the version. You should avoid using `latest`. Example: `1.1.0`|No|<version-downloaded>| |minReplicas|The minimum number of replicas to required for the deployment. Given that scheduling runs in process, this value cannot be less than `1`. This may change in the future.|No|1| |maxReplicas|The maximum number of replicas when automatic scaling engages. In most cases, you do not need more than 1.|No|1| @@ -78,8 +70,6 @@ For a one time deployment, it is similar to how you deploy other resources on Az ```bash az deployment group create --resource-group DEPENDABOT \ --template-file main.bicep \ - --parameters projectUrl= \ - --parameters projectToken= \ --parameters githubToken= \ --confirm-with-what-if ``` @@ -104,15 +94,6 @@ The parameters file (`dependabot.parameters.json`): "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { - "projectUrl": { - "value": "#{System_TeamFoundationCollectionUri}##{System_TeamProject}#" - }, - "projectToken": { - "value": "#{DependabotProjectToken}#" - }, - "autoComplete": { - "value": true - }, "githubToken": { "value": "#{DependabotGithubToken}#" }, diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs index d4810ec7..df2285f2 100644 --- a/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs @@ -16,6 +16,7 @@ namespace Tingle.Dependabot.Tests.PeriodicTasks; public class MissedTriggerCheckerTaskTests { + private const string ProjectId = "prj_1234567890"; private const string RepositoryId = "repo_1234567890"; private const int UpdateId1 = 1; @@ -103,9 +104,20 @@ private async Task TestAsync(DateTimeOffset? lastUpdate0, DateTimeOffset? lastUp var context = provider.GetRequiredService(); await context.Database.EnsureCreatedAsync(); + await context.Projects.AddAsync(new Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); await context.Repositories.AddAsync(new Repository { Id = RepositoryId, + ProjectId = ProjectId, + ProviderId = Guid.NewGuid().ToString(), Name = "test-repo", ConfigFileContents = "", Updates = new List diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs index 37a256e3..105f57c5 100644 --- a/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs @@ -1,7 +1,10 @@ -using Microsoft.Extensions.DependencyInjection; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; +using Tingle.Dependabot.Models.Management; using Tingle.Dependabot.Workflow; using Tingle.EventBus; using Tingle.EventBus.Transports.InMemory; @@ -12,6 +15,8 @@ namespace Tingle.Dependabot.Tests.PeriodicTasks; public class SynchronizationTaskTests { + private const string ProjectId = "prj_1234567890"; + private readonly ITestOutputHelper outputHelper; public SynchronizationTaskTests(ITestOutputHelper outputHelper) @@ -42,6 +47,12 @@ private async Task TestAsync(Func builder.AddXUnit(outputHelper)) .ConfigureServices((context, services) => { + var dbName = Guid.NewGuid().ToString(); + services.AddDbContext(options => + { + options.UseInMemoryDatabase(dbName, o => o.EnableNullChecks()); + options.EnableDetailedErrors(); + }); services.AddEventBus(builder => builder.AddInMemoryTransport().AddInMemoryTestHarness()); }) .Build(); @@ -49,6 +60,20 @@ private async Task TestAsync(Func(); + await context.Database.EnsureCreatedAsync(); + + await context.Projects.AddAsync(new Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); + await context.SaveChangesAsync(); + var harness = provider.GetRequiredService(); await harness.StartAsync(); diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs index 997dafcf..241ac77f 100644 --- a/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs @@ -16,6 +16,7 @@ namespace Tingle.Dependabot.Tests.PeriodicTasks; public class UpdateJobsCleanerTaskTests { + private const string ProjectId = "prj_1234567890"; private const string RepositoryId = "repo_1234567890"; private readonly ITestOutputHelper outputHelper; @@ -34,6 +35,7 @@ await TestAsync(async (harness, context, pt) => await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddMinutes(-19), @@ -46,6 +48,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddHours(-100), @@ -58,6 +61,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = targetId, + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddMinutes(-30), @@ -87,6 +91,7 @@ await TestAsync(async (harness, context, pt) => await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddDays(-80), @@ -98,6 +103,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddDays(-100), @@ -109,6 +115,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddDays(-120), @@ -146,9 +153,20 @@ private async Task TestAsync(Func(); await context.Database.EnsureCreatedAsync(); + await context.Projects.AddAsync(new Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); await context.Repositories.AddAsync(new Repository { Id = RepositoryId, + ProjectId = ProjectId, + ProviderId = Guid.NewGuid().ToString(), Name = "test-repo", ConfigFileContents = "", Updates = new List diff --git a/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs b/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs index 4570f07f..640c6a65 100644 --- a/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs +++ b/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs @@ -3,7 +3,6 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System.Net; @@ -20,6 +19,8 @@ namespace Tingle.Dependabot.Tests; public class WebhooksControllerIntegrationTests { + private const string ProjectId = "prj_1234567890"; + private readonly ITestOutputHelper outputHelper; public WebhooksControllerIntegrationTests(ITestOutputHelper outputHelper) @@ -41,7 +42,7 @@ await TestAsync(async (harness, client) => // password does not match what is on record request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump5"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump5"))); response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); Assert.Empty(await response.Content.ReadAsStringAsync()); @@ -55,7 +56,7 @@ public async Task Returns_BadRequest_NoBody() await TestAsync(async (harness, client) => { var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StringContent("", Encoding.UTF8, "application/json"); var response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); @@ -74,7 +75,7 @@ public async Task Returns_BadRequest_MissingValues() await TestAsync(async (harness, client) => { var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StringContent("{}", Encoding.UTF8, "application/json"); var response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); @@ -96,7 +97,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestUpdated1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); var response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.UnsupportedMediaType, response.StatusCode); @@ -115,7 +116,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsGitPush1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -139,7 +140,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestUpdated1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -156,7 +157,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestMerged1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -173,7 +174,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestCommentEvent1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -188,13 +189,6 @@ private async Task TestAsync(Func execute // Arrange var builder = new WebHostBuilder() .ConfigureLogging(builder => builder.AddXUnit(outputHelper)) - .ConfigureAppConfiguration(builder => - { - builder.AddInMemoryCollection(new Dictionary - { - ["Authentication:Schemes:ServiceHooks:Credentials:vsts"] = "burp-bump", - }); - }) .ConfigureServices((context, services) => { services.AddControllers() @@ -247,6 +241,17 @@ private async Task TestAsync(Func execute var context = provider.GetRequiredService(); await context.Database.EnsureCreatedAsync(); + await context.Projects.AddAsync(new Dependabot.Models.Management.Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); + await context.SaveChangesAsync(); + var harness = provider.GetRequiredService(); await harness.StartAsync(); diff --git a/server/Tingle.Dependabot/AppSetup.cs b/server/Tingle.Dependabot/AppSetup.cs index 0e2f51d5..a1ed866b 100644 --- a/server/Tingle.Dependabot/AppSetup.cs +++ b/server/Tingle.Dependabot/AppSetup.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; +using System.Security.Cryptography; +using System.Text.Json; using Tingle.Dependabot.Models; using Tingle.Dependabot.Workflow; @@ -7,6 +8,19 @@ namespace Tingle.Dependabot; internal static class AppSetup { + private class ProjectSetupInfo + { + public required Uri Url { get; set; } + public required string Token { get; set; } + public bool AutoComplete { get; set; } + public List? AutoCompleteIgnoreConfigs { get; set; } + public MergeStrategy? AutoCompleteMergeStrategy { get; set; } + public bool AutoApprove { get; set; } + public Dictionary Secrets { get; set; } = new(StringComparer.OrdinalIgnoreCase); + } + + private static readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + public static async Task SetupAsync(WebApplication app, CancellationToken cancellationToken = default) { using var scope = app.Services.CreateScope(); @@ -22,30 +36,91 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel } } - var options = provider.GetRequiredService>().Value; - if (options.SynchronizeOnStartup) + // parse projects to be setup + var setupsJson = app.Configuration.GetValue("PROJECT_SETUPS"); + var setups = new List(); + if (!string.IsNullOrWhiteSpace(setupsJson)) { - var synchronizer = provider.GetRequiredService(); - await synchronizer.SynchronizeAsync(false, cancellationToken); /* database sync should not trigger, just in case it's too many */ + setups = JsonSerializer.Deserialize>(setupsJson, serializerOptions)!; + } + + // add projects if there are projects to be added + var adoProvider = provider.GetRequiredService(); + var context = provider.GetRequiredService(); + var projects = await context.Projects.ToListAsync(cancellationToken); + foreach (var setup in setups) + { + var url = (AzureDevOpsProjectUrl)setup.Url; + var project = projects.SingleOrDefault(p => new Uri(p.Url!) == setup.Url); + if (project is null) + { + project = new Models.Management.Project + { + Id = Guid.NewGuid().ToString("n"), + Created = DateTimeOffset.UtcNow, + Password = GeneratePassword(32), + Url = setup.Url.ToString(), + Type = Models.Management.ProjectType.Azure, + }; + await context.Projects.AddAsync(project, cancellationToken); + } + + // update project using values from the setup + project.Token = setup.Token; + project.AutoComplete.Enabled = setup.AutoComplete; + project.AutoComplete.IgnoreConfigs = setup.AutoCompleteIgnoreConfigs; + project.AutoComplete.MergeStrategy = setup.AutoCompleteMergeStrategy; + project.AutoApprove.Enabled = setup.AutoApprove; + project.Secrets = setup.Secrets; + + // update values from the project + var tp = await adoProvider.GetProjectAsync(project, cancellationToken); + project.ProviderId = tp.Id.ToString(); + project.Name = tp.Name; + project.Private = tp.Visibility is not Models.Azure.AzdoProjectVisibility.Public; + + // if there are changes, set the Updated field + if (context.ChangeTracker.HasChanges()) + { + project.Updated = DateTimeOffset.UtcNow; + } + } + + // update database and list of projects + var updated = await context.SaveChangesAsync(cancellationToken); + projects = updated > 0 ? await context.Projects.ToListAsync(cancellationToken) : projects; + + // synchronize and create/update subscriptions if we have setups + var synchronizer = provider.GetRequiredService(); + if (setups.Count > 0) + { + foreach (var project in projects) + { + // synchronize project + await synchronizer.SynchronizeAsync(project, false, cancellationToken); /* database sync should not trigger, just in case it's too many */ + + // create or update webhooks/subscriptions + await adoProvider.CreateOrUpdateSubscriptionsAsync(project, cancellationToken); + } } // skip loading schedules if told to if (!app.Configuration.GetValue("SKIP_LOAD_SCHEDULES")) { - var dbContext = provider.GetRequiredService(); - var repositories = await dbContext.Repositories.ToListAsync(cancellationToken); + var repositories = await context.Repositories.ToListAsync(cancellationToken); var scheduler = provider.GetRequiredService(); foreach (var repository in repositories) { await scheduler.CreateOrUpdateAsync(repository, cancellationToken); } } + } - // create or update webhooks/subscriptions if asked to - if (options.CreateOrUpdateWebhooksOnStartup) - { - var adoProvider = provider.GetRequiredService(); - await adoProvider.CreateOrUpdateSubscriptionsAsync(cancellationToken); - } + private static string GeneratePassword(int length = 32) + { + var data = new byte[length]; + using var rng = RandomNumberGenerator.Create(); + rng.GetBytes(data); + return Convert.ToBase64String(data); } } diff --git a/server/Tingle.Dependabot/ApplicationInsights/DependabotTelemetryInitializer.cs b/server/Tingle.Dependabot/ApplicationInsights/DependabotTelemetryInitializer.cs new file mode 100644 index 00000000..1512131d --- /dev/null +++ b/server/Tingle.Dependabot/ApplicationInsights/DependabotTelemetryInitializer.cs @@ -0,0 +1,27 @@ +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; + +namespace Tingle.Dependabot.ApplicationInsights; + +internal class DependabotTelemetryInitializer : ITelemetryInitializer +{ + private const string KeyProjectId = "ProjectId"; + + private readonly IHttpContextAccessor httpContextAccessor; + + public DependabotTelemetryInitializer(IHttpContextAccessor httpContextAccessor) + { + this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); + } + + public void Initialize(ITelemetry telemetry) + { + var context = httpContextAccessor.HttpContext; + if (context is null || telemetry is not RequestTelemetry rt) return; // ensure we have a context and the telemetry is for a request + + // add properties + var props = rt.Properties; + props.TryAddIfNotDefault(KeyProjectId, context.GetProjectId()); + } +} diff --git a/server/Tingle.Dependabot/ApplicationInsights/InsightsFilteringProcessor.cs b/server/Tingle.Dependabot/ApplicationInsights/InsightsFilteringProcessor.cs new file mode 100644 index 00000000..58eb4630 --- /dev/null +++ b/server/Tingle.Dependabot/ApplicationInsights/InsightsFilteringProcessor.cs @@ -0,0 +1,58 @@ +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; + +namespace Tingle.Dependabot.ApplicationInsights; + +/// +/// Implementation of that filters out unneeded telemetry. +/// +internal class InsightsFilteringProcessor : ITelemetryProcessor +{ + private static readonly string[] excludedRequestNames = + { + "ServiceBusReceiver.Receive", + "ServiceBusProcessor.ProcessMessage", + }; + + private readonly ITelemetryProcessor next; + + public InsightsFilteringProcessor(ITelemetryProcessor next) + { + this.next = next; + } + + /// + public void Process(ITelemetry item) + { + // Skip unneeded RequestTelemetry + if (item is RequestTelemetry rt) + { + // Skip known request names + if (rt.Name is not null && excludedRequestNames.Contains(rt.Name, StringComparer.OrdinalIgnoreCase)) + { + return; // terminate the processor pipeline + } + + // Skip requests for /health and /liveness because they are better diagnosed via logs + var path = rt.Url?.AbsolutePath; + if (string.Equals(path, "/health", StringComparison.OrdinalIgnoreCase) + || string.Equals(path, "/liveness", StringComparison.OrdinalIgnoreCase)) + { + return; // terminate the processor pipeline + } + } + + // Skip requests sent to the orchestrator to find the details of a pod/host + // Sometimes they fail, like when the service is starting up + if (item is DependencyTelemetry dt + && string.Equals("http", dt.Type, StringComparison.OrdinalIgnoreCase) + && string.Equals("10.0.0.1", dt.Target, StringComparison.OrdinalIgnoreCase)) + { + return; // terminate the processor pipeline + } + + // process all the others + next.Process(item); + } +} diff --git a/server/Tingle.Dependabot/ApplicationInsights/InsightsShutdownFlushService.cs b/server/Tingle.Dependabot/ApplicationInsights/InsightsShutdownFlushService.cs new file mode 100644 index 00000000..6b1de3be --- /dev/null +++ b/server/Tingle.Dependabot/ApplicationInsights/InsightsShutdownFlushService.cs @@ -0,0 +1,30 @@ +using Microsoft.ApplicationInsights; + +namespace Tingle.Dependabot.ApplicationInsights; + +// from https://medium.com/@asimmon/prevent-net-application-insights-telemetry-loss-d82a06c3673f +internal class InsightsShutdownFlushService : IHostedService +{ + private readonly TelemetryClient telemetryClient; + + public InsightsShutdownFlushService(TelemetryClient telemetryClient) + { + this.telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + } + + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public async Task StopAsync(CancellationToken cancellationToken) + { + // Flush the remaining telemetry data when application shutdown is requested. + // Using "CancellationToken.None" ensures that the application doesn't stop until the telemetry data is flushed. + // + // If you want to use the "cancellationToken" argument, make sure to configure "HostOptions.ShutdownTimeout" with a sufficiently large duration, + // and silence the eventual "OperationCanceledException" exception. Otherwise, you will still be at risk of loosing telemetry data. + var successfullyFlushed = await telemetryClient.FlushAsync(CancellationToken.None); + if (!successfullyFlushed) + { + // Here you can handle th case where transfer of telemetry data to the server has failed with HTTP status that cannot be retried. + } + } +} diff --git a/server/Tingle.Dependabot/BasicUserValidationService.cs b/server/Tingle.Dependabot/BasicUserValidationService.cs index ca50b7ad..81ace82a 100644 --- a/server/Tingle.Dependabot/BasicUserValidationService.cs +++ b/server/Tingle.Dependabot/BasicUserValidationService.cs @@ -1,19 +1,21 @@ using AspNetCore.Authentication.Basic; +using Microsoft.EntityFrameworkCore; +using Tingle.Dependabot.Models; namespace Tingle.Dependabot; internal class BasicUserValidationService : IBasicUserValidationService { - private readonly IConfiguration configuration; + private readonly MainDbContext dbContext; - public BasicUserValidationService(IConfiguration configuration) + public BasicUserValidationService(MainDbContext dbContext) { - this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); } - public Task IsValidAsync(string username, string password) + public async Task IsValidAsync(string username, string password) { - var expected = configuration.GetValue($"Authentication:Schemes:ServiceHooks:Credentials:{username}"); - return Task.FromResult(string.Equals(expected, password, StringComparison.Ordinal)); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == username); + return project is not null && string.Equals(project.Password, password, StringComparison.Ordinal); } } diff --git a/server/Tingle.Dependabot/Constants.cs b/server/Tingle.Dependabot/Constants.cs index 82377473..6b34147e 100644 --- a/server/Tingle.Dependabot/Constants.cs +++ b/server/Tingle.Dependabot/Constants.cs @@ -15,4 +15,15 @@ internal static class AuthConstants internal static class ErrorCodes { internal const string FeaturesDisabled = "features_disabled"; + internal const string ProjectNotFound = "project_not_found"; + internal const string RepositoryNotFound = "repository_not_found"; + internal const string RepositoryUpdateNotFound = "repository_update_not_found"; +} + +internal static class FeatureNames +{ + internal const string DebugAllJobs = "DebugAllJobs"; // Whether to debug all jobs (controls environment variable value). + internal const string DebugJobs = "DebugJobs"; // Whether to debug jobs (controls value in job definition). + internal const string DeterministicUpdates = "DeterministicUpdates"; // Whether updates should be created in the same order. + internal const string UpdaterV2 = "UpdaterV2"; // Whether to use V2 updater } diff --git a/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs b/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs index 4676e446..2f397cd8 100644 --- a/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs +++ b/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs @@ -22,29 +22,37 @@ public ProcessSynchronizationConsumer(MainDbContext dbContext, Synchronizer sync public async Task ConsumeAsync(EventContext context, CancellationToken cancellationToken = default) { var evt = context.Event; - var trigger = evt.Trigger; + // ensure project exists + var projectId = evt.ProjectId ?? throw new InvalidOperationException($"'{nameof(evt.ProjectId)}' cannot be null"); + var project = await dbContext.Projects.SingleOrDefaultAsync(r => r.Id == projectId, cancellationToken); + if (project is null) + { + logger.LogWarning("Skipping trigger for update because project '{Project}' does not exist.", projectId); + return; + } + if (evt.RepositoryId is not null) { // ensure repository exists var repositoryId = evt.RepositoryId ?? throw new InvalidOperationException($"'{nameof(evt.RepositoryId)}' cannot be null"); - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == repositoryId, cancellationToken); + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == repositoryId, cancellationToken); if (repository is null) { logger.LogWarning("Skipping synchronization because repository '{Repository}' does not exist.", repositoryId); return; } - await synchronizer.SynchronizeAsync(repository, trigger, cancellationToken); + await synchronizer.SynchronizeAsync(project, repository, trigger, cancellationToken); } else if (evt.RepositoryProviderId is not null) { - await synchronizer.SynchronizeAsync(repositoryProviderId: evt.RepositoryProviderId, trigger, cancellationToken); + await synchronizer.SynchronizeAsync(project, repositoryProviderId: evt.RepositoryProviderId, trigger, cancellationToken); } else { - await synchronizer.SynchronizeAsync(evt.Trigger, cancellationToken); + await synchronizer.SynchronizeAsync(project, evt.Trigger, cancellationToken); } } } diff --git a/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs b/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs index a2a923ae..dec830fd 100644 --- a/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs +++ b/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs @@ -24,6 +24,15 @@ public async Task ConsumeAsync(EventContext context, Can { var evt = context.Event; + // ensure project exists + var projectId = evt.ProjectId ?? throw new InvalidOperationException($"'{nameof(evt.ProjectId)}' cannot be null"); + var project = await dbContext.Projects.SingleOrDefaultAsync(r => r.Id == projectId, cancellationToken); + if (project is null) + { + logger.LogWarning("Skipping trigger for update because project '{Project}' does not exist.", projectId); + return; + } + // ensure repository exists var repositoryId = evt.RepositoryId ?? throw new InvalidOperationException($"'{nameof(evt.RepositoryId)}' cannot be null"); var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == repositoryId, cancellationToken); @@ -80,6 +89,7 @@ public async Task ConsumeAsync(EventContext context, Can Status = UpdateJobStatus.Scheduled, Trigger = evt.Trigger, + ProjectId = project.Id, RepositoryId = repository.Id, RepositorySlug = repository.Slug, EventBusId = eventBusId, @@ -94,6 +104,7 @@ public async Task ConsumeAsync(EventContext context, Can End = null, Duration = null, Log = null, + Error = null, }; await dbContext.UpdateJobs.AddAsync(job, cancellationToken); @@ -107,7 +118,7 @@ public async Task ConsumeAsync(EventContext context, Can } // call the update runner to run the update - await updateRunner.CreateAsync(repository, update, job, cancellationToken); + await updateRunner.CreateAsync(project, repository, update, job, cancellationToken); // save changes that may have been made by the updateRunner update.LatestJobStatus = job.Status; diff --git a/server/Tingle.Dependabot/Controllers/ManagementController.cs b/server/Tingle.Dependabot/Controllers/ManagementController.cs index 513013a2..b25ffb31 100644 --- a/server/Tingle.Dependabot/Controllers/ManagementController.cs +++ b/server/Tingle.Dependabot/Controllers/ManagementController.cs @@ -29,8 +29,13 @@ public ManagementController(MainDbContext dbContext, IEventPublisher publisher, [HttpPost("sync")] public async Task SyncAsync([FromBody] SynchronizationRequest model) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // request synchronization of the project - var evt = new ProcessSynchronization(model.Trigger); + var evt = new ProcessSynchronization(projectId, model.Trigger); await publisher.PublishAsync(evt); return Ok(); @@ -39,33 +44,50 @@ public async Task SyncAsync([FromBody] SynchronizationRequest mod [HttpPost("/webhooks/register")] public async Task WebhooksRegisterAsync() { - await adoProvider.CreateOrUpdateSubscriptionsAsync(); + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + + await adoProvider.CreateOrUpdateSubscriptionsAsync(project); return Ok(); } [HttpGet("repos")] public async Task GetReposAsync() { - var repos = await dbContext.Repositories.ToListAsync(); + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + + var repos = await dbContext.Repositories.Where(r => r.ProjectId == project.Id).ToListAsync(); return Ok(repos); } [HttpGet("repos/{id}")] public async Task GetRepoAsync([FromRoute, Required] string id) { - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); return Ok(repository); } [HttpGet("repos/{id}/jobs/{jobId}")] public async Task GetJobAsync([FromRoute, Required] string id, [FromRoute, Required] string jobId) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure repository exists - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); - if (repository is null) - { - return Problem(title: "repository_not_found", statusCode: 400); - } + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); + if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400); // find the job var job = dbContext.UpdateJobs.Where(j => j.RepositoryId == repository.Id && j.Id == jobId).SingleOrDefaultAsync(); @@ -75,15 +97,17 @@ public async Task GetJobAsync([FromRoute, Required] string id, [F [HttpPost("repos/{id}/sync")] public async Task SyncRepoAsync([FromRoute, Required] string id, [FromBody] SynchronizationRequest model) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure repository exists - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); - if (repository is null) - { - return Problem(title: "repository_not_found", statusCode: 400); - } + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); + if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400); // request synchronization of the repository - var evt = new ProcessSynchronization(model.Trigger, repositoryId: repository.Id, null); + var evt = new ProcessSynchronization(projectId, model.Trigger, repositoryId: repository.Id, null); await publisher.PublishAsync(evt); return Ok(repository); @@ -92,23 +116,23 @@ public async Task SyncRepoAsync([FromRoute, Required] string id, [HttpPost("repos/{id}/trigger")] public async Task TriggerAsync([FromRoute, Required] string id, [FromBody] TriggerUpdateRequest model) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure repository exists - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); - if (repository is null) - { - return Problem(title: "repository_not_found", statusCode: 400); - } + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); + if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400); // ensure the repository update exists var update = repository.Updates.ElementAtOrDefault(model.Id!.Value); - if (update is null) - { - return Problem(title: "repository_update_not_found", statusCode: 400); - } + if (update is null) return Problem(title: ErrorCodes.RepositoryUpdateNotFound, statusCode: 400); // trigger update for specific update var evt = new TriggerUpdateJobsEvent { + ProjectId = project.Id, RepositoryId = repository.Id, RepositoryUpdateId = model.Id.Value, Trigger = UpdateJobTrigger.Manual, diff --git a/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs b/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs index 60ef8e3a..af4aab2c 100644 --- a/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs +++ b/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs @@ -33,7 +33,10 @@ public UpdateJobsController(MainDbContext dbContext, IEventPublisher publisher, [HttpPost("{id}/create_pull_request")] public async Task CreatePullRequestAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); + var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); + var project = await dbContext.Projects.SingleAsync(p => p.Id == job.ProjectId); + logger.LogInformation("Received request to create a pull request from job {JobId} but we did nothing.\r\n{ModelJson}", id, JsonSerializer.Serialize(model)); return Ok(); } @@ -41,7 +44,10 @@ public async Task CreatePullRequestAsync([FromRoute, Required] st [HttpPost("{id}/update_pull_request")] public async Task UpdatePullRequestAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); + var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); + var project = await dbContext.Projects.SingleAsync(p => p.Id == job.ProjectId); + logger.LogInformation("Received request to update a pull request from job {JobId} but we did nothing.\r\n{ModelJson}", id, JsonSerializer.Serialize(model)); return Ok(); } @@ -49,7 +55,10 @@ public async Task UpdatePullRequestAsync([FromRoute, Required] st [HttpPost("{id}/close_pull_request")] public async Task ClosePullRequestAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); + var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); + var project = await dbContext.Projects.SingleAsync(p => p.Id == job.ProjectId); + logger.LogInformation("Received request to close a pull request from job {JobId} but we did nothing.\r\n{ModelJson}", id, JsonSerializer.Serialize(model)); return Ok(); } @@ -57,7 +66,7 @@ public async Task ClosePullRequestAsync([FromRoute, Required] str [HttpPost("{id}/record_update_job_error")] public async Task RecordErrorAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); job.Error = new UpdateJobError { @@ -73,7 +82,7 @@ public async Task RecordErrorAsync([FromRoute, Required] string i [HttpPatch("{id}/mark_as_processed")] public async Task MarkAsProcessedAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); // publish event that will run update the job and collect logs var evt = new UpdateJobCheckStateEvent { JobId = id, }; @@ -85,7 +94,7 @@ public async Task MarkAsProcessedAsync([FromRoute, Required] stri [HttpPost("{id}/update_dependency_list")] public async Task UpdateDependencyListAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); // update the database @@ -102,7 +111,7 @@ public async Task UpdateDependencyListAsync([FromRoute, Required] [HttpPost("{id}/record_ecosystem_versions")] public async Task RecordEcosystemVersionsAsync([FromRoute, Required] string id, [FromBody] JsonNode model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); logger.LogInformation("Received request to record ecosystem version from job {JobId} but we did nothing.\r\n{ModelJson}", id, model.ToJsonString()); return Ok(); } @@ -110,7 +119,7 @@ public async Task RecordEcosystemVersionsAsync([FromRoute, Requir [HttpPost("{id}/increment_metric")] public async Task IncrementMetricAsync([FromRoute, Required] string id, [FromBody] JsonNode model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); logger.LogInformation("Received metrics from job {JobId} but we did nothing with them.\r\n{ModelJson}", id, model.ToJsonString()); return Ok(); } diff --git a/server/Tingle.Dependabot/Controllers/WebhooksController.cs b/server/Tingle.Dependabot/Controllers/WebhooksController.cs index 667046a8..da4e637f 100644 --- a/server/Tingle.Dependabot/Controllers/WebhooksController.cs +++ b/server/Tingle.Dependabot/Controllers/WebhooksController.cs @@ -1,7 +1,9 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; using System.Text.Json; using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; using Tingle.Dependabot.Models.Azure; using Tingle.EventBus; @@ -12,11 +14,13 @@ namespace Tingle.Dependabot.Controllers; [Authorize(AuthConstants.PolicyNameServiceHooks)] public class WebhooksController : ControllerBase // TODO: unit test this { + private readonly MainDbContext dbContext; private readonly IEventPublisher publisher; private readonly ILogger logger; - public WebhooksController(IEventPublisher publisher, ILogger logger) + public WebhooksController(MainDbContext dbContext, IEventPublisher publisher, ILogger logger) { + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -37,12 +41,17 @@ public async Task PostAsync([FromBody] AzureDevOpsEvent model) var adoRepositoryId = adoRepository.Id; var defaultBranch = adoRepository.DefaultBranch; + // ensure project exists + var adoProjectId = adoRepository.Project!.Id; + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // if the updates are not the default branch, then we ignore them var updatedReferences = resource.RefUpdates!.Select(ru => ru.Name).ToList(); if (updatedReferences.Contains(defaultBranch, StringComparer.OrdinalIgnoreCase)) { // request synchronization of the repository - var evt = new ProcessSynchronization(true, repositoryProviderId: adoRepositoryId); + var evt = new ProcessSynchronization(project.Id!, trigger: true, repositoryProviderId: adoRepositoryId); await publisher.PublishAsync(evt); } } @@ -53,6 +62,11 @@ public async Task PostAsync([FromBody] AzureDevOpsEvent model) var prId = resource.PullRequestId; var status = resource.Status; + // ensure project exists + var adoProjectId = adoRepository.Project!.Id; + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + if (type is AzureDevOpsEventType.GitPullRequestUpdated) { logger.LogInformation("PR {PullRequestId} in {RepositoryUrl} status updated to {PullRequestStatus}", @@ -83,6 +97,11 @@ public async Task PostAsync([FromBody] AzureDevOpsEvent model) var prId = pr.PullRequestId; var status = pr.Status; + // ensure project exists + var adoProjectId = adoRepository.Project!.Id; + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure the comment starts with @dependabot var content = comment.Content?.Trim(); if (content is not null && content.StartsWith("@dependabot")) diff --git a/server/Tingle.Dependabot/Events/RepositoryCreatedEvent.cs b/server/Tingle.Dependabot/Events/Events.cs similarity index 66% rename from server/Tingle.Dependabot/Events/RepositoryCreatedEvent.cs rename to server/Tingle.Dependabot/Events/Events.cs index f43fac45..6c4e94e3 100644 --- a/server/Tingle.Dependabot/Events/RepositoryCreatedEvent.cs +++ b/server/Tingle.Dependabot/Events/Events.cs @@ -2,10 +2,12 @@ namespace Tingle.Dependabot.Events; -public record RepositoryCreatedEvent : AbstractRepositoryEvent { } +public record ProjectCreatedEvent : AbstractProjectEvent { } +public record ProjectUpdatedEvent : AbstractProjectEvent { } +public record ProjectDeletedEvent : AbstractProjectEvent { } +public record RepositoryCreatedEvent : AbstractRepositoryEvent { } public record RepositoryUpdatedEvent : AbstractRepositoryEvent { } - public record RepositoryDeletedEvent : AbstractRepositoryEvent { } public record TriggerUpdateJobsEvent : AbstractRepositoryEvent @@ -20,8 +22,14 @@ public record TriggerUpdateJobsEvent : AbstractRepositoryEvent public required UpdateJobTrigger Trigger { get; set; } } -public abstract record AbstractRepositoryEvent +public abstract record AbstractRepositoryEvent : AbstractProjectEvent { /// Identifier of the repository. public required string? RepositoryId { get; set; } } + +public abstract record AbstractProjectEvent +{ + /// Identifier of the project. + public required string? ProjectId { get; set; } +} diff --git a/server/Tingle.Dependabot/Events/ProcessSynchronization.cs b/server/Tingle.Dependabot/Events/ProcessSynchronization.cs index 001ffd6c..dea671c8 100644 --- a/server/Tingle.Dependabot/Events/ProcessSynchronization.cs +++ b/server/Tingle.Dependabot/Events/ProcessSynchronization.cs @@ -4,13 +4,17 @@ public record ProcessSynchronization { public ProcessSynchronization() { } // required for deserialization - public ProcessSynchronization(bool trigger, string? repositoryId = null, string? repositoryProviderId = null) + public ProcessSynchronization(string projectId, bool trigger, string? repositoryId = null, string? repositoryProviderId = null) { + ProjectId = projectId ?? throw new ArgumentNullException(nameof(projectId)); Trigger = trigger; RepositoryId = repositoryId; RepositoryProviderId = repositoryProviderId; } + /// Identifier of the project. + public string? ProjectId { get; set; } + /// /// Indicates whether we should trigger the update jobs where changes have been detected. /// diff --git a/server/Tingle.Dependabot/Extensions/CollectionExtensions.cs b/server/Tingle.Dependabot/Extensions/CollectionExtensions.cs new file mode 100644 index 00000000..46d91f93 --- /dev/null +++ b/server/Tingle.Dependabot/Extensions/CollectionExtensions.cs @@ -0,0 +1,50 @@ +namespace System.Collections.Generic; + +internal static class CollectionExtensions +{ + /// + /// Adds an element with the provided key and value, + /// provided the value is not equal to the type's default value (or empty for strings). + /// + /// The type of keys in the dictionary. + /// The type of values in the dictionary. + /// The dictionary to use + /// The object to use as the key of the element to add. + /// The object to use as the value of the element to add. + /// key is null. + /// The dictionary is read-only. + /// + public static IDictionary AddIfNotDefault(this IDictionary dictionary, TKey key, TValue? value) + where TKey : notnull + { + if (value is not null || value is string s && !string.IsNullOrWhiteSpace(s)) + { + dictionary[key] = value; + } + + return dictionary; + } + + + /// + /// Tries to add an element with the provided key and value to the , + /// provided the value is not equal to the type's default value (or empty for strings). + /// + /// The type of keys in the dictionary. + /// The type of values in the dictionary. + /// The dictionary to use + /// The object to use as the key of the element to add. + /// The object to use as the value of the element to add. + /// key is null. + /// + public static bool TryAddIfNotDefault(this IDictionary dictionary, TKey key, TValue? value) + where TKey : notnull + { + if (value is not null || (value is string s && !string.IsNullOrWhiteSpace(s))) + { + return dictionary.TryAdd(key, value); + } + + return false; + } +} diff --git a/server/Tingle.Dependabot/Extensions/HttpContextExtensions.cs b/server/Tingle.Dependabot/Extensions/HttpContextExtensions.cs new file mode 100644 index 00000000..c649bdbf --- /dev/null +++ b/server/Tingle.Dependabot/Extensions/HttpContextExtensions.cs @@ -0,0 +1,9 @@ +namespace Microsoft.AspNetCore.Http; + +internal static class HttpContextExtensions +{ + public const string XProjectId = "X-Project-Id"; + + public static string? GetProjectId(this HttpContext httpContext) + => httpContext.Request.Headers.TryGetValue(XProjectId, out var values) ? values.Single() : null; +} diff --git a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs index 6cfd8f74..1e3d6e5a 100644 --- a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs +++ b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs @@ -1,6 +1,9 @@ using Medallion.Threading; using Medallion.Threading.FileSystem; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.FeatureManagement; +using Tingle.Dependabot.ApplicationInsights; +using Tingle.Dependabot.FeatureManagement; using Tingle.Dependabot.Workflow; namespace Microsoft.Extensions.DependencyInjection; @@ -8,6 +11,34 @@ namespace Microsoft.Extensions.DependencyInjection; /// Extensions on . public static class IServiceCollectionExtensions { + /// Add standard Application Insights services. + /// The instance to add to. + /// The root configuration instance from which to pull settings. + public static IServiceCollection AddStandardApplicationInsights(this IServiceCollection services, IConfiguration configuration) + { + // Add the core services + services.AddApplicationInsightsTelemetry(configuration); + + // Add background service to flush telemetry on shutdown + services.AddHostedService(); + + // Add processors + services.AddApplicationInsightsTelemetryProcessor(); + + // Enrich the telemetry with various sources of information + services.AddHttpContextAccessor(); // Required to resolve the request from the HttpContext + // according to docs link below, this registration should be singleton + // https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-core#adding-telemetryinitializers + services.AddSingleton(); + // services.AddApplicationInsightsTelemetryExtras(); // Add other extras + + // services.AddActivitySourceDependencyCollector(new[] { + // "Tingle.EventBus", + // }); + + return services; + } + public static IServiceCollection AddDistributedLockProvider(this IServiceCollection services, IHostEnvironment environment, IConfiguration configuration) { var configKey = ConfigurationPath.Combine("DistributedLocking", "FilePath"); @@ -36,10 +67,9 @@ public static IServiceCollection AddStandardFeatureManagement(this IServiceColle builder.AddFeatureFilter(); builder.AddFeatureFilter(); - - // In some scenarios (such as AspNetCore, the TargetingFilter together with an ITargetingContextAccessor - // should be used in place of ContextualTargetingFilter. builder.AddFeatureFilter(); + + builder.Services.AddSingleton(); builder.AddFeatureFilter(); // requires ITargetingContextAccessor builder.Services.Configure(o => o.IgnoreCase = true); @@ -53,7 +83,7 @@ public static IServiceCollection AddWorkflowServices(this IServiceCollection ser services.Configure(configuration); services.ConfigureOptions(); - services.AddSingleton(); + services.AddScoped(); services.AddSingleton(); services.AddScoped(); diff --git a/server/Tingle.Dependabot/Extensions/PropertyBuilderExtensions.cs b/server/Tingle.Dependabot/Extensions/PropertyBuilderExtensions.cs new file mode 100644 index 00000000..fb798fee --- /dev/null +++ b/server/Tingle.Dependabot/Extensions/PropertyBuilderExtensions.cs @@ -0,0 +1,138 @@ +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; + +namespace Microsoft.EntityFrameworkCore; + +/// +/// Extensions for . +/// +public static class PropertyBuilderExtensions +{ + /// + /// Attach conversion of property to/from stored in the database as concatenated string of strings. + /// + /// + /// The to extend. + /// The separator to use. + /// + public static PropertyBuilder> HasArrayConversion(this PropertyBuilder> propertyBuilder, string separator = ",") where T : IConvertible + => propertyBuilder.HasArrayConversion(separator: separator, serializerOptions: null); + + /// + /// Attach conversion of property to/from stored in the database as concatenated string of strings. + /// + /// + /// The to extend. + /// The to use for enums. + /// + public static PropertyBuilder> HasArrayConversion(this PropertyBuilder> propertyBuilder, JsonSerializerOptions? serializerOptions = null) where T : IConvertible + => propertyBuilder.HasArrayConversion(separator: ",", serializerOptions: serializerOptions); + + /// + /// Attach conversion of property to/from stored in the database as concatenated string of strings. + /// + /// + /// The to extend. + /// The separator to use. + /// The to use for enums. + /// + public static PropertyBuilder> HasArrayConversion(this PropertyBuilder> propertyBuilder, string separator, JsonSerializerOptions? serializerOptions) + where T : IConvertible + { + ArgumentNullException.ThrowIfNull(propertyBuilder); + + var converter = new ValueConverter, string?>( + convertToProviderExpression: v => ConvertToString(v, separator, serializerOptions), + convertFromProviderExpression: v => ConvertFromString(v, separator, serializerOptions)); + + var comparer = new ValueComparer>( + equalsExpression: (l, r) => ConvertToString(l, separator, serializerOptions) == ConvertToString(r, separator, serializerOptions), + hashCodeExpression: v => v == null ? 0 : ConvertToString(v, separator, serializerOptions).GetHashCode(), + snapshotExpression: v => ConvertFromString(ConvertToString(v, separator, serializerOptions), separator, serializerOptions)); + + propertyBuilder.HasConversion(converter); + propertyBuilder.Metadata.SetValueConverter(converter); + propertyBuilder.Metadata.SetValueComparer(comparer); + + return propertyBuilder; + } + + /// + /// Attach conversion of property to/from JSON stored in the database as a string. + /// + /// + /// The to extend. + /// The to use. + /// + public static PropertyBuilder HasJsonConversion(this PropertyBuilder propertyBuilder, JsonSerializerOptions? serializerOptions = null) + { + ArgumentNullException.ThrowIfNull(propertyBuilder); + +#pragma warning disable CS8603 // Possible null reference return. + var converter = new ValueConverter( + convertToProviderExpression: v => ConvertToJson(v, serializerOptions), + convertFromProviderExpression: v => ConvertFromJson(v, serializerOptions)); + + var comparer = new ValueComparer( + equalsExpression: (l, r) => ConvertToJson(l, serializerOptions) == ConvertToJson(r, serializerOptions), + hashCodeExpression: v => v == null ? 0 : ConvertToJson(v, serializerOptions).GetHashCode(), + snapshotExpression: v => ConvertFromJson(ConvertToJson(v, serializerOptions), serializerOptions)); +#pragma warning restore CS8603 // Possible null reference return. + + propertyBuilder.HasConversion(converter); + propertyBuilder.Metadata.SetValueConverter(converter); + propertyBuilder.Metadata.SetValueComparer(comparer); + + return propertyBuilder; + } + + [return: NotNullIfNotNull(nameof(value))] + private static string? ConvertToString(List? value, string separator, JsonSerializerOptions? serializerOptions) where T : IConvertible + { + if (value is null) return null; + if (string.IsNullOrWhiteSpace(separator)) + { + throw new ArgumentException($"'{nameof(separator)}' cannot be null or whitespace.", nameof(separator)); + } + + return typeof(T).IsEnum + ? string.Join(separator, value.Select(t => EnumToString(t, serializerOptions))) + : string.Join(separator, value); + } + + private static List ConvertFromString(string? value, string separator, JsonSerializerOptions? serializerOptions) where T : IConvertible + { + if (string.IsNullOrWhiteSpace(value)) return new List(); + if (string.IsNullOrWhiteSpace(separator)) + { + throw new ArgumentException($"'{nameof(separator)}' cannot be null or whitespace.", nameof(separator)); + } + + var split = value.Split(separator, StringSplitOptions.RemoveEmptyEntries); + return typeof(T).IsEnum + ? split.Select(v => EnumFromString(v, serializerOptions)).ToList() + : split.Select(v => (T)Convert.ChangeType(v, typeof(T))).ToList(); + } + + + private static T EnumFromString(string value, JsonSerializerOptions? serializerOptions) where T : IConvertible + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException($"'{nameof(value)}' cannot be null or whitespace.", nameof(value)); + } + + return JsonSerializer.Deserialize($"\"{value}\"", serializerOptions)!; + } + + private static string EnumToString(T value, JsonSerializerOptions? serializerOptions) + => JsonSerializer.Serialize(value, serializerOptions).Trim('"'); + + + private static string ConvertToJson(T value, JsonSerializerOptions? serializerOptions) => JsonSerializer.Serialize(value, serializerOptions); + + private static T? ConvertFromJson(string? value, JsonSerializerOptions? serializerOptions) => value is null ? default : JsonSerializer.Deserialize(value, serializerOptions); +} diff --git a/server/Tingle.Dependabot/Extensions/SystemExtensions.cs b/server/Tingle.Dependabot/Extensions/SystemExtensions.cs deleted file mode 100644 index 5d8951cd..00000000 --- a/server/Tingle.Dependabot/Extensions/SystemExtensions.cs +++ /dev/null @@ -1,64 +0,0 @@ -using Microsoft.EntityFrameworkCore.ChangeTracking; -using Microsoft.EntityFrameworkCore.Metadata.Builders; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using System.Text.Json; - -namespace System; - -internal static class SystemExtensions -{ - /// - /// Adds an element with the provided key and value, - /// provided the value is not equal to the type's default value (or empty for strings). - /// - /// The type of keys in the dictionary. - /// The type of values in the dictionary. - /// The dictionary to use - /// The object to use as the key of the element to add. - /// The object to use as the value of the element to add. - /// key is null. - /// The dictionary is read-only. - /// - public static IDictionary AddIfNotDefault(this IDictionary dictionary, TKey key, TValue? value) - where TKey : notnull - { - if (value is not null || value is string s && !string.IsNullOrWhiteSpace(s)) - { - dictionary[key] = value; - } - - return dictionary; - } - - /// - /// Attach conversion of property to/from JSON stored in the database as a string. - /// - /// - /// The to extend. - /// The to use. - /// - public static PropertyBuilder HasJsonConversion(this PropertyBuilder propertyBuilder, JsonSerializerOptions? serializerOptions = null) - { - ArgumentNullException.ThrowIfNull(propertyBuilder); - -#pragma warning disable CS8603 // Possible null reference return. - var converter = new ValueConverter( - convertToProviderExpression: v => ConvertToJson(v, serializerOptions), - convertFromProviderExpression: v => ConvertFromJson(v, serializerOptions)); - - var comparer = new ValueComparer( - equalsExpression: (l, r) => ConvertToJson(l, serializerOptions) == ConvertToJson(r, serializerOptions), - hashCodeExpression: v => v == null ? 0 : ConvertToJson(v, serializerOptions).GetHashCode(), - snapshotExpression: v => ConvertFromJson(ConvertToJson(v, serializerOptions), serializerOptions)); -#pragma warning restore CS8603 // Possible null reference return. - - propertyBuilder.HasConversion(converter); - propertyBuilder.Metadata.SetValueConverter(converter); - propertyBuilder.Metadata.SetValueComparer(comparer); - - return propertyBuilder; - } - - private static string ConvertToJson(T value, JsonSerializerOptions? serializerOptions) => JsonSerializer.Serialize(value, serializerOptions); - private static T? ConvertFromJson(string? value, JsonSerializerOptions? serializerOptions) => value is null ? default : JsonSerializer.Deserialize(value, serializerOptions); -} diff --git a/server/Tingle.Dependabot/CustomDisabledFeaturesHandler.cs b/server/Tingle.Dependabot/FeatureManagement/CustomDisabledFeaturesHandler.cs similarity index 92% rename from server/Tingle.Dependabot/CustomDisabledFeaturesHandler.cs rename to server/Tingle.Dependabot/FeatureManagement/CustomDisabledFeaturesHandler.cs index 8d94e4ac..a3226f90 100644 --- a/server/Tingle.Dependabot/CustomDisabledFeaturesHandler.cs +++ b/server/Tingle.Dependabot/FeatureManagement/CustomDisabledFeaturesHandler.cs @@ -1,5 +1,5 @@ -using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.FeatureManagement.Mvc; namespace Tingle.Dependabot; diff --git a/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs b/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs new file mode 100644 index 00000000..82af1ca6 --- /dev/null +++ b/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs @@ -0,0 +1,29 @@ +using Microsoft.FeatureManagement.FeatureFilters; + +namespace Tingle.Dependabot.FeatureManagement; + +/// +/// An implementation of +/// that creates a using the current . +/// +internal class ProjectTargetingContextAccessor : ITargetingContextAccessor +{ + private readonly IHttpContextAccessor contextAccessor; + + public ProjectTargetingContextAccessor(IHttpContextAccessor contextAccessor) + { + this.contextAccessor = contextAccessor ?? throw new ArgumentNullException(nameof(contextAccessor)); + } + + /// + public ValueTask GetContextAsync() + { + var httpContext = contextAccessor.HttpContext!; + + // Build targeting context based off project info + var projectId = httpContext.GetProjectId(); + var targetingContext = new TargetingContext { UserId = projectId, }; + + return new ValueTask(targetingContext); + } +} diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index d7c11726..6ee3f156 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -44,7 +44,77 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("DataProtectionKeys"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.Repository", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Created") + .HasColumnType("datetimeoffset"); + + b.Property("Etag") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("GithubToken") + .HasColumnType("nvarchar(max)"); + + b.Property("Location") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Password") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Private") + .HasColumnType("bit"); + + b.Property("ProviderId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Secrets") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Synchronized") + .HasColumnType("datetimeoffset"); + + b.Property("Token") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .HasColumnType("int"); + + b.Property("Updated") + .HasColumnType("datetimeoffset"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Created") + .IsDescending(); + + b.HasIndex("Password") + .IsUnique(); + + b.HasIndex("ProviderId") + .IsUnique(); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { b.Property("Id") .HasMaxLength(50) @@ -69,7 +139,12 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Name") .HasColumnType("nvarchar(max)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(50)"); + b.Property("ProviderId") + .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Registries") @@ -94,14 +169,15 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("ProviderId") - .IsUnique() - .HasFilter("[ProviderId] IS NOT NULL"); + .IsUnique(); b.ToTable("Repositories"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => { b.Property("Id") .HasMaxLength(50) @@ -128,9 +204,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("End") .HasColumnType("datetimeoffset"); - b.Property("Error") - .HasColumnType("nvarchar(max)"); - b.Property("Etag") .IsConcurrencyToken() .ValueGeneratedOnAddOrUpdate() @@ -146,6 +219,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + b.Property("RepositoryId") .IsRequired() .HasColumnType("nvarchar(450)"); @@ -171,6 +248,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("RepositoryId"); b.HasIndex("PackageEcosystem", "Directory"); @@ -182,9 +261,87 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => { - b.OwnsOne("Tingle.Dependabot.Models.UpdateJobResources", "Resources", b1 => + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoApprove", "AutoApprove", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoComplete", "AutoComplete", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.Property("IgnoreConfigs") + .HasColumnType("nvarchar(max)"); + + b1.Property("MergeStrategy") + .HasColumnType("int"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.Navigation("AutoApprove") + .IsRequired(); + + b.Navigation("AutoComplete") + .IsRequired(); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => + { + b.HasOne("Tingle.Dependabot.Models.Management.Project", null) + .WithMany("Repositories") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => + { + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobError", "Error", b1 => + { + b1.Property("UpdateJobId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Detail") + .HasColumnType("nvarchar(max)"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b1.HasKey("UpdateJobId"); + + b1.HasIndex("Type"); + + b1.ToTable("UpdateJobs"); + + b1.WithOwner() + .HasForeignKey("UpdateJobId"); + }); + + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobResources", "Resources", b1 => { b1.Property("UpdateJobId") .HasColumnType("nvarchar(50)"); @@ -203,9 +360,16 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasForeignKey("UpdateJobId"); }); + b.Navigation("Error"); + b.Navigation("Resources") .IsRequired(); }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Navigation("Repositories"); + }); #pragma warning restore 612, 618 } } diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 58210021..b863c427 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -25,25 +25,32 @@ protected override void Up(MigrationBuilder migrationBuilder) }); migrationBuilder.CreateTable( - name: "Repositories", + name: "Projects", columns: table => new { Id = table.Column(type: "nvarchar(50)", maxLength: 50, nullable: false), Created = table.Column(type: "datetimeoffset", nullable: false), Updated = table.Column(type: "datetimeoffset", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: true), - Slug = table.Column(type: "nvarchar(max)", nullable: true), - ProviderId = table.Column(type: "nvarchar(450)", nullable: true), - LatestCommit = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), - ConfigFileContents = table.Column(type: "nvarchar(max)", nullable: false), - SyncException = table.Column(type: "nvarchar(max)", nullable: true), - Updates = table.Column(type: "nvarchar(max)", nullable: false), - Registries = table.Column(type: "nvarchar(max)", nullable: false), + Type = table.Column(type: "int", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: false), + ProviderId = table.Column(type: "nvarchar(450)", nullable: false), + Url = table.Column(type: "nvarchar(max)", nullable: false), + Token = table.Column(type: "nvarchar(max)", nullable: false), + Private = table.Column(type: "bit", nullable: false), + AutoComplete_Enabled = table.Column(type: "bit", nullable: false), + AutoComplete_IgnoreConfigs = table.Column(type: "nvarchar(max)", nullable: true), + AutoComplete_MergeStrategy = table.Column(type: "int", nullable: true), + AutoApprove_Enabled = table.Column(type: "bit", nullable: false), + Password = table.Column(type: "nvarchar(450)", nullable: false), + Secrets = table.Column(type: "nvarchar(max)", nullable: false), + GithubToken = table.Column(type: "nvarchar(max)", nullable: true), + Location = table.Column(type: "nvarchar(max)", nullable: true), + Synchronized = table.Column(type: "datetimeoffset", nullable: true), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => { - table.PrimaryKey("PK_Repositories", x => x.Id); + table.PrimaryKey("PK_Projects", x => x.Id); }); migrationBuilder.CreateTable( @@ -54,6 +61,7 @@ protected override void Up(MigrationBuilder migrationBuilder) Created = table.Column(type: "datetimeoffset", nullable: false), Status = table.Column(type: "int", nullable: false), Trigger = table.Column(type: "int", nullable: false), + ProjectId = table.Column(type: "nvarchar(450)", nullable: false), RepositoryId = table.Column(type: "nvarchar(450)", nullable: false), RepositorySlug = table.Column(type: "nvarchar(max)", nullable: false), EventBusId = table.Column(type: "nvarchar(450)", nullable: true), @@ -67,7 +75,8 @@ protected override void Up(MigrationBuilder migrationBuilder) End = table.Column(type: "datetimeoffset", nullable: true), Duration = table.Column(type: "bigint", nullable: true), Log = table.Column(type: "nvarchar(max)", nullable: true), - Error = table.Column(type: "nvarchar(max)", nullable: true), + Error_Type = table.Column(type: "nvarchar(450)", nullable: true), + Error_Detail = table.Column(type: "nvarchar(max)", nullable: true), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => @@ -75,18 +84,69 @@ protected override void Up(MigrationBuilder migrationBuilder) table.PrimaryKey("PK_UpdateJobs", x => x.Id); }); + migrationBuilder.CreateTable( + name: "Repositories", + columns: table => new + { + Id = table.Column(type: "nvarchar(50)", maxLength: 50, nullable: false), + Created = table.Column(type: "datetimeoffset", nullable: false), + Updated = table.Column(type: "datetimeoffset", nullable: false), + ProjectId = table.Column(type: "nvarchar(50)", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: true), + Slug = table.Column(type: "nvarchar(max)", nullable: true), + ProviderId = table.Column(type: "nvarchar(450)", nullable: false), + LatestCommit = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), + ConfigFileContents = table.Column(type: "nvarchar(max)", nullable: false), + SyncException = table.Column(type: "nvarchar(max)", nullable: true), + Updates = table.Column(type: "nvarchar(max)", nullable: false), + Registries = table.Column(type: "nvarchar(max)", nullable: false), + Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_Repositories", x => x.Id); + table.ForeignKey( + name: "FK_Repositories_Projects_ProjectId", + column: x => x.ProjectId, + principalTable: "Projects", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_Projects_Created", + table: "Projects", + column: "Created", + descending: new bool[0]); + + migrationBuilder.CreateIndex( + name: "IX_Projects_Password", + table: "Projects", + column: "Password", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_Projects_ProviderId", + table: "Projects", + column: "ProviderId", + unique: true); + migrationBuilder.CreateIndex( name: "IX_Repositories_Created", table: "Repositories", column: "Created", descending: new bool[0]); + migrationBuilder.CreateIndex( + name: "IX_Repositories_ProjectId", + table: "Repositories", + column: "ProjectId"); + migrationBuilder.CreateIndex( name: "IX_Repositories_ProviderId", table: "Repositories", column: "ProviderId", - unique: true, - filter: "[ProviderId] IS NOT NULL"); + unique: true); migrationBuilder.CreateIndex( name: "IX_UpdateJobs_AuthKey", @@ -100,6 +160,11 @@ protected override void Up(MigrationBuilder migrationBuilder) column: "Created", descending: new bool[0]); + migrationBuilder.CreateIndex( + name: "IX_UpdateJobs_Error_Type", + table: "UpdateJobs", + column: "Error_Type"); + migrationBuilder.CreateIndex( name: "IX_UpdateJobs_PackageEcosystem_Directory", table: "UpdateJobs", @@ -112,6 +177,11 @@ protected override void Up(MigrationBuilder migrationBuilder) unique: true, filter: "[EventBusId] IS NOT NULL"); + migrationBuilder.CreateIndex( + name: "IX_UpdateJobs_ProjectId", + table: "UpdateJobs", + column: "ProjectId"); + migrationBuilder.CreateIndex( name: "IX_UpdateJobs_RepositoryId", table: "UpdateJobs", @@ -129,5 +199,8 @@ protected override void Down(MigrationBuilder migrationBuilder) migrationBuilder.DropTable( name: "UpdateJobs"); + + migrationBuilder.DropTable( + name: "Projects"); } } diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index c6fc03f1..ce9bdeb6 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -41,7 +41,77 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("DataProtectionKeys"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.Repository", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Created") + .HasColumnType("datetimeoffset"); + + b.Property("Etag") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("GithubToken") + .HasColumnType("nvarchar(max)"); + + b.Property("Location") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Password") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Private") + .HasColumnType("bit"); + + b.Property("ProviderId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Secrets") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Synchronized") + .HasColumnType("datetimeoffset"); + + b.Property("Token") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .HasColumnType("int"); + + b.Property("Updated") + .HasColumnType("datetimeoffset"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Created") + .IsDescending(); + + b.HasIndex("Password") + .IsUnique(); + + b.HasIndex("ProviderId") + .IsUnique(); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { b.Property("Id") .HasMaxLength(50) @@ -66,7 +136,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Name") .HasColumnType("nvarchar(max)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(50)"); + b.Property("ProviderId") + .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Registries") @@ -91,14 +166,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("ProviderId") - .IsUnique() - .HasFilter("[ProviderId] IS NOT NULL"); + .IsUnique(); b.ToTable("Repositories"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => { b.Property("Id") .HasMaxLength(50) @@ -125,9 +201,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("End") .HasColumnType("datetimeoffset"); - b.Property("Error") - .HasColumnType("nvarchar(max)"); - b.Property("Etag") .IsConcurrencyToken() .ValueGeneratedOnAddOrUpdate() @@ -143,6 +216,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + b.Property("RepositoryId") .IsRequired() .HasColumnType("nvarchar(450)"); @@ -168,6 +245,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("RepositoryId"); b.HasIndex("PackageEcosystem", "Directory"); @@ -179,9 +258,87 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => { - b.OwnsOne("Tingle.Dependabot.Models.UpdateJobResources", "Resources", b1 => + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoApprove", "AutoApprove", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoComplete", "AutoComplete", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.Property("IgnoreConfigs") + .HasColumnType("nvarchar(max)"); + + b1.Property("MergeStrategy") + .HasColumnType("int"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.Navigation("AutoApprove") + .IsRequired(); + + b.Navigation("AutoComplete") + .IsRequired(); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => + { + b.HasOne("Tingle.Dependabot.Models.Management.Project", null) + .WithMany("Repositories") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => + { + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobError", "Error", b1 => + { + b1.Property("UpdateJobId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Detail") + .HasColumnType("nvarchar(max)"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b1.HasKey("UpdateJobId"); + + b1.HasIndex("Type"); + + b1.ToTable("UpdateJobs"); + + b1.WithOwner() + .HasForeignKey("UpdateJobId"); + }); + + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobResources", "Resources", b1 => { b1.Property("UpdateJobId") .HasColumnType("nvarchar(50)"); @@ -200,9 +357,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasForeignKey("UpdateJobId"); }); + b.Navigation("Error"); + b.Navigation("Resources") .IsRequired(); }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Navigation("Repositories"); + }); #pragma warning restore 612, 618 } } diff --git a/server/Tingle.Dependabot/Models/Azure/AzdoProject.cs b/server/Tingle.Dependabot/Models/Azure/AzdoProject.cs new file mode 100644 index 00000000..5643de75 --- /dev/null +++ b/server/Tingle.Dependabot/Models/Azure/AzdoProject.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace Tingle.Dependabot.Models.Azure; + +public class AzdoProject +{ + [JsonPropertyName("id")] + public required string Id { get; set; } + + [JsonPropertyName("name")] + public required string Name { get; set; } + + [JsonPropertyName("visibility")] + public required AzdoProjectVisibility Visibility { get; set; } + + [JsonPropertyName("lastUpdateTime")] + public required DateTimeOffset LastUpdatedTime { get; set; } +} diff --git a/server/Tingle.Dependabot/Models/Azure/AzdoProjectVisibility.cs b/server/Tingle.Dependabot/Models/Azure/AzdoProjectVisibility.cs new file mode 100644 index 00000000..645c0ab5 --- /dev/null +++ b/server/Tingle.Dependabot/Models/Azure/AzdoProjectVisibility.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Tingle.Dependabot.Models.Azure; + +[JsonConverter(typeof(JsonStringEnumMemberConverter))] +public enum AzdoProjectVisibility +{ + Private, + Organization, + Public, + SystemPrivate, +} diff --git a/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs b/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs index d2b72cb6..fd261903 100644 --- a/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs +++ b/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs @@ -1,10 +1,12 @@ -using System.Text.Json.Nodes; +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace Tingle.Dependabot.Models.Dependabot; public class DependabotRecordUpdateJobErrorModel { + [Required] [JsonPropertyName("error-type")] public string? ErrorType { get; set; } diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index f5c0b1e9..a56bb19c 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -1,9 +1,5 @@ using Microsoft.AspNetCore.DataProtection.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.ChangeTracking; -using Microsoft.EntityFrameworkCore.Metadata.Builders; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using System.Text.Json; using Tingle.Dependabot.Models.Management; namespace Tingle.Dependabot.Models; @@ -12,6 +8,7 @@ public class MainDbContext : DbContext, IDataProtectionKeyContext { public MainDbContext(DbContextOptions options) : base(options) { } + public DbSet Projects => Set(); public DbSet Repositories => Set(); public DbSet UpdateJobs => Set(); @@ -21,52 +18,46 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); - modelBuilder.Entity(b => + modelBuilder.Entity(builder => { - b.Property(r => r.Updates).HasJsonConversion(); - HasJsonConversion(b.Property(r => r.Registries)); - - b.HasIndex(r => r.Created).IsDescending(); // faster filtering - b.HasIndex(r => r.ProviderId).IsUnique(); + builder.OwnsOne(p => p.AutoApprove); + builder.OwnsOne(p => p.AutoComplete, ownedBuilder => + { + ownedBuilder.Property(ac => ac.IgnoreConfigs).HasJsonConversion(); + }); + builder.Property(p => p.Secrets).HasJsonConversion(); + + builder.HasIndex(p => p.Created).IsDescending(); // faster filtering + builder.HasIndex(p => p.ProviderId).IsUnique(); + builder.HasIndex(p => p.Password).IsUnique(); // password should be unique per project }); - modelBuilder.Entity(b => + modelBuilder.Entity(builder => { - b.Property(j => j.PackageEcosystem).IsRequired(); - HasJsonConversion(b.Property(j => j.Error)); - - b.HasIndex(j => j.Created).IsDescending(); // faster filtering - b.HasIndex(j => j.RepositoryId); - b.HasIndex(j => new { j.PackageEcosystem, j.Directory, }); // faster filtering - b.HasIndex(j => new { j.PackageEcosystem, j.Directory, j.EventBusId, }).IsUnique(); - b.HasIndex(j => j.AuthKey).IsUnique(); + builder.Property(r => r.Updates).HasJsonConversion(); + builder.Property(r => r.Registries).HasJsonConversion(); - b.OwnsOne(j => j.Resources); + builder.HasIndex(r => r.Created).IsDescending(); // faster filtering + builder.HasIndex(r => r.ProviderId).IsUnique(); }); - } - - static PropertyBuilder HasJsonConversion(PropertyBuilder propertyBuilder, JsonSerializerOptions? serializerOptions = null) - { - ArgumentNullException.ThrowIfNull(propertyBuilder); - -#pragma warning disable CS8603 // Possible null reference return. - var converter = new ValueConverter( - convertToProviderExpression: v => ConvertToJson(v, serializerOptions), - convertFromProviderExpression: v => ConvertFromJson(v, serializerOptions)); - var comparer = new ValueComparer( - equalsExpression: (l, r) => ConvertToJson(l, serializerOptions) == ConvertToJson(r, serializerOptions), - hashCodeExpression: v => v == null ? 0 : ConvertToJson(v, serializerOptions).GetHashCode(), - snapshotExpression: v => ConvertFromJson(ConvertToJson(v, serializerOptions), serializerOptions)); -#pragma warning restore CS8603 // Possible null reference return. - - propertyBuilder.HasConversion(converter); - propertyBuilder.Metadata.SetValueConverter(converter); - propertyBuilder.Metadata.SetValueComparer(comparer); - - return propertyBuilder; + modelBuilder.Entity(builder => + { + builder.Property(j => j.PackageEcosystem).IsRequired(); + builder.OwnsOne(j => j.Error, ownedBuilder => + { + ownedBuilder.Property(e => e.Detail).HasJsonConversion(); + ownedBuilder.HasIndex(e => e.Type); // faster filtering + }); + + builder.HasIndex(j => j.Created).IsDescending(); // faster filtering + builder.HasIndex(j => j.ProjectId); + builder.HasIndex(j => j.RepositoryId); + builder.HasIndex(j => new { j.PackageEcosystem, j.Directory, }); // faster filtering + builder.HasIndex(j => new { j.PackageEcosystem, j.Directory, j.EventBusId, }).IsUnique(); + builder.HasIndex(j => j.AuthKey).IsUnique(); + + builder.OwnsOne(j => j.Resources); + }); } - - private static string ConvertToJson(T value, JsonSerializerOptions? serializerOptions) => JsonSerializer.Serialize(value, serializerOptions); - private static T? ConvertFromJson(string? value, JsonSerializerOptions? serializerOptions) => value is null ? default : JsonSerializer.Deserialize(value, serializerOptions); } diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs new file mode 100644 index 00000000..eaf2a6e4 --- /dev/null +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -0,0 +1,107 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; + +namespace Tingle.Dependabot.Models.Management; + +public class Project +{ + [Key, MaxLength(50)] + public string? Id { get; set; } + + public DateTimeOffset Created { get; set; } + + public DateTimeOffset Updated { get; set; } + + public ProjectType Type { get; set; } + + /// Name of the project as per provider. + [Required] + public string? Name { get; set; } + + /// Identifier of the repository as per provider. + [Required] + [JsonIgnore] // only for internal use + public string? ProviderId { get; set; } + + /// URL for the project. + /// https://dev.azure.com/tingle/dependabot + [Url] + [Required] + public string? Url { get; set; } + + /// + /// Token for accessing the project with permissions for repositories, pull requests, and service hooks. + /// + [Required] + [JsonIgnore] // expose this once we know how to protect the values + public string? Token { get; set; } + + /// Whether the project is private. + public bool Private { get; set; } + + /// Auto complete settings. + [Required] + public ProjectAutoComplete AutoComplete { get; set; } = new(); + + /// Auto approve settings. + [Required] + public ProjectAutoApprove AutoApprove { get; set; } = new(); + + /// Password for Webhooks, ServiceHooks, and Notifications from the provider. + [Required] + [DataType(DataType.Password)] + public string? Password { get; set; } + + /// + /// Secrets that can be replaced in the registries section of the dependabot configuration file. + /// + [JsonIgnore] // expose this once we know how to protect the values + public Dictionary Secrets { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Token for accessing GitHub APIs. + /// If no value is provided, the a default token is used. + /// Providing a value avoids being rate limited in case when there + /// are many calls at the same time from the same IP. + /// When provided, it must have read access to public repositories. + /// + /// ghp_1234567890 + [JsonIgnore] // expose this once we know how to protect the values + public string? GithubToken { get; set; } + + /// Location/region where to create update jobs. + /// westeurope + public string? Location { get; set; } + + [JsonIgnore] // only for internal use + public List Repositories { get; set; } = new(); + + /// Time at which the synchronization was last done for the project. + public DateTimeOffset? Synchronized { get; set; } + + [Timestamp] + public byte[]? Etag { get; set; } +} + +public class ProjectAutoComplete +{ + /// Whether to set auto complete on created pull requests. + public bool Enabled { get; set; } + + /// Identifiers of configs to be ignored in auto complete. + public List? IgnoreConfigs { get; set; } + + /// Merge strategy to use when setting auto complete on created pull requests. + public MergeStrategy? MergeStrategy { get; set; } +} + +public class ProjectAutoApprove +{ + /// Whether to automatically approve created pull requests. + public bool Enabled { get; set; } +} + +public enum ProjectType +{ + Azure, +} diff --git a/server/Tingle.Dependabot/Models/Management/Repository.cs b/server/Tingle.Dependabot/Models/Management/Repository.cs index d2bc4b69..a61a617b 100644 --- a/server/Tingle.Dependabot/Models/Management/Repository.cs +++ b/server/Tingle.Dependabot/Models/Management/Repository.cs @@ -13,11 +13,17 @@ public class Repository public DateTimeOffset Updated { get; set; } + /// Identifier of the project. + [Required] + [JsonIgnore] // only for internal use + public string? ProjectId { get; set; } + /// Name of the repository as per provider. public string? Name { get; set; } public string? Slug { get; set; } /// Identifier of the repository as per provider. + [Required] [JsonIgnore] // only for internal use public string? ProviderId { get; set; } diff --git a/server/Tingle.Dependabot/Models/Management/UpdateJob.cs b/server/Tingle.Dependabot/Models/Management/UpdateJob.cs index ad99fdf3..4971eecb 100644 --- a/server/Tingle.Dependabot/Models/Management/UpdateJob.cs +++ b/server/Tingle.Dependabot/Models/Management/UpdateJob.cs @@ -19,6 +19,11 @@ public class UpdateJob /// Trigger for the update job. public UpdateJobTrigger Trigger { get; set; } + /// Identifier of the project. + [Required] + [JsonIgnore] // only for internal use + public string? ProjectId { get; set; } + /// Identifier of the repository. [Required] [JsonIgnore] // only for internal use @@ -81,6 +86,7 @@ public class UpdateJob public class UpdateJobError { + [Required] public string? Type { get; set; } public JsonNode? Detail { get; set; } } diff --git a/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs b/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs index d258c43e..8d5f5dcf 100644 --- a/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs +++ b/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs @@ -28,49 +28,54 @@ public async Task ExecuteAsync(PeriodicTaskExecutionContext context, Cancellatio internal virtual async Task CheckAsync(DateTimeOffset referencePoint, CancellationToken cancellationToken = default) { - var repositories = await dbContext.Repositories.ToListAsync(cancellationToken); - - foreach (var repository in repositories) + var projects = await dbContext.Projects.ToListAsync(cancellationToken); + foreach (var project in projects) { - foreach (var update in repository.Updates) - { - var schedule = (CronSchedule)update.Schedule!.GenerateCron(); - var timezone = TimeZoneInfo.FindSystemTimeZoneById(update.Schedule.Timezone); + var repositories = await dbContext.Repositories.Where(r => r.ProjectId == project.Id).ToListAsync(cancellationToken); - // check if we missed an execution - var latestUpdate = update.LatestUpdate; - var missed = latestUpdate is null; // when null, it was missed - if (latestUpdate != null) + foreach (var repository in repositories) + { + foreach (var update in repository.Updates) { - var nextFromLast = schedule.GetNextOccurrence(latestUpdate.Value, timezone); - if (nextFromLast is null) continue; + var schedule = (CronSchedule)update.Schedule!.GenerateCron(); + var timezone = TimeZoneInfo.FindSystemTimeZoneById(update.Schedule.Timezone); - var nextFromReference = schedule.GetNextOccurrence(referencePoint, timezone); - if (nextFromReference is null) continue; + // check if we missed an execution + var latestUpdate = update.LatestUpdate; + var missed = latestUpdate is null; // when null, it was missed + if (latestUpdate != null) + { + var nextFromLast = schedule.GetNextOccurrence(latestUpdate.Value, timezone); + if (nextFromLast is null) continue; - missed = nextFromLast.Value <= referencePoint; // when next is in the past, it was missed + var nextFromReference = schedule.GetNextOccurrence(referencePoint, timezone); + if (nextFromReference is null) continue; - // for daily schedules, only check if the next is more than 12 hours away - if (missed && update.Schedule.Interval is DependabotScheduleInterval.Daily) - { - missed = (nextFromReference.Value - referencePoint).Hours > 12; - } - } + missed = nextFromLast.Value <= referencePoint; // when next is in the past, it was missed - // if we missed an execution, trigger one - if (missed) - { - logger.LogWarning("Schedule was missed for {RepositoryId}({UpdateId}). Triggering now", repository.Id, repository.Updates.IndexOf(update)); + // for daily schedules, only check if the next is more than 12 hours away + if (missed && update.Schedule.Interval is DependabotScheduleInterval.Daily) + { + missed = (nextFromReference.Value - referencePoint).Hours > 12; + } + } - // publish event for the job to be run - var evt = new TriggerUpdateJobsEvent + // if we missed an execution, trigger one + if (missed) { - RepositoryId = repository.Id, - RepositoryUpdateId = repository.Updates.IndexOf(update), - Trigger = UpdateJobTrigger.MissedSchedule, - }; + logger.LogWarning("Schedule was missed for {RepositoryId}({UpdateId}). Triggering now", repository.Id, repository.Updates.IndexOf(update)); - await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + // publish event for the job to be run + var evt = new TriggerUpdateJobsEvent + { + ProjectId = repository.ProjectId, + RepositoryId = repository.Id, + RepositoryUpdateId = repository.Updates.IndexOf(update), + Trigger = UpdateJobTrigger.MissedSchedule, + }; + + await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + } } } } diff --git a/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs b/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs index 6ccff61d..f62249a2 100644 --- a/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs +++ b/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs @@ -1,4 +1,6 @@ -using Tingle.Dependabot.Events; +using Microsoft.EntityFrameworkCore; +using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; using Tingle.EventBus; using Tingle.PeriodicTasks; @@ -6,10 +8,12 @@ namespace Tingle.Dependabot.Workflow; internal class SynchronizationTask : IPeriodicTask { + private readonly MainDbContext dbContext; private readonly IEventPublisher publisher; - public SynchronizationTask(IEventPublisher publisher) + public SynchronizationTask(MainDbContext dbContext, IEventPublisher publisher) { + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); } @@ -20,8 +24,12 @@ public async Task ExecuteAsync(PeriodicTaskExecutionContext context, Cancellatio internal virtual async Task SyncAsync(CancellationToken cancellationToken = default) { - // request synchronization of the whole project via events - var evt = new ProcessSynchronization(false); /* database sync should not trigger, just in case it's too many */ - await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + // request synchronization of the each project via events + var projects = await dbContext.Projects.ToListAsync(cancellationToken); + foreach (var project in projects) + { + var evt = new ProcessSynchronization(project.Id!, false); /* database sync should not trigger, just in case it's too many */ + await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + } } } diff --git a/server/Tingle.Dependabot/Program.cs b/server/Tingle.Dependabot/Program.cs index 72b0e003..c5a25dd6 100644 --- a/server/Tingle.Dependabot/Program.cs +++ b/server/Tingle.Dependabot/Program.cs @@ -12,6 +12,8 @@ var builder = WebApplication.CreateBuilder(args); +builder.Services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(30)); /* default is 5 seconds */ + // Add Azure AppConfiguration builder.Configuration.AddStandardAzureAppConfiguration(builder.Environment); builder.Services.AddAzureAppConfiguration(); @@ -30,22 +32,8 @@ }); }); -builder.Services.Configure(options => -{ - /* - * The shutdown timer is extended to background tasks (mostly IHostedService) time to close down gracefully - * Andrew Lock explains it in 2 of his posts - * - * https://andrewlock.net/extending-the-shutdown-timeout-setting-to-ensure-graceful-ihostedservice-shutdown/ - * https://andrewlock.net/deploying-asp-net-core-applications-to-kubernetes-part-11-avoiding-downtime-in-rolling-deployments-by-blocking-sigterm/ - * - * The default is 5 seconds but in our case 30 seconds is sufficient and matches the Kubernetes default. - * This should be enough for the services running like the EventBus or incoming HTTP requests to complete processing. - */ - options.ShutdownTimeout = TimeSpan.FromSeconds(30); -}); - -builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); +// Add Application Insights +builder.Services.AddStandardApplicationInsights(builder.Configuration); // Add DbContext builder.Services.AddDbContext(options => diff --git a/server/Tingle.Dependabot/Properties/launchSettings.json b/server/Tingle.Dependabot/Properties/launchSettings.json index 55a49472..40bb847c 100644 --- a/server/Tingle.Dependabot/Properties/launchSettings.json +++ b/server/Tingle.Dependabot/Properties/launchSettings.json @@ -9,7 +9,8 @@ "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", "EFCORE_PERFORM_MIGRATIONS": "true", - "SKIP_LOAD_SCHEDULES": "true" + "SKIP_LOAD_SCHEDULES": "true", + "PROJECT_SETUPS_": "[{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]" } }, "Docker": { diff --git a/server/Tingle.Dependabot/Tingle.Dependabot.csproj b/server/Tingle.Dependabot/Tingle.Dependabot.csproj index b23b1b3e..11cf3eee 100644 --- a/server/Tingle.Dependabot/Tingle.Dependabot.csproj +++ b/server/Tingle.Dependabot/Tingle.Dependabot.csproj @@ -39,6 +39,7 @@ + diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs index 73a7a69c..f765573d 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs @@ -14,6 +14,7 @@ public AzureDevOpsProjectUrl(string value) : this(new Uri(value)) { } public AzureDevOpsProjectUrl(Uri uri) { this.uri = uri ?? throw new ArgumentNullException(nameof(uri)); + Scheme = uri.Scheme; var host = Hostname = uri.Host; Port = uri switch { @@ -58,6 +59,7 @@ public static AzureDevOpsProjectUrl Create(string hostname, string organizationN return new(builder.Uri); } + public string Scheme { get; } public string Hostname { get; } public int? Port { get; } public string OrganizationName { get; } diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs index 2bd55263..735b0431 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs @@ -6,13 +6,26 @@ using Microsoft.VisualStudio.Services.FormInput; using Microsoft.VisualStudio.Services.ServiceHooks.WebApi; using Microsoft.VisualStudio.Services.WebApi; +using System.Net.Http.Headers; using System.Security.Cryptography; using System.Text; +using Tingle.Dependabot.Models.Azure; +using Tingle.Dependabot.Models.Management; namespace Tingle.Dependabot.Workflow; -public class AzureDevOpsProvider +public class AzureDevOpsProvider // TODO: replace the Microsoft.(TeamFoundation|VisualStudio) libraries with direct usage of HttpClient { + // Possible/allowed paths for the configuration files in a repository. + private static readonly IReadOnlyList ConfigurationFilePaths = new[] { + // TODO: restore checks in .azuredevops folder once either the code can check that folder or we are passing ignore conditions via update_jobs API + //".azuredevops/dependabot.yml", + //".azuredevops/dependabot.yaml", + + ".github/dependabot.yml", + ".github/dependabot.yaml", + }; + private static readonly (string, string)[] SubscriptionEventTypes = { ("git.push", "1.0"), @@ -22,6 +35,7 @@ private static readonly (string, string)[] SubscriptionEventTypes = }; private readonly IMemoryCache cache; + private readonly HttpClient httpClient = new(); // TODO: consider injecting this for logging and tracing purposes private readonly WorkflowOptions options; public AzureDevOpsProvider(IMemoryCache cache, IOptions optionsAccessor) @@ -30,11 +44,11 @@ public AzureDevOpsProvider(IMemoryCache cache, IOptions options options = optionsAccessor?.Value ?? throw new ArgumentNullException(nameof(optionsAccessor)); } - public async Task> CreateOrUpdateSubscriptionsAsync(CancellationToken cancellationToken = default) + public async Task> CreateOrUpdateSubscriptionsAsync(Project project, CancellationToken cancellationToken = default) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // get the projectId var projectId = (await (await connection.GetClientAsync(cancellationToken)).GetProject(url.ProjectIdOrName)).Id.ToString(); @@ -64,7 +78,7 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok ConsumerActionId = "httpRequest", })).Results; - var webhookUrl = options.WebhookEndpoint; + var webhookUrl = options.WebhookEndpoint!; var ids = new List(); foreach (var (eventType, resourceVersion) in SubscriptionEventTypes) { @@ -89,7 +103,7 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok existing.EventType = eventType; existing.ResourceVersion = resourceVersion; existing.PublisherInputs = MakeTfsPublisherInputs(eventType, projectId); - existing.ConsumerInputs = MakeWebHooksConsumerInputs(); + existing.ConsumerInputs = MakeWebHooksConsumerInputs(project, webhookUrl); existing = await client.UpdateSubscriptionAsync(existing); } else @@ -103,7 +117,7 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok PublisherInputs = MakeTfsPublisherInputs(eventType, projectId), ConsumerId = "webHooks", ConsumerActionId = "httpRequest", - ConsumerInputs = MakeWebHooksConsumerInputs(), + ConsumerInputs = MakeWebHooksConsumerInputs(project, webhookUrl), }; existing = await client.CreateSubscriptionAsync(existing); } @@ -115,11 +129,33 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok return ids; } - public async Task> GetRepositoriesAsync(CancellationToken cancellationToken) + public async Task GetProjectAsync(Project project, CancellationToken cancellationToken) + { + //// get a connection to Azure DevOps + //var url = (AzureDevOpsProjectUrl)project.Url!; + //var connection = CreateVssConnection(url, project.Token!); + + //// get the project + //var client = await connection.GetClientAsync(cancellationToken); + //return await client.GetProject(id: url.ProjectIdOrName); + + var url = (AzureDevOpsProjectUrl)project.Url!; + var uri = new UriBuilder + { + Scheme = url.Scheme, + Host = url.Hostname, + Port = url.Port ?? -1, + Path = $"{url.OrganizationName}/_apis/projects/{url.ProjectIdOrName}", + }.Uri; + var request = new HttpRequestMessage(HttpMethod.Get, uri); + return (await SendAsync(project.Token!, request, cancellationToken))!; + } + + public async Task> GetRepositoriesAsync(Project project, CancellationToken cancellationToken) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // fetch the repositories var client = await connection.GetClientAsync(cancellationToken); @@ -127,27 +163,26 @@ public async Task> GetRepositoriesAsync(CancellationToken ca return repos.OrderBy(r => r.Name).ToList(); } - public async Task GetRepositoryAsync(string repositoryIdOrName, CancellationToken cancellationToken) + public async Task GetRepositoryAsync(Project project, string repositoryIdOrName, CancellationToken cancellationToken) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // get the repository var client = await connection.GetClientAsync(cancellationToken); return await client.GetRepositoryAsync(project: url.ProjectIdOrName, repositoryId: repositoryIdOrName, cancellationToken: cancellationToken); } - public async Task GetConfigurationFileAsync(string repositoryIdOrName, CancellationToken cancellationToken = default) + public async Task GetConfigurationFileAsync(Project project, string repositoryIdOrName, CancellationToken cancellationToken = default) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // Try all known paths - var paths = options.ConfigurationFilePaths; var client = await connection.GetClientAsync(cancellationToken); - foreach (var path in paths) + foreach (var path in ConfigurationFilePaths) { try { @@ -187,7 +222,7 @@ private static Dictionary MakeTfsPublisherInputs(string type, st return result; } - private Dictionary MakeWebHooksConsumerInputs() + private static Dictionary MakeWebHooksConsumerInputs(Project project, Uri webhookUrl) { return new Dictionary { @@ -196,9 +231,9 @@ private Dictionary MakeWebHooksConsumerInputs() ["detailedMessagesToSend"] = "none", ["messagesToSend"] = "none", - ["url"] = options.WebhookEndpoint!.ToString(), - ["basicAuthUsername"] = "vsts", - ["basicAuthPassword"] = options.SubscriptionPassword!, + ["url"] = webhookUrl.ToString(), + ["basicAuthUsername"] = project.Id!, + ["basicAuthPassword"] = project.Password!, }; } @@ -224,4 +259,14 @@ static string hash(string v) return cache.Set(cacheKey, cached, TimeSpan.FromHours(1)); } + + private async Task SendAsync(string token, HttpRequestMessage request, CancellationToken cancellationToken) + { + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($":{token}"))); + + var response = await httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); + } } diff --git a/server/Tingle.Dependabot/Workflow/Synchronizer.cs b/server/Tingle.Dependabot/Workflow/Synchronizer.cs index ab9fe78f..eeac35af 100644 --- a/server/Tingle.Dependabot/Workflow/Synchronizer.cs +++ b/server/Tingle.Dependabot/Workflow/Synchronizer.cs @@ -1,5 +1,4 @@ using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; using System.ComponentModel.DataAnnotations; using Tingle.Dependabot.Events; using Tingle.Dependabot.Models; @@ -16,21 +15,15 @@ internal class Synchronizer private readonly MainDbContext dbContext; private readonly AzureDevOpsProvider adoProvider; private readonly IEventPublisher publisher; - private readonly WorkflowOptions options; private readonly ILogger logger; private readonly IDeserializer yamlDeserializer; - public Synchronizer(MainDbContext dbContext, - AzureDevOpsProvider adoProvider, - IEventPublisher publisher, - IOptions optionsAccessor, - ILogger logger) + public Synchronizer(MainDbContext dbContext, AzureDevOpsProvider adoProvider, IEventPublisher publisher, ILogger logger) { this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); this.adoProvider = adoProvider ?? throw new ArgumentNullException(nameof(adoProvider)); this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); - options = optionsAccessor?.Value ?? throw new ArgumentNullException(nameof(optionsAccessor)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); yamlDeserializer = new DeserializerBuilder().WithNamingConvention(HyphenatedNamingConvention.Instance) @@ -38,14 +31,31 @@ public Synchronizer(MainDbContext dbContext, .Build(); } - public async Task SynchronizeAsync(bool trigger, CancellationToken cancellationToken = default) + public async Task SynchronizeAsync(Project project, bool trigger, CancellationToken cancellationToken = default) { + // skip if the project last synchronization is less than 1 hour ago + if ((DateTimeOffset.UtcNow - project.Synchronized) <= TimeSpan.FromHours(1)) + { + logger.LogInformation("Skipping synchronization for {ProjectUrl} since it last happened recently at {Synchronized}.", project.Url, project.Synchronized); + return; + } + + // update the project (it may have changed name or visibility) + var tp = await adoProvider.GetProjectAsync(project, cancellationToken); + var @private = tp.Visibility is not Models.Azure.AzdoProjectVisibility.Public; + if (!string.Equals(project.Name, tp.Name, StringComparison.Ordinal) || @private != project.Private) + { + project.Name = tp.Name; + project.Private = @private; + project.Updated = DateTimeOffset.UtcNow; + } + // track the synchronization pairs var syncPairs = new List<(SynchronizerConfigurationItem, Repository?)>(); // get the repositories from Azure logger.LogDebug("Listing repositories ..."); - var adoRepos = await adoProvider.GetRepositoriesAsync(cancellationToken); + var adoRepos = await adoProvider.GetRepositoriesAsync(project, cancellationToken); logger.LogDebug("Found {RepositoriesCount} repositories", adoRepos.Count); var adoReposMap = adoRepos.ToDictionary(r => r.Id.ToString(), r => r); @@ -62,14 +72,16 @@ public async Task SynchronizeAsync(bool trigger, CancellationToken cancellationT // get the repository from the database var adoRepositoryName = adoRepo.Name; var repository = await (from r in dbContext.Repositories + where r.ProjectId == project.Id where r.ProviderId == adoRepositoryId select r).SingleOrDefaultAsync(cancellationToken); - var item = await adoProvider.GetConfigurationFileAsync(repositoryIdOrName: adoRepositoryId, + var item = await adoProvider.GetConfigurationFileAsync(project: project, + repositoryIdOrName: adoRepositoryId, cancellationToken: cancellationToken); // Track for further synchronization - var sci = new SynchronizerConfigurationItem(options.ProjectUrl!.Value.MakeRepositorySlug(adoRepo.Name), adoRepo, item); + var sci = new SynchronizerConfigurationItem(((AzureDevOpsProjectUrl)project.Url!).MakeRepositorySlug(adoRepo.Name), adoRepo, item); syncPairs.Add((sci, repository)); } @@ -84,14 +96,18 @@ public async Task SynchronizeAsync(bool trigger, CancellationToken cancellationT // synchronize each repository foreach (var (pi, repository) in syncPairs) { - await SynchronizeAsync(repository, pi, trigger, cancellationToken); + await SynchronizeAsync(project, repository, pi, trigger, cancellationToken); } + + // set the last synchronization time on the project + project.Synchronized = DateTimeOffset.UtcNow; } - public async Task SynchronizeAsync(Repository repository, bool trigger, CancellationToken cancellationToken = default) + public async Task SynchronizeAsync(Project project, Repository repository, bool trigger, CancellationToken cancellationToken = default) { // get repository - var adoRepo = await adoProvider.GetRepositoryAsync(repositoryIdOrName: repository.ProviderId!, + var adoRepo = await adoProvider.GetRepositoryAsync(project: project, + repositoryIdOrName: repository.ProviderId!, cancellationToken: cancellationToken); // skip disabled or fork repository @@ -102,18 +118,20 @@ public async Task SynchronizeAsync(Repository repository, bool trigger, Cancella } // get the configuration file - var item = await adoProvider.GetConfigurationFileAsync(repositoryIdOrName: repository.ProviderId!, + var item = await adoProvider.GetConfigurationFileAsync(project: project, + repositoryIdOrName: repository.ProviderId!, cancellationToken: cancellationToken); // perform synchronization - var sci = new SynchronizerConfigurationItem(options.ProjectUrl!.Value.MakeRepositorySlug(adoRepo.Name), adoRepo, item); - await SynchronizeAsync(repository, sci, trigger, cancellationToken); + var sci = new SynchronizerConfigurationItem(((AzureDevOpsProjectUrl)project.Url!).MakeRepositorySlug(adoRepo.Name), adoRepo, item); + await SynchronizeAsync(project, repository, sci, trigger, cancellationToken); } - public async Task SynchronizeAsync(string? repositoryProviderId, bool trigger, CancellationToken cancellationToken = default) + public async Task SynchronizeAsync(Project project, string? repositoryProviderId, bool trigger, CancellationToken cancellationToken = default) { // get repository - var adoRepo = await adoProvider.GetRepositoryAsync(repositoryIdOrName: repositoryProviderId!, + var adoRepo = await adoProvider.GetRepositoryAsync(project: project, + repositoryIdOrName: repositoryProviderId!, cancellationToken: cancellationToken); // skip disabled or fork repository @@ -124,7 +142,8 @@ public async Task SynchronizeAsync(string? repositoryProviderId, bool trigger, C } // get the configuration file - var item = await adoProvider.GetConfigurationFileAsync(repositoryIdOrName: repositoryProviderId!, + var item = await adoProvider.GetConfigurationFileAsync(project: project, + repositoryIdOrName: repositoryProviderId!, cancellationToken: cancellationToken); var repository = await (from r in dbContext.Repositories @@ -132,11 +151,12 @@ public async Task SynchronizeAsync(string? repositoryProviderId, bool trigger, C select r).SingleOrDefaultAsync(cancellationToken); // perform synchronization - var sci = new SynchronizerConfigurationItem(options.ProjectUrl!.Value.MakeRepositorySlug(adoRepo.Name), adoRepo, item); - await SynchronizeAsync(repository, sci, trigger, cancellationToken); + var sci = new SynchronizerConfigurationItem(((AzureDevOpsProjectUrl)project.Url!).MakeRepositorySlug(adoRepo.Name), adoRepo, item); + await SynchronizeAsync(project, repository, sci, trigger, cancellationToken); } - internal async Task SynchronizeAsync(Repository? repository, + internal async Task SynchronizeAsync(Project project, + Repository? repository, SynchronizerConfigurationItem providerInfo, bool trigger, CancellationToken cancellationToken = default) @@ -152,7 +172,7 @@ internal async Task SynchronizeAsync(Repository? repository, await dbContext.SaveChangesAsync(cancellationToken); // publish RepositoryDeletedEvent event - var evt = new RepositoryDeletedEvent { RepositoryId = repository.Id, }; + var evt = new RepositoryDeletedEvent { ProjectId = project.Id, RepositoryId = repository.Id, }; await publisher.PublishAsync(evt, cancellationToken: cancellationToken); } @@ -175,10 +195,11 @@ internal async Task SynchronizeAsync(Repository? repository, { Id = Guid.NewGuid().ToString("n"), Created = DateTimeOffset.UtcNow, + ProjectId = project.Id, ProviderId = providerInfo.Id, }; await dbContext.Repositories.AddAsync(repository, cancellationToken); - rce = new RepositoryCreatedEvent { RepositoryId = repository.Id, }; + rce = new RepositoryCreatedEvent { ProjectId = project.Id, RepositoryId = repository.Id, }; } // if the name of the repository has changed then we assume the commit changed so that we update stuff @@ -235,7 +256,7 @@ internal async Task SynchronizeAsync(Repository? repository, } else { - var evt = new RepositoryUpdatedEvent { RepositoryId = repository.Id, }; + var evt = new RepositoryUpdatedEvent { ProjectId = project.Id, RepositoryId = repository.Id, }; await publisher.PublishAsync(evt, cancellationToken: cancellationToken); } @@ -244,6 +265,7 @@ internal async Task SynchronizeAsync(Repository? repository, // trigger update jobs for the whole repository var evt = new TriggerUpdateJobsEvent { + ProjectId = project.Id, RepositoryId = repository.Id, RepositoryUpdateId = null, // run all Trigger = UpdateJobTrigger.Synchronization, diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 383bd4eb..07713f7e 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -5,6 +5,8 @@ using Azure.ResourceManager.AppContainers.Models; using Azure.ResourceManager.Resources; using Microsoft.Extensions.Options; +using Microsoft.FeatureManagement; +using Microsoft.FeatureManagement.FeatureFilters; using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Nodes; @@ -24,6 +26,7 @@ internal partial class UpdateRunner private static readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + private readonly IFeatureManagerSnapshot featureManager; private readonly WorkflowOptions options; private readonly ILogger logger; @@ -31,8 +34,9 @@ internal partial class UpdateRunner private readonly ResourceGroupResource resourceGroup; private readonly LogsQueryClient logsQueryClient; - public UpdateRunner(IOptions optionsAccessor, ILogger logger) + public UpdateRunner(IFeatureManagerSnapshot featureManager, IOptions optionsAccessor, ILogger logger) { + this.featureManager = featureManager ?? throw new ArgumentNullException(nameof(featureManager)); options = optionsAccessor?.Value ?? throw new ArgumentNullException(nameof(optionsAccessor)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); armClient = new ArmClient(new DefaultAzureCredential()); @@ -40,7 +44,7 @@ public UpdateRunner(IOptions optionsAccessor, ILogger(options.Secrets) { ["DEFAULT_TOKEN"] = options.ProjectToken!, }; + var secrets = new Dictionary(project.Secrets) { ["DEFAULT_TOKEN"] = project.Token!, }; var registries = update.Registries?.Select(r => repository.Registries[r]).ToList(); var credentials = MakeExtraCredentials(registries, secrets); // add source credentials when running the in v2 var directory = Path.Join(options.WorkingDirectory, job.Id); @@ -67,14 +75,14 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up Name = UpdaterContainerName, Image = options.UpdaterContainerImageTemplate!.Replace("{{ecosystem}}", job.PackageEcosystem), Resources = job.Resources!, - Args = { "update_script", }, + Args = { useV2 ? "update_files" : "update_script", }, VolumeMounts = { new ContainerAppVolumeMount { VolumeName = volumeName, MountPath = options.WorkingDirectory, }, }, }; - var env = CreateEnvironmentVariables(repository, update, job, directory, credentials); + var env = await CreateEnvironmentVariables(project, repository, update, job, directory, credentials, cancellationToken); foreach (var (key, value) in env) container.Env.Add(new ContainerAppEnvironmentVariable { Name = key, Value = value, }); // prepare the ContainerApp job - var data = new ContainerAppJobData(options.Location!) + var data = new ContainerAppJobData((project.Location ?? options.Location)!) { EnvironmentId = options.AppEnvironmentId, Configuration = new ContainerAppJobConfiguration(ContainerAppJobTriggerType.Manual, 1) @@ -113,7 +121,7 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up }; // write job definition file - var jobDefinitionPath = await WriteJobDefinitionAsync(update, job, directory, credentials, cancellationToken); + var jobDefinitionPath = await WriteJobDefinitionAsync(project, update, job, directory, credentials, cancellationToken); logger.LogInformation("Written job definition file at {JobDefinitionPath}", jobDefinitionPath); // create the ContainerApp Job @@ -216,29 +224,36 @@ public async Task DeleteAsync(UpdateJob job, CancellationToken cancellationToken internal static string MakeResourceName(UpdateJob job) => $"dependabot-{job.Id}"; - internal IDictionary CreateEnvironmentVariables(Repository repository, - RepositoryUpdate update, - UpdateJob job, - string directory, - IList> credentials) // TODO: unit test this + internal async Task> CreateEnvironmentVariables(Project project, + Repository repository, + RepositoryUpdate update, + UpdateJob job, + string directory, + IList> credentials, + CancellationToken cancellationToken = default) // TODO: unit test this { [return: NotNullIfNotNull(nameof(value))] static string? ToJson(T? value) => value is null ? null : JsonSerializer.Serialize(value, serializerOptions); // null ensures we do not add to the values + // check if debug and determinism is enabled for the project via Feature Management + var fmc = MakeTargetingContext(project, job); + var debugAllJobs = await featureManager.IsEnabledAsync(FeatureNames.DebugAllJobs, fmc); + var deterministic = await featureManager.IsEnabledAsync(FeatureNames.DeterministicUpdates, fmc); + // Add compulsory values var values = new Dictionary { // env for v2 ["DEPENDABOT_JOB_ID"] = job.Id!, ["DEPENDABOT_JOB_TOKEN"] = job.AuthKey!, - ["DEPENDABOT_DEBUG"] = (options.DebugJobs ?? false).ToString().ToLower(), - ["DEPENDABOT_API_URL"] = options.JobsApiUrl!, + ["DEPENDABOT_DEBUG"] = debugAllJobs.ToString().ToLower(), + ["DEPENDABOT_API_URL"] = options.JobsApiUrl!.ToString(), ["DEPENDABOT_JOB_PATH"] = Path.Join(directory, JobDefinitionFileName), ["DEPENDABOT_OUTPUT_PATH"] = Path.Join(directory, "output"), // Setting DEPENDABOT_REPO_CONTENTS_PATH causes some issues, ignore till we can resolve //["DEPENDABOT_REPO_CONTENTS_PATH"] = Path.Join(jobDirectory, "repo"), ["GITHUB_ACTIONS"] = "false", - ["UPDATER_DETERMINISTIC"] = (options.DeterministicUpdates ?? false).ToString().ToLower(), + ["UPDATER_DETERMINISTIC"] = deterministic.ToString().ToLower(), // env for v1 ["DEPENDABOT_PACKAGE_MANAGER"] = job.PackageEcosystem!, @@ -249,7 +264,7 @@ internal IDictionary CreateEnvironmentVariables(Repository repos }; // Add optional values - values.AddIfNotDefault("GITHUB_ACCESS_TOKEN", options.GithubToken) + values.AddIfNotDefault("GITHUB_ACCESS_TOKEN", project.GithubToken ?? options.GithubToken) .AddIfNotDefault("DEPENDABOT_REBASE_STRATEGY", update.RebaseStrategy) .AddIfNotDefault("DEPENDABOT_TARGET_BRANCH", update.TargetBranch) .AddIfNotDefault("DEPENDABOT_VENDOR", update.Vendor ? "true" : null) @@ -261,21 +276,22 @@ internal IDictionary CreateEnvironmentVariables(Repository repos .AddIfNotDefault("DEPENDABOT_MILESTONE", update.Milestone?.ToString()); // Add values for Azure DevOps - var url = options.ProjectUrl!.Value; + var url = (AzureDevOpsProjectUrl)project.Url!; values.AddIfNotDefault("AZURE_HOSTNAME", url.Hostname) .AddIfNotDefault("AZURE_ORGANIZATION", url.OrganizationName) .AddIfNotDefault("AZURE_PROJECT", url.ProjectName) .AddIfNotDefault("AZURE_REPOSITORY", Uri.EscapeDataString(repository.Name!)) - .AddIfNotDefault("AZURE_ACCESS_TOKEN", options.ProjectToken) - .AddIfNotDefault("AZURE_SET_AUTO_COMPLETE", (options.AutoComplete ?? false).ToString().ToLowerInvariant()) - .AddIfNotDefault("AZURE_AUTO_COMPLETE_IGNORE_CONFIG_IDS", ToJson(options.AutoCompleteIgnoreConfigs?.Split(';'))) - .AddIfNotDefault("AZURE_MERGE_STRATEGY", options.AutoCompleteMergeStrategy?.ToString()) - .AddIfNotDefault("AZURE_AUTO_APPROVE_PR", (options.AutoApprove ?? false).ToString().ToLowerInvariant()); + .AddIfNotDefault("AZURE_ACCESS_TOKEN", project.Token) + .AddIfNotDefault("AZURE_SET_AUTO_COMPLETE", project.AutoComplete.Enabled.ToString().ToLowerInvariant()) + .AddIfNotDefault("AZURE_AUTO_COMPLETE_IGNORE_CONFIG_IDS", ToJson(project.AutoComplete.IgnoreConfigs ?? new())) + .AddIfNotDefault("AZURE_MERGE_STRATEGY", project.AutoComplete.MergeStrategy?.ToString()) + .AddIfNotDefault("AZURE_AUTO_APPROVE_PR", project.AutoApprove.Enabled.ToString().ToLowerInvariant()); return values; } - internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, + internal async Task WriteJobDefinitionAsync(Project project, + RepositoryUpdate update, UpdateJob job, string directory, IList> credentials, @@ -284,9 +300,13 @@ internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, [return: NotNullIfNotNull(nameof(value))] static JsonNode? ToJsonNode(T? value) => value is null ? null : JsonSerializer.SerializeToNode(value, serializerOptions); // null ensures we do not add to the values - var url = options.ProjectUrl!.Value; + var url = (AzureDevOpsProjectUrl)project.Url!; var credentialsMetadata = MakeCredentialsMetadata(credentials); + // check if debug is enabled for the project via Feature Management + var fmc = MakeTargetingContext(project, job); + var debug = await featureManager.IsEnabledAsync(FeatureNames.DebugJobs, fmc); + var definition = new JsonObject { ["job"] = new JsonObject @@ -320,7 +340,7 @@ internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, // ["updating-a-pull-request"] = false, ["vendor-dependencies"] = update.Vendor, ["security-updates-only"] = update.OpenPullRequestsLimit == 0, - ["debug"] = false, + ["debug"] = debug, }, ["credentials"] = ToJsonNode(credentials).AsArray(), }; @@ -334,6 +354,18 @@ internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, return path; } + internal static TargetingContext MakeTargetingContext(Project project, UpdateJob job) + { + return new TargetingContext + { + Groups = new[] + { + $"provider:{project.Type.ToString().ToLower()}", + $"project:{project.Id}", + $"ecosystem:{job.PackageEcosystem}", + }, + }; + } internal static IList> MakeCredentialsMetadata(IList> credentials) { return credentials.Select(cred => diff --git a/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs b/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs index 984d70b9..eff79df5 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs @@ -37,12 +37,13 @@ public async Task CreateOrUpdateAsync(Repository repository, CancellationToken c updates.Add(new(repository.Updates.IndexOf(update), update.Schedule!)); } + var projectId = repository.ProjectId!; var repositoryId = repository.Id!; var timers = new List(); foreach (var (index, supplied) in updates) { var schedule = supplied.GenerateCron(); - var payload = new TimerPayload(repositoryId, index); + var payload = new TimerPayload(projectId, repositoryId, index); var timer = new CronScheduleTimer(schedule, supplied.Timezone, CustomTimerCallback, payload); timers.Add(timer); } @@ -79,6 +80,7 @@ private async Task CustomTimerCallback(CronScheduleTimer timer, object? arg2, Ca // publish event for the job to be run var evt = new TriggerUpdateJobsEvent { + ProjectId = payload.ProjectId, RepositoryId = payload.RepositoryId, RepositoryUpdateId = payload.RepositoryUpdateId, Trigger = UpdateJobTrigger.Scheduled, @@ -87,5 +89,5 @@ private async Task CustomTimerCallback(CronScheduleTimer timer, object? arg2, Ca await publisher.PublishAsync(evt, cancellationToken: cancellationToken); } - private readonly record struct TimerPayload(string RepositoryId, int RepositoryUpdateId); + private readonly record struct TimerPayload(string ProjectId, string RepositoryId, int RepositoryUpdateId); } diff --git a/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs index 473a40d1..30d58304 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs @@ -2,20 +2,8 @@ namespace Tingle.Dependabot.Workflow; -internal class WorkflowConfigureOptions : IPostConfigureOptions, IValidateOptions +internal class WorkflowConfigureOptions : IValidateOptions { - private readonly IConfiguration configuration; - - public WorkflowConfigureOptions(IConfiguration configuration) - { - this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - } - - public void PostConfigure(string? name, WorkflowOptions options) - { - options.SubscriptionPassword ??= configuration.GetValue("Authentication:Schemes:ServiceHooks:Credentials:vsts"); - } - public ValidateOptionsResult Validate(string? name, WorkflowOptions options) { if (options.WebhookEndpoint is null) @@ -23,19 +11,9 @@ public ValidateOptionsResult Validate(string? name, WorkflowOptions options) return ValidateOptionsResult.Fail($"'{nameof(options.WebhookEndpoint)}' is required"); } - if (options.ProjectUrl is null) - { - return ValidateOptionsResult.Fail($"'{nameof(options.ProjectUrl)}' is required"); - } - - if (string.IsNullOrWhiteSpace(options.ProjectToken)) - { - return ValidateOptionsResult.Fail($"'{nameof(options.ProjectToken)}' cannot be null or whitespace"); - } - - if (string.IsNullOrWhiteSpace(options.SubscriptionPassword)) + if (options.JobsApiUrl is null) { - return ValidateOptionsResult.Fail($"'{nameof(options.SubscriptionPassword)}' cannot be null or whitespace"); + return ValidateOptionsResult.Fail($"'{nameof(options.JobsApiUrl)}' is required"); } if (string.IsNullOrWhiteSpace(options.ResourceGroupId)) diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index b35205ef..0008dd63 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -1,20 +1,12 @@ -using Tingle.Dependabot.Models; - -namespace Tingle.Dependabot.Workflow; +namespace Tingle.Dependabot.Workflow; public class WorkflowOptions { - /// Whether to synchronize repositories on startup. - public bool SynchronizeOnStartup { get; set; } = true; - - /// Whether to create/update notifications on startup. - public bool CreateOrUpdateWebhooksOnStartup { get; set; } = true; - /// URL where subscription notifications shall be sent. public Uri? WebhookEndpoint { get; set; } - /// Password used for creation of subscription and authenticating incoming notifications. - public string? SubscriptionPassword { get; set; } + /// URL on which to access the API from the jobs. + public Uri? JobsApiUrl { get; set; } /// Resource identifier for the resource group to create jobs in. /// /subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT @@ -38,19 +30,6 @@ public class WorkflowOptions /// ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:1.20 public string? UpdaterContainerImageTemplate { get; set; } - /// URL for the project. - public AzureDevOpsProjectUrl? ProjectUrl { get; set; } - - /// Authentication token for accessing the project. - public string? ProjectToken { get; set; } - - /// Whether to debug all jobs. - public bool? DebugJobs { get; set; } - - /// URL on which to access the API from the jobs. - /// https://dependabot.dummy-123.westeurope.azurecontainerapps.io - public string? JobsApiUrl { get; set; } - /// /// Root working directory where file are written during job scheduling and execution. /// This directory is the root for all jobs. @@ -62,19 +41,6 @@ public class WorkflowOptions /// /mnt/dependabot public string? WorkingDirectory { get; set; } - /// Whether updates should be created in the same order. - public bool? DeterministicUpdates { get; set; } - - /// Whether to set automatic completion of pull requests. - public bool? AutoComplete { get; set; } - - public string? AutoCompleteIgnoreConfigs { get; set; } - - public MergeStrategy? AutoCompleteMergeStrategy { get; set; } - - /// Whether to automatically approve pull requests. - public bool? AutoApprove { get; set; } - /// /// Token for accessing GitHub APIs. /// If no value is provided, calls to GitHub are not authenticated. @@ -85,23 +51,7 @@ public class WorkflowOptions /// ghp_1234567890 public string? GithubToken { get; set; } - /// - /// Secrets that can be replaced in the registries section of the configuration file. - /// - public Dictionary Secrets { get; set; } = new(StringComparer.OrdinalIgnoreCase); - /// Location/region where to create new update jobs. + /// westeurope public string? Location { get; set; } // using Azure.Core.Location does not work when binding from IConfiguration - - /// - /// Possible/allowed paths for the configuration files in a repository. - /// - public IReadOnlyList ConfigurationFilePaths { get; set; } = new[] { - // TODO: restore checks in .azuredevops folder once either the code can check that folder or we are passing ignore conditions via update_jobs API - //".azuredevops/dependabot.yml", - //".azuredevops/dependabot.yaml", - - ".github/dependabot.yml", - ".github/dependabot.yaml", - }; } diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index b2f62d73..7d057074 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -22,11 +22,6 @@ "ValidAudiences": [ "http://localhost:3000" ] - }, - "ServiceHooks": { - "Credentials": { - "vsts": "AAAAAAAAAAA=" - } } } }, @@ -50,18 +45,21 @@ }, "Workflow": { - "SynchronizeOnStartup": false, - "CreateOrUpdateWebhooksOnStartup": false, - "WebhookEndpoint": "http://localhost:3000/", - "SubscriptionPassword": "", + "WebhookEndpoint": "http://localhost:3000/webhooks/azure", + "JobsApiUrl": "http://localhost:3000/", "ResourceGroupId": "/subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT", "AppEnvironmentId": "/subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT/providers/Microsoft.App/managedEnvironments/dependabot", "LogAnalyticsWorkspaceId": "00000000-0000-1111-0001-000000000000", "UpdaterContainerImageTemplate": "ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:1.20.0-ci.37", - "ProjectUrl": "https://dev.azure.com/fabrikam/DefaultCollection", - "ProjectToken": "", "WorkingDirectory": "work", "GithubToken": "", "Location": "westeurope" + }, + + "FeatureManagement": { + "DebugAllJobs": false, + "DebugJobs": false, + "DeterministicUpdates": false, + "UpdaterV2": false } } diff --git a/server/main.bicep b/server/main.bicep index 50ccac67..d98290bd 100644 --- a/server/main.bicep +++ b/server/main.bicep @@ -4,46 +4,12 @@ param location string = resourceGroup().location @description('Name of the resources') param name string = 'dependabot' -@description('URL of the project. For example "https://dev.azure.com/fabrikam/DefaultCollection"') -param projectUrl string - -@description('Token for accessing the project.') -param projectToken string - -@description('Whether to synchronize repositories on startup.') -param synchronizeOnStartup bool = false - -@description('Whether to create or update subscriptions on startup.') -param createOrUpdateWebhooksOnStartup bool = false - -@description('Whether to debug all jobs.') -param debugAllJobs bool = false +@description('JSON array string fo projects to setup. E.g. [{"url":"https://dev.azure.com/tingle/dependabot","token":"dummy","AutoComplete":true}]') +param projectSetups string = '[]' @description('Access token for authenticating requests to GitHub.') param githubToken string = '' -@description('Whether to set auto complete on created pull requests.') -param autoComplete bool = true - -@description('Identifiers of configs to be ignored in auto complete. E.g 3,4,10') -param autoCompleteIgnoreConfigs array = [] - -@allowed([ - 'NoFastForward' - 'Rebase' - 'RebaseMerge' - 'Squash' -]) -@description('Merge strategy to use when setting auto complete on created pull requests.') -param autoCompleteMergeStrategy string = 'Squash' - -@description('Whether to automatically approve created pull requests.') -param autoApprove bool = false - -@description('Password for Webhooks, ServiceHooks, and Notifications from Azure DevOps.') -#disable-next-line secure-secrets-in-params // need sensible defaults -param notificationsPassword string = uniqueString('service-hooks', resourceGroup().id) // e.g. zecnx476et7xm (13 characters) - @description('Tag of the docker images.') param imageTag string = '#{GITVERSION_NUGETVERSIONV2}#' @@ -258,6 +224,7 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { ingress: { external: true, targetPort: 80, traffic: [ { latestRevision: true, weight: 100 } ] } secrets: [ { name: 'connection-strings-application-insights', value: appInsights.properties.ConnectionString } + { name: 'project-setups', value: projectSetups } { name: 'connection-strings-sql' value: join([ @@ -272,8 +239,6 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { 'Connection Timeout=30' ], ';') } - { name: 'notifications-password', value: notificationsPassword } - { name: 'project-token', value: projectToken } { name: 'connection-strings-asb-scaler', value: serviceBusNamespace::authorizationRule.listKeys().primaryConnectionString } ] } @@ -291,6 +256,8 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { { name: 'ASPNETCORE_FORWARDEDHEADERS_ENABLED', value: 'true' } // Application is behind proxy { name: 'EFCORE_PERFORM_MIGRATIONS', value: 'true' } // Perform migrations on startup + { name: 'PROJECT_SETUPS', secretRef: 'project-setups' } + { name: 'AzureAppConfig__Endpoint', value: appConfiguration.properties.endpoint } { name: 'AzureAppConfig__Label', value: 'Production' } @@ -304,29 +271,18 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { { name: 'Logging__Seq__ServerUrl', value: '' } // set via AppConfig { name: 'Logging__Seq__ApiKey', value: '' } // set via AppConfig - { name: 'Workflow__SynchronizeOnStartup', value: synchronizeOnStartup ? 'true' : 'false' } - { name: 'Workflow__CreateOrUpdateWebhooksOnStartup', value: createOrUpdateWebhooksOnStartup ? 'true' : 'false' } - { name: 'Workflow__ProjectUrl', value: projectUrl } - { name: 'Workflow__ProjectToken', secretRef: 'project-token' } - { name: 'Workflow__DebugJobs', value: '${debugAllJobs}' } { name: 'Workflow__JobsApiUrl', value: 'https://${name}.${appEnvironment.properties.defaultDomain}' } { name: 'Workflow__WorkingDirectory', value: '/mnt/dependabot' } { name: 'Workflow__WebhookEndpoint', value: 'https://${name}.${appEnvironment.properties.defaultDomain}/webhooks/azure' } - { name: 'Workflow__SubscriptionPassword', secretRef: 'notifications-password' } { name: 'Workflow__ResourceGroupId', value: resourceGroup().id } { name: 'Workflow__AppEnvironmentId', value: appEnvironment.id } { name: 'Workflow__LogAnalyticsWorkspaceId', value: logAnalyticsWorkspace.properties.customerId } { name: 'Workflow__UpdaterContainerImageTemplate', value: 'ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:${imageTag}' } - { name: 'Workflow__AutoComplete', value: autoComplete ? 'true' : 'false' } - { name: 'Workflow__AutoCompleteIgnoreConfigs', value: join(autoCompleteIgnoreConfigs, ';') } - { name: 'Workflow__AutoCompleteMergeStrategy', value: autoCompleteMergeStrategy } - { name: 'Workflow__AutoApprove', value: autoApprove ? 'true' : 'false' } { name: 'Workflow__GithubToken', value: githubToken } { name: 'Workflow__Location', value: location } { name: 'Authentication__Schemes__Management__Authority', value: '${environment().authentication.loginEndpoint}${subscription().tenantId}/v2.0' } { name: 'Authentication__Schemes__Management__ValidAudiences__0', value: 'https://${name}.${appEnvironment.properties.defaultDomain}' } - { name: 'Authentication__Schemes__ServiceHooks__Credentials__vsts', secretRef: 'notifications-password' } { name: 'EventBus__SelectedTransport', value: 'ServiceBus' } { name: 'EventBus__Transports__azure-service-bus__FullyQualifiedNamespace', value: split(split(serviceBusNamespace.properties.serviceBusEndpoint, '/')[2], ':')[0] } // manipulating https://{your-namespace}.servicebus.windows.net:443/ @@ -418,5 +374,3 @@ resource logAnalyticsReaderRoleAssignment 'Microsoft.Authorization/roleAssignmen #disable-next-line outputs-should-not-contain-secrets output sqlServerAdministratorLoginPassword string = sqlServerAdministratorLoginPassword output webhookEndpoint string = 'https://${app.properties.configuration.ingress.fqdn}/webhooks/azure' -#disable-next-line outputs-should-not-contain-secrets -output notificationsPassword string = notificationsPassword diff --git a/server/main.json b/server/main.json index 7c47b07c..d9f20455 100644 --- a/server/main.json +++ b/server/main.json @@ -16,37 +16,11 @@ "description": "Name of the resources" } }, - "projectUrl": { + "projectSetups": { "type": "string", + "defaultValue": "[[]", "metadata": { - "description": "URL of the project. For example \"https://dev.azure.com/fabrikam/DefaultCollection\"" - } - }, - "projectToken": { - "type": "string", - "metadata": { - "description": "Token for accessing the project." - } - }, - "synchronizeOnStartup": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to synchronize repositories on startup." - } - }, - "createOrUpdateWebhooksOnStartup": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to create or update subscriptions on startup." - } - }, - "debugAllJobs": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to debug all jobs." + "description": "JSON array string fo projects to setup. E.g. [{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]" } }, "githubToken": { @@ -56,47 +30,6 @@ "description": "Access token for authenticating requests to GitHub." } }, - "autoComplete": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Whether to set auto complete on created pull requests." - } - }, - "autoCompleteIgnoreConfigs": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Identifiers of configs to be ignored in auto complete. E.g 3,4,10" - } - }, - "autoCompleteMergeStrategy": { - "type": "string", - "defaultValue": "Squash", - "allowedValues": [ - "NoFastForward", - "Rebase", - "RebaseMerge", - "Squash" - ], - "metadata": { - "description": "Merge strategy to use when setting auto complete on created pull requests." - } - }, - "autoApprove": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to automatically approve created pull requests." - } - }, - "notificationsPassword": { - "type": "string", - "defaultValue": "[uniqueString('service-hooks', resourceGroup().id)]", - "metadata": { - "description": "Password for Webhooks, ServiceHooks, and Notifications from Azure DevOps." - } - }, "imageTag": { "type": "string", "defaultValue": "#{GITVERSION_NUGETVERSIONV2}#", @@ -402,16 +335,12 @@ "value": "[reference(resourceId('Microsoft.Insights/components', parameters('name')), '2020-02-02').ConnectionString]" }, { - "name": "connection-strings-sql", - "value": "[join(createArray(format('Server=tcp:{0},1433', reference(resourceId('Microsoft.Sql/servers', format('{0}-{1}', parameters('name'), variables('collisionSuffix'))), '2022-05-01-preview').fullyQualifiedDomainName), format('Initial Catalog={0}', parameters('name')), format('User ID={0}', variables('sqlServerAdministratorLogin')), format('Password={0}', variables('sqlServerAdministratorLoginPassword')), 'Persist Security Info=False', 'MultipleActiveResultSets=False', 'Encrypt=True', 'TrustServerCertificate=False', 'Connection Timeout=30'), ';')]" - }, - { - "name": "notifications-password", - "value": "[parameters('notificationsPassword')]" + "name": "project-setups", + "value": "[parameters('projectSetups')]" }, { - "name": "project-token", - "value": "[parameters('projectToken')]" + "name": "connection-strings-sql", + "value": "[join(createArray(format('Server=tcp:{0},1433', reference(resourceId('Microsoft.Sql/servers', format('{0}-{1}', parameters('name'), variables('collisionSuffix'))), '2022-05-01-preview').fullyQualifiedDomainName), format('Initial Catalog={0}', parameters('name')), format('User ID={0}', variables('sqlServerAdministratorLogin')), format('Password={0}', variables('sqlServerAdministratorLoginPassword')), 'Persist Security Info=False', 'MultipleActiveResultSets=False', 'Encrypt=True', 'TrustServerCertificate=False', 'Connection Timeout=30'), ';')]" }, { "name": "connection-strings-asb-scaler", @@ -447,6 +376,10 @@ "name": "EFCORE_PERFORM_MIGRATIONS", "value": "true" }, + { + "name": "PROJECT_SETUPS", + "secretRef": "project-setups" + }, { "name": "AzureAppConfig__Endpoint", "value": "[reference(resourceId('Microsoft.AppConfiguration/configurationStores', format('{0}-{1}', parameters('name'), variables('collisionSuffix'))), '2023-03-01').endpoint]" @@ -483,26 +416,6 @@ "name": "Logging__Seq__ApiKey", "value": "" }, - { - "name": "Workflow__SynchronizeOnStartup", - "value": "[if(parameters('synchronizeOnStartup'), 'true', 'false')]" - }, - { - "name": "Workflow__CreateOrUpdateWebhooksOnStartup", - "value": "[if(parameters('createOrUpdateWebhooksOnStartup'), 'true', 'false')]" - }, - { - "name": "Workflow__ProjectUrl", - "value": "[parameters('projectUrl')]" - }, - { - "name": "Workflow__ProjectToken", - "secretRef": "project-token" - }, - { - "name": "Workflow__DebugJobs", - "value": "[format('{0}', parameters('debugAllJobs'))]" - }, { "name": "Workflow__JobsApiUrl", "value": "[format('https://{0}.{1}', parameters('name'), reference(resourceId('Microsoft.App/managedEnvironments', parameters('name')), '2023-05-01').defaultDomain)]" @@ -515,10 +428,6 @@ "name": "Workflow__WebhookEndpoint", "value": "[format('https://{0}.{1}/webhooks/azure', parameters('name'), reference(resourceId('Microsoft.App/managedEnvironments', parameters('name')), '2023-05-01').defaultDomain)]" }, - { - "name": "Workflow__SubscriptionPassword", - "secretRef": "notifications-password" - }, { "name": "Workflow__ResourceGroupId", "value": "[resourceGroup().id]" @@ -535,22 +444,6 @@ "name": "Workflow__UpdaterContainerImageTemplate", "value": "[format('ghcr.io/tinglesoftware/dependabot-updater-{{{{ecosystem}}}}:{0}', parameters('imageTag'))]" }, - { - "name": "Workflow__AutoComplete", - "value": "[if(parameters('autoComplete'), 'true', 'false')]" - }, - { - "name": "Workflow__AutoCompleteIgnoreConfigs", - "value": "[join(parameters('autoCompleteIgnoreConfigs'), ';')]" - }, - { - "name": "Workflow__AutoCompleteMergeStrategy", - "value": "[parameters('autoCompleteMergeStrategy')]" - }, - { - "name": "Workflow__AutoApprove", - "value": "[if(parameters('autoApprove'), 'true', 'false')]" - }, { "name": "Workflow__GithubToken", "value": "[parameters('githubToken')]" @@ -567,10 +460,6 @@ "name": "Authentication__Schemes__Management__ValidAudiences__0", "value": "[format('https://{0}.{1}', parameters('name'), reference(resourceId('Microsoft.App/managedEnvironments', parameters('name')), '2023-05-01').defaultDomain)]" }, - { - "name": "Authentication__Schemes__ServiceHooks__Credentials__vsts", - "secretRef": "notifications-password" - }, { "name": "EventBus__SelectedTransport", "value": "ServiceBus" @@ -715,10 +604,6 @@ "webhookEndpoint": { "type": "string", "value": "[format('https://{0}/webhooks/azure', reference(resourceId('Microsoft.App/containerApps', parameters('name')), '2023-05-01').configuration.ingress.fqdn)]" - }, - "notificationsPassword": { - "type": "string", - "value": "[parameters('notificationsPassword')]" } } } \ No newline at end of file