Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support multiple projects on the server #812

Merged
merged 24 commits into from
Sep 21, 2023
Merged
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
2075e5b
Save projects in the database
mburumaxwell Sep 20, 2023
2638c76
Use feature management to set debug env/property
mburumaxwell Sep 20, 2023
509eebe
Make use of values in the project instead of WorkflowOptions
mburumaxwell Sep 20, 2023
1aecbaa
Make ProviderId required
mburumaxwell Sep 20, 2023
abfcb9d
Use project from database for operations in AzureDevOpsProvider
mburumaxwell Sep 20, 2023
fb4d64b
Remove unused options in WorkflowOptions
mburumaxwell Sep 20, 2023
2b12599
Move secrets from WorkflowOptions to Project
mburumaxwell Sep 20, 2023
8a684de
Use feature flag to set UPDATER_DETERMINISTIC
mburumaxwell Sep 20, 2023
2542790
Change WorkflowOptions.JobsApiUrl to Uri
mburumaxwell Sep 20, 2023
5abe665
Update basic authentication to check credentials against database
mburumaxwell Sep 20, 2023
2716105
Do not expose Project.Token
mburumaxwell Sep 20, 2023
14ee059
Move ConfigurationFilePaths to AzureDevOpsProvider
mburumaxwell Sep 20, 2023
2e2be77
Nest settings for auto complete and auto approve
mburumaxwell Sep 20, 2023
7c5ed12
Expand UpdateJob.Error so that we can search the error types
mburumaxwell Sep 20, 2023
45c6279
Track project identifier in Application Insights
mburumaxwell Sep 20, 2023
7a68813
Added support to setup projects on startup
mburumaxwell Sep 20, 2023
a0dfbbe
synchronize and create/update subscription on startup if there are se…
mburumaxwell Sep 20, 2023
1fcb130
Allow setting GitHubToken and Location per project
mburumaxwell Sep 20, 2023
76392cc
Added ProjectId to the UpdateJob
mburumaxwell Sep 20, 2023
5d63aa5
Update project during sync and skip sync if done within the last hour
mburumaxwell Sep 20, 2023
bd2cef5
Save project visibility
mburumaxwell Sep 20, 2023
32c6d50
Select V2 updater using feature flag
mburumaxwell Sep 20, 2023
365d226
Update unit tests
mburumaxwell Sep 20, 2023
ff2b861
Fix issue with app insights
mburumaxwell Sep 20, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Make use of values in the project instead of WorkflowOptions
mburumaxwell committed Sep 20, 2023
commit 509eebee80d6b9f2daf915d36ac1db4754a56ece
7 changes: 6 additions & 1 deletion server/Tingle.Dependabot/AppSetup.cs
Original file line number Diff line number Diff line change
@@ -26,7 +26,12 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel
if (options.SynchronizeOnStartup)
{
var synchronizer = provider.GetRequiredService<Synchronizer>();
await synchronizer.SynchronizeAsync(false, cancellationToken); /* database sync should not trigger, just in case it's too many */
var context = provider.GetRequiredService<MainDbContext>();
var projects = await context.Projects.ToListAsync(cancellationToken);
foreach (var project in projects)
{
await synchronizer.SynchronizeAsync(project, false, cancellationToken); /* database sync should not trigger, just in case it's too many */
}
}

// skip loading schedules if told to
3 changes: 3 additions & 0 deletions server/Tingle.Dependabot/Constants.cs
Original file line number Diff line number Diff line change
@@ -15,6 +15,9 @@ internal static class AuthConstants
internal static class ErrorCodes
{
internal const string FeaturesDisabled = "features_disabled";
internal const string ProjectNotFound = "project_not_found";
internal const string RepositoryNotFound = "repository_not_found";
internal const string RepositoryUpdateNotFound = "repository_update_not_found";
}

internal static class FeatureNames
Original file line number Diff line number Diff line change
@@ -22,29 +22,37 @@ public ProcessSynchronizationConsumer(MainDbContext dbContext, Synchronizer sync
public async Task ConsumeAsync(EventContext<ProcessSynchronization> context, CancellationToken cancellationToken = default)
{
var evt = context.Event;

var trigger = evt.Trigger;

// ensure project exists
var projectId = evt.ProjectId ?? throw new InvalidOperationException($"'{nameof(evt.ProjectId)}' cannot be null");
var project = await dbContext.Projects.SingleOrDefaultAsync(r => r.Id == projectId, cancellationToken);
if (project is null)
{
logger.LogWarning("Skipping trigger for update because project '{Project}' does not exist.", projectId);
return;
}

if (evt.RepositoryId is not null)
{
// ensure repository exists
var repositoryId = evt.RepositoryId ?? throw new InvalidOperationException($"'{nameof(evt.RepositoryId)}' cannot be null");
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == repositoryId, cancellationToken);
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == repositoryId, cancellationToken);
if (repository is null)
{
logger.LogWarning("Skipping synchronization because repository '{Repository}' does not exist.", repositoryId);
return;
}

await synchronizer.SynchronizeAsync(repository, trigger, cancellationToken);
await synchronizer.SynchronizeAsync(project, repository, trigger, cancellationToken);
}
else if (evt.RepositoryProviderId is not null)
{
await synchronizer.SynchronizeAsync(repositoryProviderId: evt.RepositoryProviderId, trigger, cancellationToken);
await synchronizer.SynchronizeAsync(project, repositoryProviderId: evt.RepositoryProviderId, trigger, cancellationToken);
}
else
{
await synchronizer.SynchronizeAsync(evt.Trigger, cancellationToken);
await synchronizer.SynchronizeAsync(project, evt.Trigger, cancellationToken);
}
}
}
Original file line number Diff line number Diff line change
@@ -24,6 +24,15 @@ public async Task ConsumeAsync(EventContext<TriggerUpdateJobsEvent> context, Can
{
var evt = context.Event;

// ensure project exists
var projectId = evt.ProjectId ?? throw new InvalidOperationException($"'{nameof(evt.ProjectId)}' cannot be null");
var project = await dbContext.Projects.SingleOrDefaultAsync(r => r.Id == projectId, cancellationToken);
if (project is null)
{
logger.LogWarning("Skipping trigger for update because project '{Project}' does not exist.", projectId);
return;
}

// ensure repository exists
var repositoryId = evt.RepositoryId ?? throw new InvalidOperationException($"'{nameof(evt.RepositoryId)}' cannot be null");
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == repositoryId, cancellationToken);
@@ -107,7 +116,7 @@ public async Task ConsumeAsync(EventContext<TriggerUpdateJobsEvent> context, Can
}

// call the update runner to run the update
await updateRunner.CreateAsync(repository, update, job, cancellationToken);
await updateRunner.CreateAsync(project, repository, update, job, cancellationToken);

// save changes that may have been made by the updateRunner
update.LatestJobStatus = job.Status;
70 changes: 47 additions & 23 deletions server/Tingle.Dependabot/Controllers/ManagementController.cs
Original file line number Diff line number Diff line change
@@ -29,8 +29,13 @@ public ManagementController(MainDbContext dbContext, IEventPublisher publisher,
[HttpPost("sync")]
public async Task<IActionResult> SyncAsync([FromBody] SynchronizationRequest model)
{
// ensure project exists
var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided");
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

// request synchronization of the project
var evt = new ProcessSynchronization(model.Trigger);
var evt = new ProcessSynchronization(projectId, model.Trigger);
await publisher.PublishAsync(evt);

return Ok();
@@ -39,33 +44,50 @@ public async Task<IActionResult> SyncAsync([FromBody] SynchronizationRequest mod
[HttpPost("/webhooks/register")]
public async Task<IActionResult> WebhooksRegisterAsync()
{
// ensure project exists
var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided");
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

await adoProvider.CreateOrUpdateSubscriptionsAsync();
return Ok();
}

[HttpGet("repos")]
public async Task<IActionResult> GetReposAsync()
{
var repos = await dbContext.Repositories.ToListAsync();
// ensure project exists
var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided");
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

var repos = await dbContext.Repositories.Where(r => r.ProjectId == project.Id).ToListAsync();
return Ok(repos);
}

[HttpGet("repos/{id}")]
public async Task<IActionResult> GetRepoAsync([FromRoute, Required] string id)
{
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id);
// ensure project exists
var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided");
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id);
return Ok(repository);
}

[HttpGet("repos/{id}/jobs/{jobId}")]
public async Task<IActionResult> GetJobAsync([FromRoute, Required] string id, [FromRoute, Required] string jobId)
{
// ensure project exists
var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided");
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

// ensure repository exists
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id);
if (repository is null)
{
return Problem(title: "repository_not_found", statusCode: 400);
}
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id);
if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400);

// find the job
var job = dbContext.UpdateJobs.Where(j => j.RepositoryId == repository.Id && j.Id == jobId).SingleOrDefaultAsync();
@@ -75,15 +97,17 @@ public async Task<IActionResult> GetJobAsync([FromRoute, Required] string id, [F
[HttpPost("repos/{id}/sync")]
public async Task<IActionResult> SyncRepoAsync([FromRoute, Required] string id, [FromBody] SynchronizationRequest model)
{
// ensure project exists
var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided");
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

// ensure repository exists
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id);
if (repository is null)
{
return Problem(title: "repository_not_found", statusCode: 400);
}
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id);
if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400);

// request synchronization of the repository
var evt = new ProcessSynchronization(model.Trigger, repositoryId: repository.Id, null);
var evt = new ProcessSynchronization(projectId, model.Trigger, repositoryId: repository.Id, null);
await publisher.PublishAsync(evt);

return Ok(repository);
@@ -92,23 +116,23 @@ public async Task<IActionResult> SyncRepoAsync([FromRoute, Required] string id,
[HttpPost("repos/{id}/trigger")]
public async Task<IActionResult> TriggerAsync([FromRoute, Required] string id, [FromBody] TriggerUpdateRequest model)
{
// ensure project exists
var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided");
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

// ensure repository exists
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id);
if (repository is null)
{
return Problem(title: "repository_not_found", statusCode: 400);
}
var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id);
if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400);

// ensure the repository update exists
var update = repository.Updates.ElementAtOrDefault(model.Id!.Value);
if (update is null)
{
return Problem(title: "repository_update_not_found", statusCode: 400);
}
if (update is null) return Problem(title: ErrorCodes.RepositoryUpdateNotFound, statusCode: 400);

// trigger update for specific update
var evt = new TriggerUpdateJobsEvent
{
ProjectId = project.Id,
RepositoryId = repository.Id,
RepositoryUpdateId = model.Id.Value,
Trigger = UpdateJobTrigger.Manual,
23 changes: 21 additions & 2 deletions server/Tingle.Dependabot/Controllers/WebhooksController.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using System.Text.Json;
using Tingle.Dependabot.Events;
using Tingle.Dependabot.Models;
using Tingle.Dependabot.Models.Azure;
using Tingle.EventBus;

@@ -12,11 +14,13 @@ namespace Tingle.Dependabot.Controllers;
[Authorize(AuthConstants.PolicyNameServiceHooks)]
public class WebhooksController : ControllerBase // TODO: unit test this
{
private readonly MainDbContext dbContext;
private readonly IEventPublisher publisher;
private readonly ILogger logger;

public WebhooksController(IEventPublisher publisher, ILogger<WebhooksController> logger)
public WebhooksController(MainDbContext dbContext, IEventPublisher publisher, ILogger<WebhooksController> logger)
{
this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext));
this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -37,12 +41,17 @@ public async Task<IActionResult> PostAsync([FromBody] AzureDevOpsEvent model)
var adoRepositoryId = adoRepository.Id;
var defaultBranch = adoRepository.DefaultBranch;

// ensure project exists
var adoProjectId = adoRepository.Project!.Id;
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

// if the updates are not the default branch, then we ignore them
var updatedReferences = resource.RefUpdates!.Select(ru => ru.Name).ToList();
if (updatedReferences.Contains(defaultBranch, StringComparer.OrdinalIgnoreCase))
{
// request synchronization of the repository
var evt = new ProcessSynchronization(true, repositoryProviderId: adoRepositoryId);
var evt = new ProcessSynchronization(project.Id!, trigger: true, repositoryProviderId: adoRepositoryId);
await publisher.PublishAsync(evt);
}
}
@@ -53,6 +62,11 @@ public async Task<IActionResult> PostAsync([FromBody] AzureDevOpsEvent model)
var prId = resource.PullRequestId;
var status = resource.Status;

// ensure project exists
var adoProjectId = adoRepository.Project!.Id;
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

if (type is AzureDevOpsEventType.GitPullRequestUpdated)
{
logger.LogInformation("PR {PullRequestId} in {RepositoryUrl} status updated to {PullRequestStatus}",
@@ -83,6 +97,11 @@ public async Task<IActionResult> PostAsync([FromBody] AzureDevOpsEvent model)
var prId = pr.PullRequestId;
var status = pr.Status;

// ensure project exists
var adoProjectId = adoRepository.Project!.Id;
var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId);
if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400);

// ensure the comment starts with @dependabot
var content = comment.Content?.Trim();
if (content is not null && content.StartsWith("@dependabot"))
Original file line number Diff line number Diff line change
@@ -2,10 +2,12 @@

namespace Tingle.Dependabot.Events;

public record RepositoryCreatedEvent : AbstractRepositoryEvent { }
public record ProjectCreatedEvent : AbstractProjectEvent { }
public record ProjectUpdatedEvent : AbstractProjectEvent { }
public record ProjectDeletedEvent : AbstractProjectEvent { }

public record RepositoryCreatedEvent : AbstractRepositoryEvent { }
public record RepositoryUpdatedEvent : AbstractRepositoryEvent { }

public record RepositoryDeletedEvent : AbstractRepositoryEvent { }

public record TriggerUpdateJobsEvent : AbstractRepositoryEvent
@@ -20,8 +22,14 @@ public record TriggerUpdateJobsEvent : AbstractRepositoryEvent
public required UpdateJobTrigger Trigger { get; set; }
}

public abstract record AbstractRepositoryEvent
public abstract record AbstractRepositoryEvent : AbstractProjectEvent
{
/// <summary>Identifier of the repository.</summary>
public required string? RepositoryId { get; set; }
}

public abstract record AbstractProjectEvent
{
/// <summary>Identifier of the project.</summary>
public required string? ProjectId { get; set; }
}
6 changes: 5 additions & 1 deletion server/Tingle.Dependabot/Events/ProcessSynchronization.cs
Original file line number Diff line number Diff line change
@@ -4,13 +4,17 @@ public record ProcessSynchronization
{
public ProcessSynchronization() { } // required for deserialization

public ProcessSynchronization(bool trigger, string? repositoryId = null, string? repositoryProviderId = null)
public ProcessSynchronization(string projectId, bool trigger, string? repositoryId = null, string? repositoryProviderId = null)
{
ProjectId = projectId ?? throw new ArgumentNullException(nameof(projectId));
Trigger = trigger;
RepositoryId = repositoryId;
RepositoryProviderId = repositoryProviderId;
}

/// <summary>Identifier of the project.</summary>
public string? ProjectId { get; set; }

/// <summary>
/// Indicates whether we should trigger the update jobs where changes have been detected.
/// </summary>

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading