From 2075e5b3d89869677d8079c6eeb12a94c313de0e Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 12:04:45 +0300 Subject: [PATCH 01/24] Save projects in the database --- .../Extensions/PropertyBuilderExtensions.cs | 138 ++++++++++++++++++ .../Extensions/SystemExtensions.cs | 39 +---- .../20230824083425_InitialCreate.Designer.cs | 87 ++++++++++- .../20230824083425_InitialCreate.cs | 68 +++++++-- .../Migrations/MainDbContextModelSnapshot.cs | 87 ++++++++++- .../Tingle.Dependabot/Models/MainDbContext.cs | 38 ++--- .../Models/Management/Project.cs | 58 ++++++++ .../Models/Management/Repository.cs | 5 + 8 files changed, 438 insertions(+), 82 deletions(-) create mode 100644 server/Tingle.Dependabot/Extensions/PropertyBuilderExtensions.cs create mode 100644 server/Tingle.Dependabot/Models/Management/Project.cs diff --git a/server/Tingle.Dependabot/Extensions/PropertyBuilderExtensions.cs b/server/Tingle.Dependabot/Extensions/PropertyBuilderExtensions.cs new file mode 100644 index 00000000..fb798fee --- /dev/null +++ b/server/Tingle.Dependabot/Extensions/PropertyBuilderExtensions.cs @@ -0,0 +1,138 @@ +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; + +namespace Microsoft.EntityFrameworkCore; + +/// +/// Extensions for . +/// +public static class PropertyBuilderExtensions +{ + /// + /// Attach conversion of property to/from stored in the database as concatenated string of strings. + /// + /// + /// The to extend. + /// The separator to use. + /// + public static PropertyBuilder> HasArrayConversion(this PropertyBuilder> propertyBuilder, string separator = ",") where T : IConvertible + => propertyBuilder.HasArrayConversion(separator: separator, serializerOptions: null); + + /// + /// Attach conversion of property to/from stored in the database as concatenated string of strings. + /// + /// + /// The to extend. + /// The to use for enums. + /// + public static PropertyBuilder> HasArrayConversion(this PropertyBuilder> propertyBuilder, JsonSerializerOptions? serializerOptions = null) where T : IConvertible + => propertyBuilder.HasArrayConversion(separator: ",", serializerOptions: serializerOptions); + + /// + /// Attach conversion of property to/from stored in the database as concatenated string of strings. + /// + /// + /// The to extend. + /// The separator to use. + /// The to use for enums. + /// + public static PropertyBuilder> HasArrayConversion(this PropertyBuilder> propertyBuilder, string separator, JsonSerializerOptions? serializerOptions) + where T : IConvertible + { + ArgumentNullException.ThrowIfNull(propertyBuilder); + + var converter = new ValueConverter, string?>( + convertToProviderExpression: v => ConvertToString(v, separator, serializerOptions), + convertFromProviderExpression: v => ConvertFromString(v, separator, serializerOptions)); + + var comparer = new ValueComparer>( + equalsExpression: (l, r) => ConvertToString(l, separator, serializerOptions) == ConvertToString(r, separator, serializerOptions), + hashCodeExpression: v => v == null ? 0 : ConvertToString(v, separator, serializerOptions).GetHashCode(), + snapshotExpression: v => ConvertFromString(ConvertToString(v, separator, serializerOptions), separator, serializerOptions)); + + propertyBuilder.HasConversion(converter); + propertyBuilder.Metadata.SetValueConverter(converter); + propertyBuilder.Metadata.SetValueComparer(comparer); + + return propertyBuilder; + } + + /// + /// Attach conversion of property to/from JSON stored in the database as a string. + /// + /// + /// The to extend. + /// The to use. + /// + public static PropertyBuilder HasJsonConversion(this PropertyBuilder propertyBuilder, JsonSerializerOptions? serializerOptions = null) + { + ArgumentNullException.ThrowIfNull(propertyBuilder); + +#pragma warning disable CS8603 // Possible null reference return. + var converter = new ValueConverter( + convertToProviderExpression: v => ConvertToJson(v, serializerOptions), + convertFromProviderExpression: v => ConvertFromJson(v, serializerOptions)); + + var comparer = new ValueComparer( + equalsExpression: (l, r) => ConvertToJson(l, serializerOptions) == ConvertToJson(r, serializerOptions), + hashCodeExpression: v => v == null ? 0 : ConvertToJson(v, serializerOptions).GetHashCode(), + snapshotExpression: v => ConvertFromJson(ConvertToJson(v, serializerOptions), serializerOptions)); +#pragma warning restore CS8603 // Possible null reference return. + + propertyBuilder.HasConversion(converter); + propertyBuilder.Metadata.SetValueConverter(converter); + propertyBuilder.Metadata.SetValueComparer(comparer); + + return propertyBuilder; + } + + [return: NotNullIfNotNull(nameof(value))] + private static string? ConvertToString(List? value, string separator, JsonSerializerOptions? serializerOptions) where T : IConvertible + { + if (value is null) return null; + if (string.IsNullOrWhiteSpace(separator)) + { + throw new ArgumentException($"'{nameof(separator)}' cannot be null or whitespace.", nameof(separator)); + } + + return typeof(T).IsEnum + ? string.Join(separator, value.Select(t => EnumToString(t, serializerOptions))) + : string.Join(separator, value); + } + + private static List ConvertFromString(string? value, string separator, JsonSerializerOptions? serializerOptions) where T : IConvertible + { + if (string.IsNullOrWhiteSpace(value)) return new List(); + if (string.IsNullOrWhiteSpace(separator)) + { + throw new ArgumentException($"'{nameof(separator)}' cannot be null or whitespace.", nameof(separator)); + } + + var split = value.Split(separator, StringSplitOptions.RemoveEmptyEntries); + return typeof(T).IsEnum + ? split.Select(v => EnumFromString(v, serializerOptions)).ToList() + : split.Select(v => (T)Convert.ChangeType(v, typeof(T))).ToList(); + } + + + private static T EnumFromString(string value, JsonSerializerOptions? serializerOptions) where T : IConvertible + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException($"'{nameof(value)}' cannot be null or whitespace.", nameof(value)); + } + + return JsonSerializer.Deserialize($"\"{value}\"", serializerOptions)!; + } + + private static string EnumToString(T value, JsonSerializerOptions? serializerOptions) + => JsonSerializer.Serialize(value, serializerOptions).Trim('"'); + + + private static string ConvertToJson(T value, JsonSerializerOptions? serializerOptions) => JsonSerializer.Serialize(value, serializerOptions); + + private static T? ConvertFromJson(string? value, JsonSerializerOptions? serializerOptions) => value is null ? default : JsonSerializer.Deserialize(value, serializerOptions); +} diff --git a/server/Tingle.Dependabot/Extensions/SystemExtensions.cs b/server/Tingle.Dependabot/Extensions/SystemExtensions.cs index 5d8951cd..b77eada8 100644 --- a/server/Tingle.Dependabot/Extensions/SystemExtensions.cs +++ b/server/Tingle.Dependabot/Extensions/SystemExtensions.cs @@ -1,9 +1,4 @@ -using Microsoft.EntityFrameworkCore.ChangeTracking; -using Microsoft.EntityFrameworkCore.Metadata.Builders; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using System.Text.Json; - -namespace System; +namespace System; internal static class SystemExtensions { @@ -29,36 +24,4 @@ public static IDictionary AddIfNotDefault(this IDict return dictionary; } - - /// - /// Attach conversion of property to/from JSON stored in the database as a string. - /// - /// - /// The to extend. - /// The to use. - /// - public static PropertyBuilder HasJsonConversion(this PropertyBuilder propertyBuilder, JsonSerializerOptions? serializerOptions = null) - { - ArgumentNullException.ThrowIfNull(propertyBuilder); - -#pragma warning disable CS8603 // Possible null reference return. - var converter = new ValueConverter( - convertToProviderExpression: v => ConvertToJson(v, serializerOptions), - convertFromProviderExpression: v => ConvertFromJson(v, serializerOptions)); - - var comparer = new ValueComparer( - equalsExpression: (l, r) => ConvertToJson(l, serializerOptions) == ConvertToJson(r, serializerOptions), - hashCodeExpression: v => v == null ? 0 : ConvertToJson(v, serializerOptions).GetHashCode(), - snapshotExpression: v => ConvertFromJson(ConvertToJson(v, serializerOptions), serializerOptions)); -#pragma warning restore CS8603 // Possible null reference return. - - propertyBuilder.HasConversion(converter); - propertyBuilder.Metadata.SetValueConverter(converter); - propertyBuilder.Metadata.SetValueComparer(comparer); - - return propertyBuilder; - } - - private static string ConvertToJson(T value, JsonSerializerOptions? serializerOptions) => JsonSerializer.Serialize(value, serializerOptions); - private static T? ConvertFromJson(string? value, JsonSerializerOptions? serializerOptions) => value is null ? default : JsonSerializer.Deserialize(value, serializerOptions); } diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index d7c11726..22a458f2 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -44,7 +44,66 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("DataProtectionKeys"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.Repository", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("AutoApprove") + .HasColumnType("bit"); + + b.Property("AutoComplete") + .HasColumnType("bit"); + + b.Property("AutoCompleteIgnoreConfigs") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("AutoCompleteMergeStrategy") + .HasColumnType("int"); + + b.Property("Created") + .HasColumnType("datetimeoffset"); + + b.Property("Etag") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("NotificationsPassword") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Token") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .HasColumnType("int"); + + b.Property("Updated") + .HasColumnType("datetimeoffset"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Created") + .IsDescending(); + + b.HasIndex("NotificationsPassword") + .IsDescending(); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { b.Property("Id") .HasMaxLength(50) @@ -69,6 +128,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Name") .HasColumnType("nvarchar(max)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(50)"); + b.Property("ProviderId") .HasColumnType("nvarchar(450)"); @@ -94,6 +157,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("ProviderId") .IsUnique() .HasFilter("[ProviderId] IS NOT NULL"); @@ -101,7 +166,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("Repositories"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => { b.Property("Id") .HasMaxLength(50) @@ -182,9 +247,18 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { - b.OwnsOne("Tingle.Dependabot.Models.UpdateJobResources", "Resources", b1 => + b.HasOne("Tingle.Dependabot.Models.Management.Project", null) + .WithMany("Repositories") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => + { + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobResources", "Resources", b1 => { b1.Property("UpdateJobId") .HasColumnType("nvarchar(50)"); @@ -206,6 +280,11 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Navigation("Resources") .IsRequired(); }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Navigation("Repositories"); + }); #pragma warning restore 612, 618 } } diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 58210021..3b42918f 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -25,25 +25,26 @@ protected override void Up(MigrationBuilder migrationBuilder) }); migrationBuilder.CreateTable( - name: "Repositories", + name: "Projects", columns: table => new { Id = table.Column(type: "nvarchar(50)", maxLength: 50, nullable: false), Created = table.Column(type: "datetimeoffset", nullable: false), Updated = table.Column(type: "datetimeoffset", nullable: false), Name = table.Column(type: "nvarchar(max)", nullable: true), - Slug = table.Column(type: "nvarchar(max)", nullable: true), - ProviderId = table.Column(type: "nvarchar(450)", nullable: true), - LatestCommit = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), - ConfigFileContents = table.Column(type: "nvarchar(max)", nullable: false), - SyncException = table.Column(type: "nvarchar(max)", nullable: true), - Updates = table.Column(type: "nvarchar(max)", nullable: false), - Registries = table.Column(type: "nvarchar(max)", nullable: false), + Type = table.Column(type: "int", nullable: false), + Url = table.Column(type: "nvarchar(max)", nullable: false), + Token = table.Column(type: "nvarchar(max)", nullable: false), + AutoComplete = table.Column(type: "bit", nullable: false), + AutoCompleteIgnoreConfigs = table.Column(type: "nvarchar(max)", nullable: false), + AutoCompleteMergeStrategy = table.Column(type: "int", nullable: false), + AutoApprove = table.Column(type: "bit", nullable: false), + NotificationsPassword = table.Column(type: "nvarchar(450)", nullable: false), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => { - table.PrimaryKey("PK_Repositories", x => x.Id); + table.PrimaryKey("PK_Projects", x => x.Id); }); migrationBuilder.CreateTable( @@ -75,12 +76,58 @@ protected override void Up(MigrationBuilder migrationBuilder) table.PrimaryKey("PK_UpdateJobs", x => x.Id); }); + migrationBuilder.CreateTable( + name: "Repositories", + columns: table => new + { + Id = table.Column(type: "nvarchar(50)", maxLength: 50, nullable: false), + Created = table.Column(type: "datetimeoffset", nullable: false), + Updated = table.Column(type: "datetimeoffset", nullable: false), + ProjectId = table.Column(type: "nvarchar(50)", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: true), + Slug = table.Column(type: "nvarchar(max)", nullable: true), + ProviderId = table.Column(type: "nvarchar(450)", nullable: true), + LatestCommit = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), + ConfigFileContents = table.Column(type: "nvarchar(max)", nullable: false), + SyncException = table.Column(type: "nvarchar(max)", nullable: true), + Updates = table.Column(type: "nvarchar(max)", nullable: false), + Registries = table.Column(type: "nvarchar(max)", nullable: false), + Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_Repositories", x => x.Id); + table.ForeignKey( + name: "FK_Repositories_Projects_ProjectId", + column: x => x.ProjectId, + principalTable: "Projects", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_Projects_Created", + table: "Projects", + column: "Created", + descending: new bool[0]); + + migrationBuilder.CreateIndex( + name: "IX_Projects_NotificationsPassword", + table: "Projects", + column: "NotificationsPassword", + descending: new bool[0]); + migrationBuilder.CreateIndex( name: "IX_Repositories_Created", table: "Repositories", column: "Created", descending: new bool[0]); + migrationBuilder.CreateIndex( + name: "IX_Repositories_ProjectId", + table: "Repositories", + column: "ProjectId"); + migrationBuilder.CreateIndex( name: "IX_Repositories_ProviderId", table: "Repositories", @@ -129,5 +176,8 @@ protected override void Down(MigrationBuilder migrationBuilder) migrationBuilder.DropTable( name: "UpdateJobs"); + + migrationBuilder.DropTable( + name: "Projects"); } } diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index c6fc03f1..2da6f071 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -41,7 +41,66 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("DataProtectionKeys"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.Repository", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Property("Id") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("AutoApprove") + .HasColumnType("bit"); + + b.Property("AutoComplete") + .HasColumnType("bit"); + + b.Property("AutoCompleteIgnoreConfigs") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("AutoCompleteMergeStrategy") + .HasColumnType("int"); + + b.Property("Created") + .HasColumnType("datetimeoffset"); + + b.Property("Etag") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("NotificationsPassword") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Token") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .HasColumnType("int"); + + b.Property("Updated") + .HasColumnType("datetimeoffset"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Created") + .IsDescending(); + + b.HasIndex("NotificationsPassword") + .IsDescending(); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { b.Property("Id") .HasMaxLength(50) @@ -66,6 +125,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Name") .HasColumnType("nvarchar(max)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(50)"); + b.Property("ProviderId") .HasColumnType("nvarchar(450)"); @@ -91,6 +154,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("ProviderId") .IsUnique() .HasFilter("[ProviderId] IS NOT NULL"); @@ -98,7 +163,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("Repositories"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => { b.Property("Id") .HasMaxLength(50) @@ -179,9 +244,18 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { - b.OwnsOne("Tingle.Dependabot.Models.UpdateJobResources", "Resources", b1 => + b.HasOne("Tingle.Dependabot.Models.Management.Project", null) + .WithMany("Repositories") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => + { + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobResources", "Resources", b1 => { b1.Property("UpdateJobId") .HasColumnType("nvarchar(50)"); @@ -203,6 +277,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Resources") .IsRequired(); }); + + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.Navigation("Repositories"); + }); #pragma warning restore 612, 618 } } diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index f5c0b1e9..a2c8a809 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -12,6 +12,7 @@ public class MainDbContext : DbContext, IDataProtectionKeyContext { public MainDbContext(DbContextOptions options) : base(options) { } + public DbSet Projects => Set(); public DbSet Repositories => Set(); public DbSet UpdateJobs => Set(); @@ -21,10 +22,18 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); + modelBuilder.Entity(b => + { + b.Property(p => p.AutoCompleteIgnoreConfigs).HasJsonConversion(); + + b.HasIndex(p => p.Created).IsDescending(); // faster filtering + b.HasIndex(p => p.NotificationsPassword).IsDescending(); // faster filtering + }); + modelBuilder.Entity(b => { b.Property(r => r.Updates).HasJsonConversion(); - HasJsonConversion(b.Property(r => r.Registries)); + b.Property(r => r.Registries).HasJsonConversion(); b.HasIndex(r => r.Created).IsDescending(); // faster filtering b.HasIndex(r => r.ProviderId).IsUnique(); @@ -33,7 +42,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity(b => { b.Property(j => j.PackageEcosystem).IsRequired(); - HasJsonConversion(b.Property(j => j.Error)); + b.Property(j => j.Error).HasJsonConversion(); b.HasIndex(j => j.Created).IsDescending(); // faster filtering b.HasIndex(j => j.RepositoryId); @@ -44,29 +53,4 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.OwnsOne(j => j.Resources); }); } - - static PropertyBuilder HasJsonConversion(PropertyBuilder propertyBuilder, JsonSerializerOptions? serializerOptions = null) - { - ArgumentNullException.ThrowIfNull(propertyBuilder); - -#pragma warning disable CS8603 // Possible null reference return. - var converter = new ValueConverter( - convertToProviderExpression: v => ConvertToJson(v, serializerOptions), - convertFromProviderExpression: v => ConvertFromJson(v, serializerOptions)); - - var comparer = new ValueComparer( - equalsExpression: (l, r) => ConvertToJson(l, serializerOptions) == ConvertToJson(r, serializerOptions), - hashCodeExpression: v => v == null ? 0 : ConvertToJson(v, serializerOptions).GetHashCode(), - snapshotExpression: v => ConvertFromJson(ConvertToJson(v, serializerOptions), serializerOptions)); -#pragma warning restore CS8603 // Possible null reference return. - - propertyBuilder.HasConversion(converter); - propertyBuilder.Metadata.SetValueConverter(converter); - propertyBuilder.Metadata.SetValueComparer(comparer); - - return propertyBuilder; - } - - private static string ConvertToJson(T value, JsonSerializerOptions? serializerOptions) => JsonSerializer.Serialize(value, serializerOptions); - private static T? ConvertFromJson(string? value, JsonSerializerOptions? serializerOptions) => value is null ? default : JsonSerializer.Deserialize(value, serializerOptions); } diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs new file mode 100644 index 00000000..a002e3ac --- /dev/null +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -0,0 +1,58 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; + +namespace Tingle.Dependabot.Models.Management; + +public class Project +{ + [Key, MaxLength(50)] + public string? Id { get; set; } + + public DateTimeOffset Created { get; set; } + + public DateTimeOffset Updated { get; set; } + + /// Name of the project as per provider. + public string? Name { get; set; } + + public ProjectType Type { get; set; } + + /// URL for the project. + /// https://dev.azure.com/tingle/dependabot + [Url] + [Required] + public string? Url { get; set; } + + /// + /// Token for accessing the project with permissions for repositories, pull requests, and service hooks. + /// + [Required] + public string? Token { get; set; } + + /// Whether to set auto complete on created pull requests. + public bool AutoComplete { get; set; } + + /// Identifiers of configs to be ignored in auto complete. + public List AutoCompleteIgnoreConfigs { get; set; } = new(); + + /// Merge strategy to use when setting auto complete on created pull requests. + public MergeStrategy AutoCompleteMergeStrategy { get; set; } = MergeStrategy.Squash; + + /// Whether to automatically approve created pull requests. + public bool AutoApprove { get; set; } + + /// Password for Webhooks, ServiceHooks, and Notifications from the provider. + [Required] + public string? NotificationsPassword { get; set; } + + [JsonIgnore] // only for internal use + public List Repositories { get; set; } = new(); + + [Timestamp] + public byte[]? Etag { get; set; } +} + +public enum ProjectType +{ + Azure, +} diff --git a/server/Tingle.Dependabot/Models/Management/Repository.cs b/server/Tingle.Dependabot/Models/Management/Repository.cs index d2bc4b69..e229c4cc 100644 --- a/server/Tingle.Dependabot/Models/Management/Repository.cs +++ b/server/Tingle.Dependabot/Models/Management/Repository.cs @@ -13,6 +13,11 @@ public class Repository public DateTimeOffset Updated { get; set; } + /// Identifier of the project. + [Required] + [JsonIgnore] // only for internal use + public string? ProjectId { get; set; } + /// Name of the repository as per provider. public string? Name { get; set; } public string? Slug { get; set; } From 2638c76d651966c2b9af59f0d91392ad098a1403 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 12:39:25 +0300 Subject: [PATCH 02/24] Use feature management to set debug env/property --- server/Tingle.Dependabot/Constants.cs | 6 +++ .../Extensions/HttpContextExtensions.cs | 9 +++++ .../IServiceCollectionExtensions.cs | 8 ++-- .../CustomDisabledFeaturesHandler.cs | 4 +- .../ProjectTargetingContextAccessor.cs | 36 +++++++++++++++++ .../Workflow/UpdateRunner.cs | 39 +++++++++++++------ .../Workflow/WorkflowOptions.cs | 3 -- server/Tingle.Dependabot/appsettings.json | 5 +++ 8 files changed, 89 insertions(+), 21 deletions(-) create mode 100644 server/Tingle.Dependabot/Extensions/HttpContextExtensions.cs rename server/Tingle.Dependabot/{ => FeatureManagement}/CustomDisabledFeaturesHandler.cs (92%) create mode 100644 server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs diff --git a/server/Tingle.Dependabot/Constants.cs b/server/Tingle.Dependabot/Constants.cs index 82377473..7a40fefa 100644 --- a/server/Tingle.Dependabot/Constants.cs +++ b/server/Tingle.Dependabot/Constants.cs @@ -16,3 +16,9 @@ internal static class ErrorCodes { internal const string FeaturesDisabled = "features_disabled"; } + +internal static class FeatureNames +{ + internal const string DebugAllJobs = "DebugAllJobs"; + internal const string DebugJobs = "DebugJobs"; +} diff --git a/server/Tingle.Dependabot/Extensions/HttpContextExtensions.cs b/server/Tingle.Dependabot/Extensions/HttpContextExtensions.cs new file mode 100644 index 00000000..c649bdbf --- /dev/null +++ b/server/Tingle.Dependabot/Extensions/HttpContextExtensions.cs @@ -0,0 +1,9 @@ +namespace Microsoft.AspNetCore.Http; + +internal static class HttpContextExtensions +{ + public const string XProjectId = "X-Project-Id"; + + public static string? GetProjectId(this HttpContext httpContext) + => httpContext.Request.Headers.TryGetValue(XProjectId, out var values) ? values.Single() : null; +} diff --git a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs index 6cfd8f74..daedc413 100644 --- a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs +++ b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Medallion.Threading; using Medallion.Threading.FileSystem; using Microsoft.FeatureManagement; +using Tingle.Dependabot.FeatureManagement; using Tingle.Dependabot.Workflow; namespace Microsoft.Extensions.DependencyInjection; @@ -36,10 +37,9 @@ public static IServiceCollection AddStandardFeatureManagement(this IServiceColle builder.AddFeatureFilter(); builder.AddFeatureFilter(); - - // In some scenarios (such as AspNetCore, the TargetingFilter together with an ITargetingContextAccessor - // should be used in place of ContextualTargetingFilter. builder.AddFeatureFilter(); + + builder.Services.AddSingleton(); builder.AddFeatureFilter(); // requires ITargetingContextAccessor builder.Services.Configure(o => o.IgnoreCase = true); @@ -53,7 +53,7 @@ public static IServiceCollection AddWorkflowServices(this IServiceCollection ser services.Configure(configuration); services.ConfigureOptions(); - services.AddSingleton(); + services.AddScoped(); services.AddSingleton(); services.AddScoped(); diff --git a/server/Tingle.Dependabot/CustomDisabledFeaturesHandler.cs b/server/Tingle.Dependabot/FeatureManagement/CustomDisabledFeaturesHandler.cs similarity index 92% rename from server/Tingle.Dependabot/CustomDisabledFeaturesHandler.cs rename to server/Tingle.Dependabot/FeatureManagement/CustomDisabledFeaturesHandler.cs index 8d94e4ac..a3226f90 100644 --- a/server/Tingle.Dependabot/CustomDisabledFeaturesHandler.cs +++ b/server/Tingle.Dependabot/FeatureManagement/CustomDisabledFeaturesHandler.cs @@ -1,5 +1,5 @@ -using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.FeatureManagement.Mvc; namespace Tingle.Dependabot; diff --git a/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs b/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs new file mode 100644 index 00000000..bbafcaa4 --- /dev/null +++ b/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs @@ -0,0 +1,36 @@ +using Microsoft.FeatureManagement.FeatureFilters; + +namespace Tingle.Dependabot.FeatureManagement; + +/// +/// An implementation of +/// that creates a using the current . +/// +internal class ProjectTargetingContextAccessor : ITargetingContextAccessor +{ + private readonly IHttpContextAccessor contextAccessor; + + public ProjectTargetingContextAccessor(IHttpContextAccessor contextAccessor) + { + this.contextAccessor = contextAccessor ?? throw new ArgumentNullException(nameof(contextAccessor)); + } + + /// + public ValueTask GetContextAsync() + { + var httpContext = contextAccessor.HttpContext!; + + // Prepare the groups + var groups = new List(); // where can we get the workspace groups? + + // Build targeting context based off workspace info + var workspaceId = httpContext.GetProjectId(); + var targetingContext = new TargetingContext + { + UserId = workspaceId, + Groups = groups + }; + + return new ValueTask(targetingContext); + } +} diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 383bd4eb..b1d67c05 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -5,6 +5,8 @@ using Azure.ResourceManager.AppContainers.Models; using Azure.ResourceManager.Resources; using Microsoft.Extensions.Options; +using Microsoft.FeatureManagement; +using Microsoft.FeatureManagement.FeatureFilters; using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Nodes; @@ -24,6 +26,7 @@ internal partial class UpdateRunner private static readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + private readonly IFeatureManagerSnapshot featureManager; private readonly WorkflowOptions options; private readonly ILogger logger; @@ -31,8 +34,9 @@ internal partial class UpdateRunner private readonly ResourceGroupResource resourceGroup; private readonly LogsQueryClient logsQueryClient; - public UpdateRunner(IOptions optionsAccessor, ILogger logger) + public UpdateRunner(IFeatureManagerSnapshot featureManager, IOptions optionsAccessor, ILogger logger) { + this.featureManager = featureManager ?? throw new ArgumentNullException(nameof(featureManager)); options = optionsAccessor?.Value ?? throw new ArgumentNullException(nameof(optionsAccessor)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); armClient = new ArmClient(new DefaultAzureCredential()); @@ -40,7 +44,7 @@ public UpdateRunner(IOptions optionsAccessor, ILogger $"dependabot-{job.Id}"; - internal IDictionary CreateEnvironmentVariables(Repository repository, - RepositoryUpdate update, - UpdateJob job, - string directory, - IList> credentials) // TODO: unit test this + internal async Task> CreateEnvironmentVariables(Project project, + Repository repository, + RepositoryUpdate update, + UpdateJob job, + string directory, + IList> credentials, + CancellationToken cancellationToken = default) // TODO: unit test this { [return: NotNullIfNotNull(nameof(value))] static string? ToJson(T? value) => value is null ? null : JsonSerializer.Serialize(value, serializerOptions); // null ensures we do not add to the values + // check if debug is enabled for the project via Feature Management + var fmc = new TargetingContext { Groups = new[] { $"provider:{project.Type.ToString().ToLower()}", $"project:{project.Id}", $"ecosystem:{job.PackageEcosystem}", }, }; + var debugAllJobs = await featureManager.IsEnabledAsync(FeatureNames.DebugAllJobs, fmc); + // Add compulsory values var values = new Dictionary { // env for v2 ["DEPENDABOT_JOB_ID"] = job.Id!, ["DEPENDABOT_JOB_TOKEN"] = job.AuthKey!, - ["DEPENDABOT_DEBUG"] = (options.DebugJobs ?? false).ToString().ToLower(), + ["DEPENDABOT_DEBUG"] = debugAllJobs.ToString().ToLower(), ["DEPENDABOT_API_URL"] = options.JobsApiUrl!, ["DEPENDABOT_JOB_PATH"] = Path.Join(directory, JobDefinitionFileName), ["DEPENDABOT_OUTPUT_PATH"] = Path.Join(directory, "output"), @@ -275,7 +285,8 @@ internal IDictionary CreateEnvironmentVariables(Repository repos return values; } - internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, + internal async Task WriteJobDefinitionAsync(Project project, + RepositoryUpdate update, UpdateJob job, string directory, IList> credentials, @@ -287,6 +298,10 @@ internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, var url = options.ProjectUrl!.Value; var credentialsMetadata = MakeCredentialsMetadata(credentials); + // check if debug is enabled for the project via Feature Management + var fmc = new TargetingContext { Groups = new[] { $"provider:{project.Type.ToString().ToLower()}", $"project:{project.Id}", $"ecosystem:{job.PackageEcosystem}", }, }; + var debug = await featureManager.IsEnabledAsync(FeatureNames.DebugJobs, fmc); + var definition = new JsonObject { ["job"] = new JsonObject @@ -320,7 +335,7 @@ internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, // ["updating-a-pull-request"] = false, ["vendor-dependencies"] = update.Vendor, ["security-updates-only"] = update.OpenPullRequestsLimit == 0, - ["debug"] = false, + ["debug"] = debug, }, ["credentials"] = ToJsonNode(credentials).AsArray(), }; diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index b35205ef..51c23393 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -44,9 +44,6 @@ public class WorkflowOptions /// Authentication token for accessing the project. public string? ProjectToken { get; set; } - /// Whether to debug all jobs. - public bool? DebugJobs { get; set; } - /// URL on which to access the API from the jobs. /// https://dependabot.dummy-123.westeurope.azurecontainerapps.io public string? JobsApiUrl { get; set; } diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index b2f62d73..ac6488d2 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -63,5 +63,10 @@ "WorkingDirectory": "work", "GithubToken": "", "Location": "westeurope" + }, + + "FeatureManagement": { + "DebugAllJobs": false, + "DebugJobs": false } } From 509eebee80d6b9f2daf915d36ac1db4754a56ece Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:09:29 +0300 Subject: [PATCH 03/24] Make use of values in the project instead of WorkflowOptions --- server/Tingle.Dependabot/AppSetup.cs | 7 +- server/Tingle.Dependabot/Constants.cs | 3 + .../ProcessSynchronizationConsumer.cs | 18 +++-- .../TriggerUpdateJobsEventConsumer.cs | 11 ++- .../Controllers/ManagementController.cs | 70 ++++++++++++------ .../Controllers/WebhooksController.cs | 23 +++++- .../{RepositoryCreatedEvent.cs => Events.cs} | 14 +++- .../Events/ProcessSynchronization.cs | 6 +- .../20230824083425_InitialCreate.Designer.cs | 9 ++- .../20230824083425_InitialCreate.cs | 12 +++- .../Migrations/MainDbContextModelSnapshot.cs | 9 ++- .../Tingle.Dependabot/Models/MainDbContext.cs | 5 +- .../Models/Management/Project.cs | 8 ++- .../PeriodicTasks/MissedTriggerCheckerTask.cs | 71 ++++++++++--------- .../PeriodicTasks/SynchronizationTask.cs | 18 +++-- server/Tingle.Dependabot/Program.cs | 17 +---- .../Workflow/Synchronizer.cs | 32 +++++---- .../Workflow/UpdateRunner.cs | 16 ++--- .../Workflow/UpdateScheduler.cs | 6 +- 19 files changed, 233 insertions(+), 122 deletions(-) rename server/Tingle.Dependabot/Events/{RepositoryCreatedEvent.cs => Events.cs} (66%) diff --git a/server/Tingle.Dependabot/AppSetup.cs b/server/Tingle.Dependabot/AppSetup.cs index 0e2f51d5..c3a069d4 100644 --- a/server/Tingle.Dependabot/AppSetup.cs +++ b/server/Tingle.Dependabot/AppSetup.cs @@ -26,7 +26,12 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel if (options.SynchronizeOnStartup) { var synchronizer = provider.GetRequiredService(); - await synchronizer.SynchronizeAsync(false, cancellationToken); /* database sync should not trigger, just in case it's too many */ + var context = provider.GetRequiredService(); + var projects = await context.Projects.ToListAsync(cancellationToken); + foreach (var project in projects) + { + await synchronizer.SynchronizeAsync(project, false, cancellationToken); /* database sync should not trigger, just in case it's too many */ + } } // skip loading schedules if told to diff --git a/server/Tingle.Dependabot/Constants.cs b/server/Tingle.Dependabot/Constants.cs index 7a40fefa..017ebde1 100644 --- a/server/Tingle.Dependabot/Constants.cs +++ b/server/Tingle.Dependabot/Constants.cs @@ -15,6 +15,9 @@ internal static class AuthConstants internal static class ErrorCodes { internal const string FeaturesDisabled = "features_disabled"; + internal const string ProjectNotFound = "project_not_found"; + internal const string RepositoryNotFound = "repository_not_found"; + internal const string RepositoryUpdateNotFound = "repository_update_not_found"; } internal static class FeatureNames diff --git a/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs b/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs index 4676e446..2f397cd8 100644 --- a/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs +++ b/server/Tingle.Dependabot/Consumers/ProcessSynchronizationConsumer.cs @@ -22,29 +22,37 @@ public ProcessSynchronizationConsumer(MainDbContext dbContext, Synchronizer sync public async Task ConsumeAsync(EventContext context, CancellationToken cancellationToken = default) { var evt = context.Event; - var trigger = evt.Trigger; + // ensure project exists + var projectId = evt.ProjectId ?? throw new InvalidOperationException($"'{nameof(evt.ProjectId)}' cannot be null"); + var project = await dbContext.Projects.SingleOrDefaultAsync(r => r.Id == projectId, cancellationToken); + if (project is null) + { + logger.LogWarning("Skipping trigger for update because project '{Project}' does not exist.", projectId); + return; + } + if (evt.RepositoryId is not null) { // ensure repository exists var repositoryId = evt.RepositoryId ?? throw new InvalidOperationException($"'{nameof(evt.RepositoryId)}' cannot be null"); - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == repositoryId, cancellationToken); + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == repositoryId, cancellationToken); if (repository is null) { logger.LogWarning("Skipping synchronization because repository '{Repository}' does not exist.", repositoryId); return; } - await synchronizer.SynchronizeAsync(repository, trigger, cancellationToken); + await synchronizer.SynchronizeAsync(project, repository, trigger, cancellationToken); } else if (evt.RepositoryProviderId is not null) { - await synchronizer.SynchronizeAsync(repositoryProviderId: evt.RepositoryProviderId, trigger, cancellationToken); + await synchronizer.SynchronizeAsync(project, repositoryProviderId: evt.RepositoryProviderId, trigger, cancellationToken); } else { - await synchronizer.SynchronizeAsync(evt.Trigger, cancellationToken); + await synchronizer.SynchronizeAsync(project, evt.Trigger, cancellationToken); } } } diff --git a/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs b/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs index a2a923ae..e3b1b896 100644 --- a/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs +++ b/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs @@ -24,6 +24,15 @@ public async Task ConsumeAsync(EventContext context, Can { var evt = context.Event; + // ensure project exists + var projectId = evt.ProjectId ?? throw new InvalidOperationException($"'{nameof(evt.ProjectId)}' cannot be null"); + var project = await dbContext.Projects.SingleOrDefaultAsync(r => r.Id == projectId, cancellationToken); + if (project is null) + { + logger.LogWarning("Skipping trigger for update because project '{Project}' does not exist.", projectId); + return; + } + // ensure repository exists var repositoryId = evt.RepositoryId ?? throw new InvalidOperationException($"'{nameof(evt.RepositoryId)}' cannot be null"); var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == repositoryId, cancellationToken); @@ -107,7 +116,7 @@ public async Task ConsumeAsync(EventContext context, Can } // call the update runner to run the update - await updateRunner.CreateAsync(repository, update, job, cancellationToken); + await updateRunner.CreateAsync(project, repository, update, job, cancellationToken); // save changes that may have been made by the updateRunner update.LatestJobStatus = job.Status; diff --git a/server/Tingle.Dependabot/Controllers/ManagementController.cs b/server/Tingle.Dependabot/Controllers/ManagementController.cs index 513013a2..a1c083e1 100644 --- a/server/Tingle.Dependabot/Controllers/ManagementController.cs +++ b/server/Tingle.Dependabot/Controllers/ManagementController.cs @@ -29,8 +29,13 @@ public ManagementController(MainDbContext dbContext, IEventPublisher publisher, [HttpPost("sync")] public async Task SyncAsync([FromBody] SynchronizationRequest model) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // request synchronization of the project - var evt = new ProcessSynchronization(model.Trigger); + var evt = new ProcessSynchronization(projectId, model.Trigger); await publisher.PublishAsync(evt); return Ok(); @@ -39,6 +44,11 @@ public async Task SyncAsync([FromBody] SynchronizationRequest mod [HttpPost("/webhooks/register")] public async Task WebhooksRegisterAsync() { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + await adoProvider.CreateOrUpdateSubscriptionsAsync(); return Ok(); } @@ -46,26 +56,38 @@ public async Task WebhooksRegisterAsync() [HttpGet("repos")] public async Task GetReposAsync() { - var repos = await dbContext.Repositories.ToListAsync(); + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + + var repos = await dbContext.Repositories.Where(r => r.ProjectId == project.Id).ToListAsync(); return Ok(repos); } [HttpGet("repos/{id}")] public async Task GetRepoAsync([FromRoute, Required] string id) { - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); return Ok(repository); } [HttpGet("repos/{id}/jobs/{jobId}")] public async Task GetJobAsync([FromRoute, Required] string id, [FromRoute, Required] string jobId) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure repository exists - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); - if (repository is null) - { - return Problem(title: "repository_not_found", statusCode: 400); - } + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); + if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400); // find the job var job = dbContext.UpdateJobs.Where(j => j.RepositoryId == repository.Id && j.Id == jobId).SingleOrDefaultAsync(); @@ -75,15 +97,17 @@ public async Task GetJobAsync([FromRoute, Required] string id, [F [HttpPost("repos/{id}/sync")] public async Task SyncRepoAsync([FromRoute, Required] string id, [FromBody] SynchronizationRequest model) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure repository exists - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); - if (repository is null) - { - return Problem(title: "repository_not_found", statusCode: 400); - } + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); + if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400); // request synchronization of the repository - var evt = new ProcessSynchronization(model.Trigger, repositoryId: repository.Id, null); + var evt = new ProcessSynchronization(projectId, model.Trigger, repositoryId: repository.Id, null); await publisher.PublishAsync(evt); return Ok(repository); @@ -92,23 +116,23 @@ public async Task SyncRepoAsync([FromRoute, Required] string id, [HttpPost("repos/{id}/trigger")] public async Task TriggerAsync([FromRoute, Required] string id, [FromBody] TriggerUpdateRequest model) { + // ensure project exists + var projectId = HttpContext.GetProjectId() ?? throw new InvalidOperationException("Project identifier must be provided"); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure repository exists - var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.Id == id); - if (repository is null) - { - return Problem(title: "repository_not_found", statusCode: 400); - } + var repository = await dbContext.Repositories.SingleOrDefaultAsync(r => r.ProjectId == project.Id && r.Id == id); + if (repository is null) return Problem(title: ErrorCodes.RepositoryNotFound, statusCode: 400); // ensure the repository update exists var update = repository.Updates.ElementAtOrDefault(model.Id!.Value); - if (update is null) - { - return Problem(title: "repository_update_not_found", statusCode: 400); - } + if (update is null) return Problem(title: ErrorCodes.RepositoryUpdateNotFound, statusCode: 400); // trigger update for specific update var evt = new TriggerUpdateJobsEvent { + ProjectId = project.Id, RepositoryId = repository.Id, RepositoryUpdateId = model.Id.Value, Trigger = UpdateJobTrigger.Manual, diff --git a/server/Tingle.Dependabot/Controllers/WebhooksController.cs b/server/Tingle.Dependabot/Controllers/WebhooksController.cs index 667046a8..da4e637f 100644 --- a/server/Tingle.Dependabot/Controllers/WebhooksController.cs +++ b/server/Tingle.Dependabot/Controllers/WebhooksController.cs @@ -1,7 +1,9 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; using System.Text.Json; using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; using Tingle.Dependabot.Models.Azure; using Tingle.EventBus; @@ -12,11 +14,13 @@ namespace Tingle.Dependabot.Controllers; [Authorize(AuthConstants.PolicyNameServiceHooks)] public class WebhooksController : ControllerBase // TODO: unit test this { + private readonly MainDbContext dbContext; private readonly IEventPublisher publisher; private readonly ILogger logger; - public WebhooksController(IEventPublisher publisher, ILogger logger) + public WebhooksController(MainDbContext dbContext, IEventPublisher publisher, ILogger logger) { + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -37,12 +41,17 @@ public async Task PostAsync([FromBody] AzureDevOpsEvent model) var adoRepositoryId = adoRepository.Id; var defaultBranch = adoRepository.DefaultBranch; + // ensure project exists + var adoProjectId = adoRepository.Project!.Id; + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // if the updates are not the default branch, then we ignore them var updatedReferences = resource.RefUpdates!.Select(ru => ru.Name).ToList(); if (updatedReferences.Contains(defaultBranch, StringComparer.OrdinalIgnoreCase)) { // request synchronization of the repository - var evt = new ProcessSynchronization(true, repositoryProviderId: adoRepositoryId); + var evt = new ProcessSynchronization(project.Id!, trigger: true, repositoryProviderId: adoRepositoryId); await publisher.PublishAsync(evt); } } @@ -53,6 +62,11 @@ public async Task PostAsync([FromBody] AzureDevOpsEvent model) var prId = resource.PullRequestId; var status = resource.Status; + // ensure project exists + var adoProjectId = adoRepository.Project!.Id; + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + if (type is AzureDevOpsEventType.GitPullRequestUpdated) { logger.LogInformation("PR {PullRequestId} in {RepositoryUrl} status updated to {PullRequestStatus}", @@ -83,6 +97,11 @@ public async Task PostAsync([FromBody] AzureDevOpsEvent model) var prId = pr.PullRequestId; var status = pr.Status; + // ensure project exists + var adoProjectId = adoRepository.Project!.Id; + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.ProviderId == adoProjectId); + if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); + // ensure the comment starts with @dependabot var content = comment.Content?.Trim(); if (content is not null && content.StartsWith("@dependabot")) diff --git a/server/Tingle.Dependabot/Events/RepositoryCreatedEvent.cs b/server/Tingle.Dependabot/Events/Events.cs similarity index 66% rename from server/Tingle.Dependabot/Events/RepositoryCreatedEvent.cs rename to server/Tingle.Dependabot/Events/Events.cs index f43fac45..6c4e94e3 100644 --- a/server/Tingle.Dependabot/Events/RepositoryCreatedEvent.cs +++ b/server/Tingle.Dependabot/Events/Events.cs @@ -2,10 +2,12 @@ namespace Tingle.Dependabot.Events; -public record RepositoryCreatedEvent : AbstractRepositoryEvent { } +public record ProjectCreatedEvent : AbstractProjectEvent { } +public record ProjectUpdatedEvent : AbstractProjectEvent { } +public record ProjectDeletedEvent : AbstractProjectEvent { } +public record RepositoryCreatedEvent : AbstractRepositoryEvent { } public record RepositoryUpdatedEvent : AbstractRepositoryEvent { } - public record RepositoryDeletedEvent : AbstractRepositoryEvent { } public record TriggerUpdateJobsEvent : AbstractRepositoryEvent @@ -20,8 +22,14 @@ public record TriggerUpdateJobsEvent : AbstractRepositoryEvent public required UpdateJobTrigger Trigger { get; set; } } -public abstract record AbstractRepositoryEvent +public abstract record AbstractRepositoryEvent : AbstractProjectEvent { /// Identifier of the repository. public required string? RepositoryId { get; set; } } + +public abstract record AbstractProjectEvent +{ + /// Identifier of the project. + public required string? ProjectId { get; set; } +} diff --git a/server/Tingle.Dependabot/Events/ProcessSynchronization.cs b/server/Tingle.Dependabot/Events/ProcessSynchronization.cs index 001ffd6c..dea671c8 100644 --- a/server/Tingle.Dependabot/Events/ProcessSynchronization.cs +++ b/server/Tingle.Dependabot/Events/ProcessSynchronization.cs @@ -4,13 +4,17 @@ public record ProcessSynchronization { public ProcessSynchronization() { } // required for deserialization - public ProcessSynchronization(bool trigger, string? repositoryId = null, string? repositoryProviderId = null) + public ProcessSynchronization(string projectId, bool trigger, string? repositoryId = null, string? repositoryProviderId = null) { + ProjectId = projectId ?? throw new ArgumentNullException(nameof(projectId)); Trigger = trigger; RepositoryId = repositoryId; RepositoryProviderId = repositoryProviderId; } + /// Identifier of the project. + public string? ProjectId { get; set; } + /// /// Indicates whether we should trigger the update jobs where changes have been detected. /// diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index 22a458f2..a31e7ced 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -60,7 +60,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("AutoCompleteMergeStrategy") + b.Property("AutoCompleteMergeStrategy") .HasColumnType("int"); b.Property("Created") @@ -78,6 +78,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("ProviderId") + .HasColumnType("nvarchar(450)"); + b.Property("Token") .IsRequired() .HasColumnType("nvarchar(max)"); @@ -100,6 +103,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("NotificationsPassword") .IsDescending(); + b.HasIndex("ProviderId") + .IsUnique() + .HasFilter("[ProviderId] IS NOT NULL"); + b.ToTable("Projects"); }); diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 3b42918f..9e49157e 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -31,13 +31,14 @@ protected override void Up(MigrationBuilder migrationBuilder) Id = table.Column(type: "nvarchar(50)", maxLength: 50, nullable: false), Created = table.Column(type: "datetimeoffset", nullable: false), Updated = table.Column(type: "datetimeoffset", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: true), Type = table.Column(type: "int", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: true), + ProviderId = table.Column(type: "nvarchar(450)", nullable: true), Url = table.Column(type: "nvarchar(max)", nullable: false), Token = table.Column(type: "nvarchar(max)", nullable: false), AutoComplete = table.Column(type: "bit", nullable: false), AutoCompleteIgnoreConfigs = table.Column(type: "nvarchar(max)", nullable: false), - AutoCompleteMergeStrategy = table.Column(type: "int", nullable: false), + AutoCompleteMergeStrategy = table.Column(type: "int", nullable: true), AutoApprove = table.Column(type: "bit", nullable: false), NotificationsPassword = table.Column(type: "nvarchar(450)", nullable: false), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) @@ -117,6 +118,13 @@ protected override void Up(MigrationBuilder migrationBuilder) column: "NotificationsPassword", descending: new bool[0]); + migrationBuilder.CreateIndex( + name: "IX_Projects_ProviderId", + table: "Projects", + column: "ProviderId", + unique: true, + filter: "[ProviderId] IS NOT NULL"); + migrationBuilder.CreateIndex( name: "IX_Repositories_Created", table: "Repositories", diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index 2da6f071..d4390fde 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -57,7 +57,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("AutoCompleteMergeStrategy") + b.Property("AutoCompleteMergeStrategy") .HasColumnType("int"); b.Property("Created") @@ -75,6 +75,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("ProviderId") + .HasColumnType("nvarchar(450)"); + b.Property("Token") .IsRequired() .HasColumnType("nvarchar(max)"); @@ -97,6 +100,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("NotificationsPassword") .IsDescending(); + b.HasIndex("ProviderId") + .IsUnique() + .HasFilter("[ProviderId] IS NOT NULL"); + b.ToTable("Projects"); }); diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index a2c8a809..17aeadbe 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -1,9 +1,5 @@ using Microsoft.AspNetCore.DataProtection.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.ChangeTracking; -using Microsoft.EntityFrameworkCore.Metadata.Builders; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using System.Text.Json; using Tingle.Dependabot.Models.Management; namespace Tingle.Dependabot.Models; @@ -27,6 +23,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.AutoCompleteIgnoreConfigs).HasJsonConversion(); b.HasIndex(p => p.Created).IsDescending(); // faster filtering + b.HasIndex(p => p.ProviderId).IsUnique(); b.HasIndex(p => p.NotificationsPassword).IsDescending(); // faster filtering }); diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index a002e3ac..b0b7660c 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -12,10 +12,14 @@ public class Project public DateTimeOffset Updated { get; set; } + public ProjectType Type { get; set; } + /// Name of the project as per provider. public string? Name { get; set; } - public ProjectType Type { get; set; } + /// Identifier of the repository as per provider. + [JsonIgnore] // only for internal use + public string? ProviderId { get; set; } /// URL for the project. /// https://dev.azure.com/tingle/dependabot @@ -36,7 +40,7 @@ public class Project public List AutoCompleteIgnoreConfigs { get; set; } = new(); /// Merge strategy to use when setting auto complete on created pull requests. - public MergeStrategy AutoCompleteMergeStrategy { get; set; } = MergeStrategy.Squash; + public MergeStrategy? AutoCompleteMergeStrategy { get; set; } /// Whether to automatically approve created pull requests. public bool AutoApprove { get; set; } diff --git a/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs b/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs index d258c43e..8d5f5dcf 100644 --- a/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs +++ b/server/Tingle.Dependabot/PeriodicTasks/MissedTriggerCheckerTask.cs @@ -28,49 +28,54 @@ public async Task ExecuteAsync(PeriodicTaskExecutionContext context, Cancellatio internal virtual async Task CheckAsync(DateTimeOffset referencePoint, CancellationToken cancellationToken = default) { - var repositories = await dbContext.Repositories.ToListAsync(cancellationToken); - - foreach (var repository in repositories) + var projects = await dbContext.Projects.ToListAsync(cancellationToken); + foreach (var project in projects) { - foreach (var update in repository.Updates) - { - var schedule = (CronSchedule)update.Schedule!.GenerateCron(); - var timezone = TimeZoneInfo.FindSystemTimeZoneById(update.Schedule.Timezone); + var repositories = await dbContext.Repositories.Where(r => r.ProjectId == project.Id).ToListAsync(cancellationToken); - // check if we missed an execution - var latestUpdate = update.LatestUpdate; - var missed = latestUpdate is null; // when null, it was missed - if (latestUpdate != null) + foreach (var repository in repositories) + { + foreach (var update in repository.Updates) { - var nextFromLast = schedule.GetNextOccurrence(latestUpdate.Value, timezone); - if (nextFromLast is null) continue; + var schedule = (CronSchedule)update.Schedule!.GenerateCron(); + var timezone = TimeZoneInfo.FindSystemTimeZoneById(update.Schedule.Timezone); - var nextFromReference = schedule.GetNextOccurrence(referencePoint, timezone); - if (nextFromReference is null) continue; + // check if we missed an execution + var latestUpdate = update.LatestUpdate; + var missed = latestUpdate is null; // when null, it was missed + if (latestUpdate != null) + { + var nextFromLast = schedule.GetNextOccurrence(latestUpdate.Value, timezone); + if (nextFromLast is null) continue; - missed = nextFromLast.Value <= referencePoint; // when next is in the past, it was missed + var nextFromReference = schedule.GetNextOccurrence(referencePoint, timezone); + if (nextFromReference is null) continue; - // for daily schedules, only check if the next is more than 12 hours away - if (missed && update.Schedule.Interval is DependabotScheduleInterval.Daily) - { - missed = (nextFromReference.Value - referencePoint).Hours > 12; - } - } + missed = nextFromLast.Value <= referencePoint; // when next is in the past, it was missed - // if we missed an execution, trigger one - if (missed) - { - logger.LogWarning("Schedule was missed for {RepositoryId}({UpdateId}). Triggering now", repository.Id, repository.Updates.IndexOf(update)); + // for daily schedules, only check if the next is more than 12 hours away + if (missed && update.Schedule.Interval is DependabotScheduleInterval.Daily) + { + missed = (nextFromReference.Value - referencePoint).Hours > 12; + } + } - // publish event for the job to be run - var evt = new TriggerUpdateJobsEvent + // if we missed an execution, trigger one + if (missed) { - RepositoryId = repository.Id, - RepositoryUpdateId = repository.Updates.IndexOf(update), - Trigger = UpdateJobTrigger.MissedSchedule, - }; + logger.LogWarning("Schedule was missed for {RepositoryId}({UpdateId}). Triggering now", repository.Id, repository.Updates.IndexOf(update)); - await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + // publish event for the job to be run + var evt = new TriggerUpdateJobsEvent + { + ProjectId = repository.ProjectId, + RepositoryId = repository.Id, + RepositoryUpdateId = repository.Updates.IndexOf(update), + Trigger = UpdateJobTrigger.MissedSchedule, + }; + + await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + } } } } diff --git a/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs b/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs index 6ccff61d..f62249a2 100644 --- a/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs +++ b/server/Tingle.Dependabot/PeriodicTasks/SynchronizationTask.cs @@ -1,4 +1,6 @@ -using Tingle.Dependabot.Events; +using Microsoft.EntityFrameworkCore; +using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; using Tingle.EventBus; using Tingle.PeriodicTasks; @@ -6,10 +8,12 @@ namespace Tingle.Dependabot.Workflow; internal class SynchronizationTask : IPeriodicTask { + private readonly MainDbContext dbContext; private readonly IEventPublisher publisher; - public SynchronizationTask(IEventPublisher publisher) + public SynchronizationTask(MainDbContext dbContext, IEventPublisher publisher) { + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); } @@ -20,8 +24,12 @@ public async Task ExecuteAsync(PeriodicTaskExecutionContext context, Cancellatio internal virtual async Task SyncAsync(CancellationToken cancellationToken = default) { - // request synchronization of the whole project via events - var evt = new ProcessSynchronization(false); /* database sync should not trigger, just in case it's too many */ - await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + // request synchronization of the each project via events + var projects = await dbContext.Projects.ToListAsync(cancellationToken); + foreach (var project in projects) + { + var evt = new ProcessSynchronization(project.Id!, false); /* database sync should not trigger, just in case it's too many */ + await publisher.PublishAsync(evt, cancellationToken: cancellationToken); + } } } diff --git a/server/Tingle.Dependabot/Program.cs b/server/Tingle.Dependabot/Program.cs index 72b0e003..3f7d8eee 100644 --- a/server/Tingle.Dependabot/Program.cs +++ b/server/Tingle.Dependabot/Program.cs @@ -12,6 +12,8 @@ var builder = WebApplication.CreateBuilder(args); +builder.Services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(30)); /* default is 5 seconds */ + // Add Azure AppConfiguration builder.Configuration.AddStandardAzureAppConfiguration(builder.Environment); builder.Services.AddAzureAppConfiguration(); @@ -30,21 +32,6 @@ }); }); -builder.Services.Configure(options => -{ - /* - * The shutdown timer is extended to background tasks (mostly IHostedService) time to close down gracefully - * Andrew Lock explains it in 2 of his posts - * - * https://andrewlock.net/extending-the-shutdown-timeout-setting-to-ensure-graceful-ihostedservice-shutdown/ - * https://andrewlock.net/deploying-asp-net-core-applications-to-kubernetes-part-11-avoiding-downtime-in-rolling-deployments-by-blocking-sigterm/ - * - * The default is 5 seconds but in our case 30 seconds is sufficient and matches the Kubernetes default. - * This should be enough for the services running like the EventBus or incoming HTTP requests to complete processing. - */ - options.ShutdownTimeout = TimeSpan.FromSeconds(30); -}); - builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); // Add DbContext diff --git a/server/Tingle.Dependabot/Workflow/Synchronizer.cs b/server/Tingle.Dependabot/Workflow/Synchronizer.cs index ab9fe78f..06881e99 100644 --- a/server/Tingle.Dependabot/Workflow/Synchronizer.cs +++ b/server/Tingle.Dependabot/Workflow/Synchronizer.cs @@ -38,8 +38,10 @@ public Synchronizer(MainDbContext dbContext, .Build(); } - public async Task SynchronizeAsync(bool trigger, CancellationToken cancellationToken = default) + public async Task SynchronizeAsync(Project project, bool trigger, CancellationToken cancellationToken = default) { + // TODO: skip if the project last synchronization is less than 1 hour ago + // track the synchronization pairs var syncPairs = new List<(SynchronizerConfigurationItem, Repository?)>(); @@ -62,6 +64,7 @@ public async Task SynchronizeAsync(bool trigger, CancellationToken cancellationT // get the repository from the database var adoRepositoryName = adoRepo.Name; var repository = await (from r in dbContext.Repositories + where r.ProjectId == project.Id where r.ProviderId == adoRepositoryId select r).SingleOrDefaultAsync(cancellationToken); @@ -69,7 +72,7 @@ public async Task SynchronizeAsync(bool trigger, CancellationToken cancellationT cancellationToken: cancellationToken); // Track for further synchronization - var sci = new SynchronizerConfigurationItem(options.ProjectUrl!.Value.MakeRepositorySlug(adoRepo.Name), adoRepo, item); + var sci = new SynchronizerConfigurationItem(((AzureDevOpsProjectUrl)project.Url!).MakeRepositorySlug(adoRepo.Name), adoRepo, item); syncPairs.Add((sci, repository)); } @@ -84,11 +87,11 @@ public async Task SynchronizeAsync(bool trigger, CancellationToken cancellationT // synchronize each repository foreach (var (pi, repository) in syncPairs) { - await SynchronizeAsync(repository, pi, trigger, cancellationToken); + await SynchronizeAsync(project, repository, pi, trigger, cancellationToken); } } - public async Task SynchronizeAsync(Repository repository, bool trigger, CancellationToken cancellationToken = default) + public async Task SynchronizeAsync(Project project, Repository repository, bool trigger, CancellationToken cancellationToken = default) { // get repository var adoRepo = await adoProvider.GetRepositoryAsync(repositoryIdOrName: repository.ProviderId!, @@ -106,11 +109,11 @@ public async Task SynchronizeAsync(Repository repository, bool trigger, Cancella cancellationToken: cancellationToken); // perform synchronization - var sci = new SynchronizerConfigurationItem(options.ProjectUrl!.Value.MakeRepositorySlug(adoRepo.Name), adoRepo, item); - await SynchronizeAsync(repository, sci, trigger, cancellationToken); + var sci = new SynchronizerConfigurationItem(((AzureDevOpsProjectUrl)project.Url!).MakeRepositorySlug(adoRepo.Name), adoRepo, item); + await SynchronizeAsync(project, repository, sci, trigger, cancellationToken); } - public async Task SynchronizeAsync(string? repositoryProviderId, bool trigger, CancellationToken cancellationToken = default) + public async Task SynchronizeAsync(Project project, string? repositoryProviderId, bool trigger, CancellationToken cancellationToken = default) { // get repository var adoRepo = await adoProvider.GetRepositoryAsync(repositoryIdOrName: repositoryProviderId!, @@ -132,11 +135,12 @@ public async Task SynchronizeAsync(string? repositoryProviderId, bool trigger, C select r).SingleOrDefaultAsync(cancellationToken); // perform synchronization - var sci = new SynchronizerConfigurationItem(options.ProjectUrl!.Value.MakeRepositorySlug(adoRepo.Name), adoRepo, item); - await SynchronizeAsync(repository, sci, trigger, cancellationToken); + var sci = new SynchronizerConfigurationItem(((AzureDevOpsProjectUrl)project.Url!).MakeRepositorySlug(adoRepo.Name), adoRepo, item); + await SynchronizeAsync(project, repository, sci, trigger, cancellationToken); } - internal async Task SynchronizeAsync(Repository? repository, + internal async Task SynchronizeAsync(Project project, + Repository? repository, SynchronizerConfigurationItem providerInfo, bool trigger, CancellationToken cancellationToken = default) @@ -152,7 +156,7 @@ internal async Task SynchronizeAsync(Repository? repository, await dbContext.SaveChangesAsync(cancellationToken); // publish RepositoryDeletedEvent event - var evt = new RepositoryDeletedEvent { RepositoryId = repository.Id, }; + var evt = new RepositoryDeletedEvent { ProjectId = project.Id, RepositoryId = repository.Id, }; await publisher.PublishAsync(evt, cancellationToken: cancellationToken); } @@ -175,10 +179,11 @@ internal async Task SynchronizeAsync(Repository? repository, { Id = Guid.NewGuid().ToString("n"), Created = DateTimeOffset.UtcNow, + ProjectId = project.Id, ProviderId = providerInfo.Id, }; await dbContext.Repositories.AddAsync(repository, cancellationToken); - rce = new RepositoryCreatedEvent { RepositoryId = repository.Id, }; + rce = new RepositoryCreatedEvent { ProjectId = project.Id, RepositoryId = repository.Id, }; } // if the name of the repository has changed then we assume the commit changed so that we update stuff @@ -235,7 +240,7 @@ internal async Task SynchronizeAsync(Repository? repository, } else { - var evt = new RepositoryUpdatedEvent { RepositoryId = repository.Id, }; + var evt = new RepositoryUpdatedEvent { ProjectId = project.Id, RepositoryId = repository.Id, }; await publisher.PublishAsync(evt, cancellationToken: cancellationToken); } @@ -244,6 +249,7 @@ internal async Task SynchronizeAsync(Repository? repository, // trigger update jobs for the whole repository var evt = new TriggerUpdateJobsEvent { + ProjectId = project.Id, RepositoryId = repository.Id, RepositoryUpdateId = null, // run all Trigger = UpdateJobTrigger.Synchronization, diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index b1d67c05..1a706f0e 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -58,7 +58,7 @@ public async Task CreateAsync(Project project, Repository repository, Repository catch (Azure.RequestFailedException rfe) when (rfe.Status is 404) { } // prepare credentials with replaced secrets - var secrets = new Dictionary(options.Secrets) { ["DEFAULT_TOKEN"] = options.ProjectToken!, }; + var secrets = new Dictionary(options.Secrets) { ["DEFAULT_TOKEN"] = project.Token!, }; var registries = update.Registries?.Select(r => repository.Registries[r]).ToList(); var credentials = MakeExtraCredentials(registries, secrets); // add source credentials when running the in v2 var directory = Path.Join(options.WorkingDirectory, job.Id); @@ -271,16 +271,16 @@ internal async Task> CreateEnvironmentVariables(Proj .AddIfNotDefault("DEPENDABOT_MILESTONE", update.Milestone?.ToString()); // Add values for Azure DevOps - var url = options.ProjectUrl!.Value; + var url = (AzureDevOpsProjectUrl)project.Url!; values.AddIfNotDefault("AZURE_HOSTNAME", url.Hostname) .AddIfNotDefault("AZURE_ORGANIZATION", url.OrganizationName) .AddIfNotDefault("AZURE_PROJECT", url.ProjectName) .AddIfNotDefault("AZURE_REPOSITORY", Uri.EscapeDataString(repository.Name!)) - .AddIfNotDefault("AZURE_ACCESS_TOKEN", options.ProjectToken) - .AddIfNotDefault("AZURE_SET_AUTO_COMPLETE", (options.AutoComplete ?? false).ToString().ToLowerInvariant()) - .AddIfNotDefault("AZURE_AUTO_COMPLETE_IGNORE_CONFIG_IDS", ToJson(options.AutoCompleteIgnoreConfigs?.Split(';'))) - .AddIfNotDefault("AZURE_MERGE_STRATEGY", options.AutoCompleteMergeStrategy?.ToString()) - .AddIfNotDefault("AZURE_AUTO_APPROVE_PR", (options.AutoApprove ?? false).ToString().ToLowerInvariant()); + .AddIfNotDefault("AZURE_ACCESS_TOKEN", project.Token) + .AddIfNotDefault("AZURE_SET_AUTO_COMPLETE", project.AutoComplete.ToString().ToLowerInvariant()) + .AddIfNotDefault("AZURE_AUTO_COMPLETE_IGNORE_CONFIG_IDS", ToJson(project.AutoCompleteIgnoreConfigs ?? new())) + .AddIfNotDefault("AZURE_MERGE_STRATEGY", project.AutoCompleteMergeStrategy?.ToString()) + .AddIfNotDefault("AZURE_AUTO_APPROVE_PR", project.AutoApprove.ToString().ToLowerInvariant()); return values; } @@ -295,7 +295,7 @@ internal async Task WriteJobDefinitionAsync(Project project, [return: NotNullIfNotNull(nameof(value))] static JsonNode? ToJsonNode(T? value) => value is null ? null : JsonSerializer.SerializeToNode(value, serializerOptions); // null ensures we do not add to the values - var url = options.ProjectUrl!.Value; + var url = (AzureDevOpsProjectUrl)project.Url!; var credentialsMetadata = MakeCredentialsMetadata(credentials); // check if debug is enabled for the project via Feature Management diff --git a/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs b/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs index 984d70b9..eff79df5 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateScheduler.cs @@ -37,12 +37,13 @@ public async Task CreateOrUpdateAsync(Repository repository, CancellationToken c updates.Add(new(repository.Updates.IndexOf(update), update.Schedule!)); } + var projectId = repository.ProjectId!; var repositoryId = repository.Id!; var timers = new List(); foreach (var (index, supplied) in updates) { var schedule = supplied.GenerateCron(); - var payload = new TimerPayload(repositoryId, index); + var payload = new TimerPayload(projectId, repositoryId, index); var timer = new CronScheduleTimer(schedule, supplied.Timezone, CustomTimerCallback, payload); timers.Add(timer); } @@ -79,6 +80,7 @@ private async Task CustomTimerCallback(CronScheduleTimer timer, object? arg2, Ca // publish event for the job to be run var evt = new TriggerUpdateJobsEvent { + ProjectId = payload.ProjectId, RepositoryId = payload.RepositoryId, RepositoryUpdateId = payload.RepositoryUpdateId, Trigger = UpdateJobTrigger.Scheduled, @@ -87,5 +89,5 @@ private async Task CustomTimerCallback(CronScheduleTimer timer, object? arg2, Ca await publisher.PublishAsync(evt, cancellationToken: cancellationToken); } - private readonly record struct TimerPayload(string RepositoryId, int RepositoryUpdateId); + private readonly record struct TimerPayload(string ProjectId, string RepositoryId, int RepositoryUpdateId); } From 1aecbaafb77b5b59fadc53ba7a2a3d4a391fbf62 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:11:03 +0300 Subject: [PATCH 04/24] Make ProviderId required --- .../20230824083425_InitialCreate.Designer.cs | 8 ++++---- .../Migrations/20230824083425_InitialCreate.cs | 10 ++++------ .../Migrations/MainDbContextModelSnapshot.cs | 8 ++++---- server/Tingle.Dependabot/Models/Management/Project.cs | 1 + .../Tingle.Dependabot/Models/Management/Repository.cs | 1 + 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index a31e7ced..42d8218e 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -79,6 +79,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("nvarchar(450)"); b.Property("ProviderId") + .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Token") @@ -104,8 +105,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsDescending(); b.HasIndex("ProviderId") - .IsUnique() - .HasFilter("[ProviderId] IS NOT NULL"); + .IsUnique(); b.ToTable("Projects"); }); @@ -140,6 +140,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("nvarchar(50)"); b.Property("ProviderId") + .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Registries") @@ -167,8 +168,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("ProjectId"); b.HasIndex("ProviderId") - .IsUnique() - .HasFilter("[ProviderId] IS NOT NULL"); + .IsUnique(); b.ToTable("Repositories"); }); diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 9e49157e..21019bcc 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -33,7 +33,7 @@ protected override void Up(MigrationBuilder migrationBuilder) Updated = table.Column(type: "datetimeoffset", nullable: false), Type = table.Column(type: "int", nullable: false), Name = table.Column(type: "nvarchar(max)", nullable: true), - ProviderId = table.Column(type: "nvarchar(450)", nullable: true), + ProviderId = table.Column(type: "nvarchar(450)", nullable: false), Url = table.Column(type: "nvarchar(max)", nullable: false), Token = table.Column(type: "nvarchar(max)", nullable: false), AutoComplete = table.Column(type: "bit", nullable: false), @@ -87,7 +87,7 @@ protected override void Up(MigrationBuilder migrationBuilder) ProjectId = table.Column(type: "nvarchar(50)", nullable: false), Name = table.Column(type: "nvarchar(max)", nullable: true), Slug = table.Column(type: "nvarchar(max)", nullable: true), - ProviderId = table.Column(type: "nvarchar(450)", nullable: true), + ProviderId = table.Column(type: "nvarchar(450)", nullable: false), LatestCommit = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), ConfigFileContents = table.Column(type: "nvarchar(max)", nullable: false), SyncException = table.Column(type: "nvarchar(max)", nullable: true), @@ -122,8 +122,7 @@ protected override void Up(MigrationBuilder migrationBuilder) name: "IX_Projects_ProviderId", table: "Projects", column: "ProviderId", - unique: true, - filter: "[ProviderId] IS NOT NULL"); + unique: true); migrationBuilder.CreateIndex( name: "IX_Repositories_Created", @@ -140,8 +139,7 @@ protected override void Up(MigrationBuilder migrationBuilder) name: "IX_Repositories_ProviderId", table: "Repositories", column: "ProviderId", - unique: true, - filter: "[ProviderId] IS NOT NULL"); + unique: true); migrationBuilder.CreateIndex( name: "IX_UpdateJobs_AuthKey", diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index d4390fde..f2f04343 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -76,6 +76,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("nvarchar(450)"); b.Property("ProviderId") + .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Token") @@ -101,8 +102,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsDescending(); b.HasIndex("ProviderId") - .IsUnique() - .HasFilter("[ProviderId] IS NOT NULL"); + .IsUnique(); b.ToTable("Projects"); }); @@ -137,6 +137,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("nvarchar(50)"); b.Property("ProviderId") + .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Registries") @@ -164,8 +165,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("ProjectId"); b.HasIndex("ProviderId") - .IsUnique() - .HasFilter("[ProviderId] IS NOT NULL"); + .IsUnique(); b.ToTable("Repositories"); }); diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index b0b7660c..e51774b0 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -18,6 +18,7 @@ public class Project public string? Name { get; set; } /// Identifier of the repository as per provider. + [Required] [JsonIgnore] // only for internal use public string? ProviderId { get; set; } diff --git a/server/Tingle.Dependabot/Models/Management/Repository.cs b/server/Tingle.Dependabot/Models/Management/Repository.cs index e229c4cc..a61a617b 100644 --- a/server/Tingle.Dependabot/Models/Management/Repository.cs +++ b/server/Tingle.Dependabot/Models/Management/Repository.cs @@ -23,6 +23,7 @@ public class Repository public string? Slug { get; set; } /// Identifier of the repository as per provider. + [Required] [JsonIgnore] // only for internal use public string? ProviderId { get; set; } From abfcb9de92306b2bc440b97d9c95da443aa83829 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:22:19 +0300 Subject: [PATCH 05/24] Use project from database for operations in AzureDevOpsProvider --- server/Tingle.Dependabot/AppSetup.cs | 13 ++++--- .../Controllers/ManagementController.cs | 2 +- .../20230824083425_InitialCreate.Designer.cs | 4 +- .../20230824083425_InitialCreate.cs | 6 +-- .../Migrations/MainDbContextModelSnapshot.cs | 4 +- .../Tingle.Dependabot/Models/MainDbContext.cs | 2 +- .../Models/Management/Project.cs | 3 +- .../Workflow/AzureDevOpsProvider.cs | 37 ++++++++++--------- .../Workflow/Synchronizer.cs | 26 ++++++------- 9 files changed, 50 insertions(+), 47 deletions(-) diff --git a/server/Tingle.Dependabot/AppSetup.cs b/server/Tingle.Dependabot/AppSetup.cs index c3a069d4..6109f17e 100644 --- a/server/Tingle.Dependabot/AppSetup.cs +++ b/server/Tingle.Dependabot/AppSetup.cs @@ -22,12 +22,13 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel } } + var context = provider.GetRequiredService(); + var projects = await context.Projects.ToListAsync(cancellationToken); + var options = provider.GetRequiredService>().Value; if (options.SynchronizeOnStartup) { var synchronizer = provider.GetRequiredService(); - var context = provider.GetRequiredService(); - var projects = await context.Projects.ToListAsync(cancellationToken); foreach (var project in projects) { await synchronizer.SynchronizeAsync(project, false, cancellationToken); /* database sync should not trigger, just in case it's too many */ @@ -37,8 +38,7 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel // skip loading schedules if told to if (!app.Configuration.GetValue("SKIP_LOAD_SCHEDULES")) { - var dbContext = provider.GetRequiredService(); - var repositories = await dbContext.Repositories.ToListAsync(cancellationToken); + var repositories = await context.Repositories.ToListAsync(cancellationToken); var scheduler = provider.GetRequiredService(); foreach (var repository in repositories) { @@ -50,7 +50,10 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel if (options.CreateOrUpdateWebhooksOnStartup) { var adoProvider = provider.GetRequiredService(); - await adoProvider.CreateOrUpdateSubscriptionsAsync(cancellationToken); + foreach (var project in projects) + { + await adoProvider.CreateOrUpdateSubscriptionsAsync(project, cancellationToken); + } } } } diff --git a/server/Tingle.Dependabot/Controllers/ManagementController.cs b/server/Tingle.Dependabot/Controllers/ManagementController.cs index a1c083e1..b25ffb31 100644 --- a/server/Tingle.Dependabot/Controllers/ManagementController.cs +++ b/server/Tingle.Dependabot/Controllers/ManagementController.cs @@ -49,7 +49,7 @@ public async Task WebhooksRegisterAsync() var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == projectId); if (project is null) return Problem(title: ErrorCodes.ProjectNotFound, statusCode: 400); - await adoProvider.CreateOrUpdateSubscriptionsAsync(); + await adoProvider.CreateOrUpdateSubscriptionsAsync(project); return Ok(); } diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index 42d8218e..fc54ac20 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -74,7 +74,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Name") .HasColumnType("nvarchar(max)"); - b.Property("NotificationsPassword") + b.Property("Password") .IsRequired() .HasColumnType("nvarchar(450)"); @@ -101,7 +101,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); - b.HasIndex("NotificationsPassword") + b.HasIndex("Password") .IsDescending(); b.HasIndex("ProviderId") diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 21019bcc..80698650 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -40,7 +40,7 @@ protected override void Up(MigrationBuilder migrationBuilder) AutoCompleteIgnoreConfigs = table.Column(type: "nvarchar(max)", nullable: false), AutoCompleteMergeStrategy = table.Column(type: "int", nullable: true), AutoApprove = table.Column(type: "bit", nullable: false), - NotificationsPassword = table.Column(type: "nvarchar(450)", nullable: false), + Password = table.Column(type: "nvarchar(450)", nullable: false), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => @@ -113,9 +113,9 @@ protected override void Up(MigrationBuilder migrationBuilder) descending: new bool[0]); migrationBuilder.CreateIndex( - name: "IX_Projects_NotificationsPassword", + name: "IX_Projects_Password", table: "Projects", - column: "NotificationsPassword", + column: "Password", descending: new bool[0]); migrationBuilder.CreateIndex( diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index f2f04343..0ba9b8c5 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -71,7 +71,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Name") .HasColumnType("nvarchar(max)"); - b.Property("NotificationsPassword") + b.Property("Password") .IsRequired() .HasColumnType("nvarchar(450)"); @@ -98,7 +98,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); - b.HasIndex("NotificationsPassword") + b.HasIndex("Password") .IsDescending(); b.HasIndex("ProviderId") diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index 17aeadbe..8b8a44a1 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -24,7 +24,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.HasIndex(p => p.Created).IsDescending(); // faster filtering b.HasIndex(p => p.ProviderId).IsUnique(); - b.HasIndex(p => p.NotificationsPassword).IsDescending(); // faster filtering + b.HasIndex(p => p.Password).IsDescending(); // faster filtering }); modelBuilder.Entity(b => diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index e51774b0..f7d10199 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -48,7 +48,8 @@ public class Project /// Password for Webhooks, ServiceHooks, and Notifications from the provider. [Required] - public string? NotificationsPassword { get; set; } + [DataType(DataType.Password)] + public string? Password { get; set; } [JsonIgnore] // only for internal use public List Repositories { get; set; } = new(); diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs index 2bd55263..0a95dd89 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs @@ -8,6 +8,7 @@ using Microsoft.VisualStudio.Services.WebApi; using System.Security.Cryptography; using System.Text; +using Tingle.Dependabot.Models.Management; namespace Tingle.Dependabot.Workflow; @@ -30,11 +31,11 @@ public AzureDevOpsProvider(IMemoryCache cache, IOptions options options = optionsAccessor?.Value ?? throw new ArgumentNullException(nameof(optionsAccessor)); } - public async Task> CreateOrUpdateSubscriptionsAsync(CancellationToken cancellationToken = default) + public async Task> CreateOrUpdateSubscriptionsAsync(Project project, CancellationToken cancellationToken = default) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // get the projectId var projectId = (await (await connection.GetClientAsync(cancellationToken)).GetProject(url.ProjectIdOrName)).Id.ToString(); @@ -64,7 +65,7 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok ConsumerActionId = "httpRequest", })).Results; - var webhookUrl = options.WebhookEndpoint; + var webhookUrl = options.WebhookEndpoint!; var ids = new List(); foreach (var (eventType, resourceVersion) in SubscriptionEventTypes) { @@ -89,7 +90,7 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok existing.EventType = eventType; existing.ResourceVersion = resourceVersion; existing.PublisherInputs = MakeTfsPublisherInputs(eventType, projectId); - existing.ConsumerInputs = MakeWebHooksConsumerInputs(); + existing.ConsumerInputs = MakeWebHooksConsumerInputs(project, webhookUrl); existing = await client.UpdateSubscriptionAsync(existing); } else @@ -103,7 +104,7 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok PublisherInputs = MakeTfsPublisherInputs(eventType, projectId), ConsumerId = "webHooks", ConsumerActionId = "httpRequest", - ConsumerInputs = MakeWebHooksConsumerInputs(), + ConsumerInputs = MakeWebHooksConsumerInputs(project, webhookUrl), }; existing = await client.CreateSubscriptionAsync(existing); } @@ -115,11 +116,11 @@ public async Task> CreateOrUpdateSubscriptionsAsync(CancellationTok return ids; } - public async Task> GetRepositoriesAsync(CancellationToken cancellationToken) + public async Task> GetRepositoriesAsync(Project project, CancellationToken cancellationToken) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // fetch the repositories var client = await connection.GetClientAsync(cancellationToken); @@ -127,22 +128,22 @@ public async Task> GetRepositoriesAsync(CancellationToken ca return repos.OrderBy(r => r.Name).ToList(); } - public async Task GetRepositoryAsync(string repositoryIdOrName, CancellationToken cancellationToken) + public async Task GetRepositoryAsync(Project project, string repositoryIdOrName, CancellationToken cancellationToken) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // get the repository var client = await connection.GetClientAsync(cancellationToken); return await client.GetRepositoryAsync(project: url.ProjectIdOrName, repositoryId: repositoryIdOrName, cancellationToken: cancellationToken); } - public async Task GetConfigurationFileAsync(string repositoryIdOrName, CancellationToken cancellationToken = default) + public async Task GetConfigurationFileAsync(Project project, string repositoryIdOrName, CancellationToken cancellationToken = default) { // get a connection to Azure DevOps - var url = options.ProjectUrl!.Value; - var connection = CreateVssConnection(url, options.ProjectToken!); + var url = (AzureDevOpsProjectUrl)project.Url!; + var connection = CreateVssConnection(url, project.Token!); // Try all known paths var paths = options.ConfigurationFilePaths; @@ -187,7 +188,7 @@ private static Dictionary MakeTfsPublisherInputs(string type, st return result; } - private Dictionary MakeWebHooksConsumerInputs() + private static Dictionary MakeWebHooksConsumerInputs(Project project, Uri webhookUrl) { return new Dictionary { @@ -196,9 +197,9 @@ private Dictionary MakeWebHooksConsumerInputs() ["detailedMessagesToSend"] = "none", ["messagesToSend"] = "none", - ["url"] = options.WebhookEndpoint!.ToString(), + ["url"] = webhookUrl.ToString(), ["basicAuthUsername"] = "vsts", - ["basicAuthPassword"] = options.SubscriptionPassword!, + ["basicAuthPassword"] = project.Password!, }; } diff --git a/server/Tingle.Dependabot/Workflow/Synchronizer.cs b/server/Tingle.Dependabot/Workflow/Synchronizer.cs index 06881e99..92db2245 100644 --- a/server/Tingle.Dependabot/Workflow/Synchronizer.cs +++ b/server/Tingle.Dependabot/Workflow/Synchronizer.cs @@ -1,5 +1,4 @@ using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; using System.ComponentModel.DataAnnotations; using Tingle.Dependabot.Events; using Tingle.Dependabot.Models; @@ -16,21 +15,15 @@ internal class Synchronizer private readonly MainDbContext dbContext; private readonly AzureDevOpsProvider adoProvider; private readonly IEventPublisher publisher; - private readonly WorkflowOptions options; private readonly ILogger logger; private readonly IDeserializer yamlDeserializer; - public Synchronizer(MainDbContext dbContext, - AzureDevOpsProvider adoProvider, - IEventPublisher publisher, - IOptions optionsAccessor, - ILogger logger) + public Synchronizer(MainDbContext dbContext, AzureDevOpsProvider adoProvider, IEventPublisher publisher, ILogger logger) { this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); this.adoProvider = adoProvider ?? throw new ArgumentNullException(nameof(adoProvider)); this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); - options = optionsAccessor?.Value ?? throw new ArgumentNullException(nameof(optionsAccessor)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); yamlDeserializer = new DeserializerBuilder().WithNamingConvention(HyphenatedNamingConvention.Instance) @@ -47,7 +40,7 @@ public async Task SynchronizeAsync(Project project, bool trigger, CancellationTo // get the repositories from Azure logger.LogDebug("Listing repositories ..."); - var adoRepos = await adoProvider.GetRepositoriesAsync(cancellationToken); + var adoRepos = await adoProvider.GetRepositoriesAsync(project, cancellationToken); logger.LogDebug("Found {RepositoriesCount} repositories", adoRepos.Count); var adoReposMap = adoRepos.ToDictionary(r => r.Id.ToString(), r => r); @@ -68,7 +61,8 @@ public async Task SynchronizeAsync(Project project, bool trigger, CancellationTo where r.ProviderId == adoRepositoryId select r).SingleOrDefaultAsync(cancellationToken); - var item = await adoProvider.GetConfigurationFileAsync(repositoryIdOrName: adoRepositoryId, + var item = await adoProvider.GetConfigurationFileAsync(project: project, + repositoryIdOrName: adoRepositoryId, cancellationToken: cancellationToken); // Track for further synchronization @@ -94,7 +88,8 @@ public async Task SynchronizeAsync(Project project, bool trigger, CancellationTo public async Task SynchronizeAsync(Project project, Repository repository, bool trigger, CancellationToken cancellationToken = default) { // get repository - var adoRepo = await adoProvider.GetRepositoryAsync(repositoryIdOrName: repository.ProviderId!, + var adoRepo = await adoProvider.GetRepositoryAsync(project: project, + repositoryIdOrName: repository.ProviderId!, cancellationToken: cancellationToken); // skip disabled or fork repository @@ -105,7 +100,8 @@ public async Task SynchronizeAsync(Project project, Repository repository, bool } // get the configuration file - var item = await adoProvider.GetConfigurationFileAsync(repositoryIdOrName: repository.ProviderId!, + var item = await adoProvider.GetConfigurationFileAsync(project: project, + repositoryIdOrName: repository.ProviderId!, cancellationToken: cancellationToken); // perform synchronization @@ -116,7 +112,8 @@ public async Task SynchronizeAsync(Project project, Repository repository, bool public async Task SynchronizeAsync(Project project, string? repositoryProviderId, bool trigger, CancellationToken cancellationToken = default) { // get repository - var adoRepo = await adoProvider.GetRepositoryAsync(repositoryIdOrName: repositoryProviderId!, + var adoRepo = await adoProvider.GetRepositoryAsync(project: project, + repositoryIdOrName: repositoryProviderId!, cancellationToken: cancellationToken); // skip disabled or fork repository @@ -127,7 +124,8 @@ public async Task SynchronizeAsync(Project project, string? repositoryProviderId } // get the configuration file - var item = await adoProvider.GetConfigurationFileAsync(repositoryIdOrName: repositoryProviderId!, + var item = await adoProvider.GetConfigurationFileAsync(project: project, + repositoryIdOrName: repositoryProviderId!, cancellationToken: cancellationToken); var repository = await (from r in dbContext.Repositories From fb4d64b0a2b67ea6c74f0bc69eb230f228c87126 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:31:45 +0300 Subject: [PATCH 06/24] Remove unused options in WorkflowOptions --- docs/server.md | 18 --- .../Workflow/WorkflowConfigureOptions.cs | 28 +---- .../Workflow/WorkflowOptions.cs | 27 +---- server/Tingle.Dependabot/appsettings.json | 6 +- server/main.bicep | 43 -------- server/main.json | 104 ------------------ 6 files changed, 9 insertions(+), 217 deletions(-) diff --git a/docs/server.md b/docs/server.md index 616acc77..72893b04 100644 --- a/docs/server.md +++ b/docs/server.md @@ -51,18 +51,11 @@ The deployment exposes the following parameters that can be tuned to suit the se |Parameter Name|Remarks|Required|Default| |--|--|--|--| -|projectUrl|The URL of the Azure DevOps project or collection. For example `https://dev.azure.com/fabrikam/DefaultCollection`. This URL must be accessible from the network that the deployment is done in. You can modify the deployment to be done in an private network but you are on your own there.|Yes|**none**| -|projectToken|Personal Access Token (PAT) for accessing the Azure DevOps project. The required permissions are:
- Code (Full)
- Pull Requests Threads (Read & Write).
- Notifications (Read, Write & Manage).
See the [documentation](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#create-a-pat) to know more about creating a Personal Access Token|Yes|**none**| |location|Location to deploy the resources.|No|<resource-group-location>| |name|The name of all resources.|No|`dependabot`| |synchronizeOnStartup|Whether to synchronize repositories on startup. This option is useful for initial deployments since the server synchronizes every 6 hours. Leaving it on has no harm, it actually helps you find out if the token works based on the logs.|No|false| |createOrUpdateWebhooksOnStartup|Whether to create or update Azure DevOps subscriptions on startup. This is required if you want configuration files to be picked up automatically and other event driven functionality.
When this is set to `true`, ensure the value provided for `projectToken` has permissions for service hooks and the owner is a Project Administrator. Leaving this on has no harm because the server will only create new subscription if there are no existing ones based on the URL.|No|false| |githubToken|Access token for authenticating requests to GitHub. Required for vulnerability checks and to avoid rate limiting on free requests|No|<empty>| -|autoComplete|Whether to set auto complete on created pull requests.|No|true| -|autoCompleteIgnoreConfigs|Identifiers of configs to be ignored in auto complete. E.g 3,4,10|No|<empty>| -|autoCompleteMergeStrategy|Merge strategy to use when setting auto complete on created pull requests. Allowed values: `NoFastForward`, `Rebase`, `RebaseMerge`, or `Squash`|No|`Squash`| -|autoApprove|Whether to automatically approve created pull requests.|No|false| -|notificationsPassword|The password used to authenticate incoming requests from Azure DevOps|No|<auto-generated>| |imageTag|The image tag to use when pulling the docker containers. A tag also defines the version. You should avoid using `latest`. Example: `1.1.0`|No|<version-downloaded>| |minReplicas|The minimum number of replicas to required for the deployment. Given that scheduling runs in process, this value cannot be less than `1`. This may change in the future.|No|1| |maxReplicas|The maximum number of replicas when automatic scaling engages. In most cases, you do not need more than 1.|No|1| @@ -78,8 +71,6 @@ For a one time deployment, it is similar to how you deploy other resources on Az ```bash az deployment group create --resource-group DEPENDABOT \ --template-file main.bicep \ - --parameters projectUrl= \ - --parameters projectToken= \ --parameters githubToken= \ --confirm-with-what-if ``` @@ -104,15 +95,6 @@ The parameters file (`dependabot.parameters.json`): "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { - "projectUrl": { - "value": "#{System_TeamFoundationCollectionUri}##{System_TeamProject}#" - }, - "projectToken": { - "value": "#{DependabotProjectToken}#" - }, - "autoComplete": { - "value": true - }, "githubToken": { "value": "#{DependabotGithubToken}#" }, diff --git a/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs index 473a40d1..30d58304 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs @@ -2,20 +2,8 @@ namespace Tingle.Dependabot.Workflow; -internal class WorkflowConfigureOptions : IPostConfigureOptions, IValidateOptions +internal class WorkflowConfigureOptions : IValidateOptions { - private readonly IConfiguration configuration; - - public WorkflowConfigureOptions(IConfiguration configuration) - { - this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - } - - public void PostConfigure(string? name, WorkflowOptions options) - { - options.SubscriptionPassword ??= configuration.GetValue("Authentication:Schemes:ServiceHooks:Credentials:vsts"); - } - public ValidateOptionsResult Validate(string? name, WorkflowOptions options) { if (options.WebhookEndpoint is null) @@ -23,19 +11,9 @@ public ValidateOptionsResult Validate(string? name, WorkflowOptions options) return ValidateOptionsResult.Fail($"'{nameof(options.WebhookEndpoint)}' is required"); } - if (options.ProjectUrl is null) - { - return ValidateOptionsResult.Fail($"'{nameof(options.ProjectUrl)}' is required"); - } - - if (string.IsNullOrWhiteSpace(options.ProjectToken)) - { - return ValidateOptionsResult.Fail($"'{nameof(options.ProjectToken)}' cannot be null or whitespace"); - } - - if (string.IsNullOrWhiteSpace(options.SubscriptionPassword)) + if (options.JobsApiUrl is null) { - return ValidateOptionsResult.Fail($"'{nameof(options.SubscriptionPassword)}' cannot be null or whitespace"); + return ValidateOptionsResult.Fail($"'{nameof(options.JobsApiUrl)}' is required"); } if (string.IsNullOrWhiteSpace(options.ResourceGroupId)) diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index 51c23393..35f86050 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -1,6 +1,4 @@ -using Tingle.Dependabot.Models; - -namespace Tingle.Dependabot.Workflow; +namespace Tingle.Dependabot.Workflow; public class WorkflowOptions { @@ -13,6 +11,9 @@ public class WorkflowOptions /// URL where subscription notifications shall be sent. public Uri? WebhookEndpoint { get; set; } + /// URL on which to access the API from the jobs. + public string? JobsApiUrl { get; set; } + /// Password used for creation of subscription and authenticating incoming notifications. public string? SubscriptionPassword { get; set; } @@ -38,16 +39,6 @@ public class WorkflowOptions /// ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:1.20 public string? UpdaterContainerImageTemplate { get; set; } - /// URL for the project. - public AzureDevOpsProjectUrl? ProjectUrl { get; set; } - - /// Authentication token for accessing the project. - public string? ProjectToken { get; set; } - - /// URL on which to access the API from the jobs. - /// https://dependabot.dummy-123.westeurope.azurecontainerapps.io - public string? JobsApiUrl { get; set; } - /// /// Root working directory where file are written during job scheduling and execution. /// This directory is the root for all jobs. @@ -62,16 +53,6 @@ public class WorkflowOptions /// Whether updates should be created in the same order. public bool? DeterministicUpdates { get; set; } - /// Whether to set automatic completion of pull requests. - public bool? AutoComplete { get; set; } - - public string? AutoCompleteIgnoreConfigs { get; set; } - - public MergeStrategy? AutoCompleteMergeStrategy { get; set; } - - /// Whether to automatically approve pull requests. - public bool? AutoApprove { get; set; } - /// /// Token for accessing GitHub APIs. /// If no value is provided, calls to GitHub are not authenticated. diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index ac6488d2..9ecacde1 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -52,14 +52,12 @@ "Workflow": { "SynchronizeOnStartup": false, "CreateOrUpdateWebhooksOnStartup": false, - "WebhookEndpoint": "http://localhost:3000/", - "SubscriptionPassword": "", + "WebhookEndpoint": "http://localhost:3000/webhooks/azure", + "JobsApiUrl": "http://localhost:3000/", "ResourceGroupId": "/subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT", "AppEnvironmentId": "/subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT/providers/Microsoft.App/managedEnvironments/dependabot", "LogAnalyticsWorkspaceId": "00000000-0000-1111-0001-000000000000", "UpdaterContainerImageTemplate": "ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:1.20.0-ci.37", - "ProjectUrl": "https://dev.azure.com/fabrikam/DefaultCollection", - "ProjectToken": "", "WorkingDirectory": "work", "GithubToken": "", "Location": "westeurope" diff --git a/server/main.bicep b/server/main.bicep index 50ccac67..7ec9f7e5 100644 --- a/server/main.bicep +++ b/server/main.bicep @@ -4,46 +4,15 @@ param location string = resourceGroup().location @description('Name of the resources') param name string = 'dependabot' -@description('URL of the project. For example "https://dev.azure.com/fabrikam/DefaultCollection"') -param projectUrl string - -@description('Token for accessing the project.') -param projectToken string - @description('Whether to synchronize repositories on startup.') param synchronizeOnStartup bool = false @description('Whether to create or update subscriptions on startup.') param createOrUpdateWebhooksOnStartup bool = false -@description('Whether to debug all jobs.') -param debugAllJobs bool = false - @description('Access token for authenticating requests to GitHub.') param githubToken string = '' -@description('Whether to set auto complete on created pull requests.') -param autoComplete bool = true - -@description('Identifiers of configs to be ignored in auto complete. E.g 3,4,10') -param autoCompleteIgnoreConfigs array = [] - -@allowed([ - 'NoFastForward' - 'Rebase' - 'RebaseMerge' - 'Squash' -]) -@description('Merge strategy to use when setting auto complete on created pull requests.') -param autoCompleteMergeStrategy string = 'Squash' - -@description('Whether to automatically approve created pull requests.') -param autoApprove bool = false - -@description('Password for Webhooks, ServiceHooks, and Notifications from Azure DevOps.') -#disable-next-line secure-secrets-in-params // need sensible defaults -param notificationsPassword string = uniqueString('service-hooks', resourceGroup().id) // e.g. zecnx476et7xm (13 characters) - @description('Tag of the docker images.') param imageTag string = '#{GITVERSION_NUGETVERSIONV2}#' @@ -272,8 +241,6 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { 'Connection Timeout=30' ], ';') } - { name: 'notifications-password', value: notificationsPassword } - { name: 'project-token', value: projectToken } { name: 'connection-strings-asb-scaler', value: serviceBusNamespace::authorizationRule.listKeys().primaryConnectionString } ] } @@ -306,21 +273,13 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { { name: 'Workflow__SynchronizeOnStartup', value: synchronizeOnStartup ? 'true' : 'false' } { name: 'Workflow__CreateOrUpdateWebhooksOnStartup', value: createOrUpdateWebhooksOnStartup ? 'true' : 'false' } - { name: 'Workflow__ProjectUrl', value: projectUrl } - { name: 'Workflow__ProjectToken', secretRef: 'project-token' } - { name: 'Workflow__DebugJobs', value: '${debugAllJobs}' } { name: 'Workflow__JobsApiUrl', value: 'https://${name}.${appEnvironment.properties.defaultDomain}' } { name: 'Workflow__WorkingDirectory', value: '/mnt/dependabot' } { name: 'Workflow__WebhookEndpoint', value: 'https://${name}.${appEnvironment.properties.defaultDomain}/webhooks/azure' } - { name: 'Workflow__SubscriptionPassword', secretRef: 'notifications-password' } { name: 'Workflow__ResourceGroupId', value: resourceGroup().id } { name: 'Workflow__AppEnvironmentId', value: appEnvironment.id } { name: 'Workflow__LogAnalyticsWorkspaceId', value: logAnalyticsWorkspace.properties.customerId } { name: 'Workflow__UpdaterContainerImageTemplate', value: 'ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:${imageTag}' } - { name: 'Workflow__AutoComplete', value: autoComplete ? 'true' : 'false' } - { name: 'Workflow__AutoCompleteIgnoreConfigs', value: join(autoCompleteIgnoreConfigs, ';') } - { name: 'Workflow__AutoCompleteMergeStrategy', value: autoCompleteMergeStrategy } - { name: 'Workflow__AutoApprove', value: autoApprove ? 'true' : 'false' } { name: 'Workflow__GithubToken', value: githubToken } { name: 'Workflow__Location', value: location } @@ -418,5 +377,3 @@ resource logAnalyticsReaderRoleAssignment 'Microsoft.Authorization/roleAssignmen #disable-next-line outputs-should-not-contain-secrets output sqlServerAdministratorLoginPassword string = sqlServerAdministratorLoginPassword output webhookEndpoint string = 'https://${app.properties.configuration.ingress.fqdn}/webhooks/azure' -#disable-next-line outputs-should-not-contain-secrets -output notificationsPassword string = notificationsPassword diff --git a/server/main.json b/server/main.json index 7c47b07c..7706cf39 100644 --- a/server/main.json +++ b/server/main.json @@ -16,18 +16,6 @@ "description": "Name of the resources" } }, - "projectUrl": { - "type": "string", - "metadata": { - "description": "URL of the project. For example \"https://dev.azure.com/fabrikam/DefaultCollection\"" - } - }, - "projectToken": { - "type": "string", - "metadata": { - "description": "Token for accessing the project." - } - }, "synchronizeOnStartup": { "type": "bool", "defaultValue": false, @@ -42,13 +30,6 @@ "description": "Whether to create or update subscriptions on startup." } }, - "debugAllJobs": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to debug all jobs." - } - }, "githubToken": { "type": "string", "defaultValue": "", @@ -56,47 +37,6 @@ "description": "Access token for authenticating requests to GitHub." } }, - "autoComplete": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Whether to set auto complete on created pull requests." - } - }, - "autoCompleteIgnoreConfigs": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Identifiers of configs to be ignored in auto complete. E.g 3,4,10" - } - }, - "autoCompleteMergeStrategy": { - "type": "string", - "defaultValue": "Squash", - "allowedValues": [ - "NoFastForward", - "Rebase", - "RebaseMerge", - "Squash" - ], - "metadata": { - "description": "Merge strategy to use when setting auto complete on created pull requests." - } - }, - "autoApprove": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to automatically approve created pull requests." - } - }, - "notificationsPassword": { - "type": "string", - "defaultValue": "[uniqueString('service-hooks', resourceGroup().id)]", - "metadata": { - "description": "Password for Webhooks, ServiceHooks, and Notifications from Azure DevOps." - } - }, "imageTag": { "type": "string", "defaultValue": "#{GITVERSION_NUGETVERSIONV2}#", @@ -405,14 +345,6 @@ "name": "connection-strings-sql", "value": "[join(createArray(format('Server=tcp:{0},1433', reference(resourceId('Microsoft.Sql/servers', format('{0}-{1}', parameters('name'), variables('collisionSuffix'))), '2022-05-01-preview').fullyQualifiedDomainName), format('Initial Catalog={0}', parameters('name')), format('User ID={0}', variables('sqlServerAdministratorLogin')), format('Password={0}', variables('sqlServerAdministratorLoginPassword')), 'Persist Security Info=False', 'MultipleActiveResultSets=False', 'Encrypt=True', 'TrustServerCertificate=False', 'Connection Timeout=30'), ';')]" }, - { - "name": "notifications-password", - "value": "[parameters('notificationsPassword')]" - }, - { - "name": "project-token", - "value": "[parameters('projectToken')]" - }, { "name": "connection-strings-asb-scaler", "value": "[listKeys(resourceId('Microsoft.ServiceBus/namespaces/AuthorizationRules', format('{0}-{1}', parameters('name'), variables('collisionSuffix')), 'RootManageSharedAccessKey'), '2021-11-01').primaryConnectionString]" @@ -491,18 +423,6 @@ "name": "Workflow__CreateOrUpdateWebhooksOnStartup", "value": "[if(parameters('createOrUpdateWebhooksOnStartup'), 'true', 'false')]" }, - { - "name": "Workflow__ProjectUrl", - "value": "[parameters('projectUrl')]" - }, - { - "name": "Workflow__ProjectToken", - "secretRef": "project-token" - }, - { - "name": "Workflow__DebugJobs", - "value": "[format('{0}', parameters('debugAllJobs'))]" - }, { "name": "Workflow__JobsApiUrl", "value": "[format('https://{0}.{1}', parameters('name'), reference(resourceId('Microsoft.App/managedEnvironments', parameters('name')), '2023-05-01').defaultDomain)]" @@ -515,10 +435,6 @@ "name": "Workflow__WebhookEndpoint", "value": "[format('https://{0}.{1}/webhooks/azure', parameters('name'), reference(resourceId('Microsoft.App/managedEnvironments', parameters('name')), '2023-05-01').defaultDomain)]" }, - { - "name": "Workflow__SubscriptionPassword", - "secretRef": "notifications-password" - }, { "name": "Workflow__ResourceGroupId", "value": "[resourceGroup().id]" @@ -535,22 +451,6 @@ "name": "Workflow__UpdaterContainerImageTemplate", "value": "[format('ghcr.io/tinglesoftware/dependabot-updater-{{{{ecosystem}}}}:{0}', parameters('imageTag'))]" }, - { - "name": "Workflow__AutoComplete", - "value": "[if(parameters('autoComplete'), 'true', 'false')]" - }, - { - "name": "Workflow__AutoCompleteIgnoreConfigs", - "value": "[join(parameters('autoCompleteIgnoreConfigs'), ';')]" - }, - { - "name": "Workflow__AutoCompleteMergeStrategy", - "value": "[parameters('autoCompleteMergeStrategy')]" - }, - { - "name": "Workflow__AutoApprove", - "value": "[if(parameters('autoApprove'), 'true', 'false')]" - }, { "name": "Workflow__GithubToken", "value": "[parameters('githubToken')]" @@ -715,10 +615,6 @@ "webhookEndpoint": { "type": "string", "value": "[format('https://{0}/webhooks/azure', reference(resourceId('Microsoft.App/containerApps', parameters('name')), '2023-05-01').configuration.ingress.fqdn)]" - }, - "notificationsPassword": { - "type": "string", - "value": "[parameters('notificationsPassword')]" } } } \ No newline at end of file From 2b1259993ffc36503cc1eb84a866e7a2cfe8a807 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:34:28 +0300 Subject: [PATCH 07/24] Move secrets from WorkflowOptions to Project --- .../Migrations/20230824083425_InitialCreate.Designer.cs | 6 +++++- .../Migrations/20230824083425_InitialCreate.cs | 3 ++- .../Migrations/MainDbContextModelSnapshot.cs | 6 +++++- server/Tingle.Dependabot/Models/MainDbContext.cs | 3 ++- server/Tingle.Dependabot/Models/Management/Project.cs | 6 ++++++ server/Tingle.Dependabot/Workflow/UpdateRunner.cs | 2 +- server/Tingle.Dependabot/Workflow/WorkflowOptions.cs | 5 ----- 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index fc54ac20..e872b662 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -82,6 +82,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("Secrets") + .IsRequired() + .HasColumnType("nvarchar(max)"); + b.Property("Token") .IsRequired() .HasColumnType("nvarchar(max)"); @@ -102,7 +106,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsDescending(); b.HasIndex("Password") - .IsDescending(); + .IsUnique(); b.HasIndex("ProviderId") .IsUnique(); diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 80698650..dcfb7329 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -41,6 +41,7 @@ protected override void Up(MigrationBuilder migrationBuilder) AutoCompleteMergeStrategy = table.Column(type: "int", nullable: true), AutoApprove = table.Column(type: "bit", nullable: false), Password = table.Column(type: "nvarchar(450)", nullable: false), + Secrets = table.Column(type: "nvarchar(max)", nullable: false), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => @@ -116,7 +117,7 @@ protected override void Up(MigrationBuilder migrationBuilder) name: "IX_Projects_Password", table: "Projects", column: "Password", - descending: new bool[0]); + unique: true); migrationBuilder.CreateIndex( name: "IX_Projects_ProviderId", diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index 0ba9b8c5..7c13dc90 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -79,6 +79,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("Secrets") + .IsRequired() + .HasColumnType("nvarchar(max)"); + b.Property("Token") .IsRequired() .HasColumnType("nvarchar(max)"); @@ -99,7 +103,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsDescending(); b.HasIndex("Password") - .IsDescending(); + .IsUnique(); b.HasIndex("ProviderId") .IsUnique(); diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index 8b8a44a1..7884c9cd 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -21,10 +21,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity(b => { b.Property(p => p.AutoCompleteIgnoreConfigs).HasJsonConversion(); + b.Property(p => p.Secrets).HasJsonConversion(); b.HasIndex(p => p.Created).IsDescending(); // faster filtering b.HasIndex(p => p.ProviderId).IsUnique(); - b.HasIndex(p => p.Password).IsDescending(); // faster filtering + b.HasIndex(p => p.Password).IsUnique(); // password should be unique per project }); modelBuilder.Entity(b => diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index f7d10199..0b57754b 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -51,6 +51,12 @@ public class Project [DataType(DataType.Password)] public string? Password { get; set; } + /// + /// Secrets that can be replaced in the registries section of the dependabot configuration file. + /// + [JsonIgnore] // expose this once we know how to protect the values + public Dictionary Secrets { get; set; } = new(StringComparer.OrdinalIgnoreCase); + [JsonIgnore] // only for internal use public List Repositories { get; set; } = new(); diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 1a706f0e..40a9c56c 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -58,7 +58,7 @@ public async Task CreateAsync(Project project, Repository repository, Repository catch (Azure.RequestFailedException rfe) when (rfe.Status is 404) { } // prepare credentials with replaced secrets - var secrets = new Dictionary(options.Secrets) { ["DEFAULT_TOKEN"] = project.Token!, }; + var secrets = new Dictionary(project.Secrets) { ["DEFAULT_TOKEN"] = project.Token!, }; var registries = update.Registries?.Select(r => repository.Registries[r]).ToList(); var credentials = MakeExtraCredentials(registries, secrets); // add source credentials when running the in v2 var directory = Path.Join(options.WorkingDirectory, job.Id); diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index 35f86050..f16d6295 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -63,11 +63,6 @@ public class WorkflowOptions /// ghp_1234567890 public string? GithubToken { get; set; } - /// - /// Secrets that can be replaced in the registries section of the configuration file. - /// - public Dictionary Secrets { get; set; } = new(StringComparer.OrdinalIgnoreCase); - /// Location/region where to create new update jobs. public string? Location { get; set; } // using Azure.Core.Location does not work when binding from IConfiguration From 8a684defedd74faa9ade00b41cc1f34ec1c126e7 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:36:57 +0300 Subject: [PATCH 08/24] Use feature flag to set UPDATER_DETERMINISTIC --- server/Tingle.Dependabot/Constants.cs | 1 + server/Tingle.Dependabot/Workflow/UpdateRunner.cs | 3 ++- server/Tingle.Dependabot/Workflow/WorkflowOptions.cs | 6 ------ server/Tingle.Dependabot/appsettings.json | 3 ++- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/server/Tingle.Dependabot/Constants.cs b/server/Tingle.Dependabot/Constants.cs index 017ebde1..c5fb0346 100644 --- a/server/Tingle.Dependabot/Constants.cs +++ b/server/Tingle.Dependabot/Constants.cs @@ -24,4 +24,5 @@ internal static class FeatureNames { internal const string DebugAllJobs = "DebugAllJobs"; internal const string DebugJobs = "DebugJobs"; + internal const string DeterministicUpdates = "DeterministicUpdates"; // Whether updates should be created in the same order. } diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 40a9c56c..a039f625 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -234,6 +234,7 @@ internal async Task> CreateEnvironmentVariables(Proj // check if debug is enabled for the project via Feature Management var fmc = new TargetingContext { Groups = new[] { $"provider:{project.Type.ToString().ToLower()}", $"project:{project.Id}", $"ecosystem:{job.PackageEcosystem}", }, }; var debugAllJobs = await featureManager.IsEnabledAsync(FeatureNames.DebugAllJobs, fmc); + var deterministic = await featureManager.IsEnabledAsync(FeatureNames.DeterministicUpdates, fmc); // Add compulsory values var values = new Dictionary @@ -248,7 +249,7 @@ internal async Task> CreateEnvironmentVariables(Proj // Setting DEPENDABOT_REPO_CONTENTS_PATH causes some issues, ignore till we can resolve //["DEPENDABOT_REPO_CONTENTS_PATH"] = Path.Join(jobDirectory, "repo"), ["GITHUB_ACTIONS"] = "false", - ["UPDATER_DETERMINISTIC"] = (options.DeterministicUpdates ?? false).ToString().ToLower(), + ["UPDATER_DETERMINISTIC"] = deterministic.ToString().ToLower(), // env for v1 ["DEPENDABOT_PACKAGE_MANAGER"] = job.PackageEcosystem!, diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index f16d6295..72569cfc 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -14,9 +14,6 @@ public class WorkflowOptions /// URL on which to access the API from the jobs. public string? JobsApiUrl { get; set; } - /// Password used for creation of subscription and authenticating incoming notifications. - public string? SubscriptionPassword { get; set; } - /// Resource identifier for the resource group to create jobs in. /// /subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT public string? ResourceGroupId { get; set; } @@ -50,9 +47,6 @@ public class WorkflowOptions /// /mnt/dependabot public string? WorkingDirectory { get; set; } - /// Whether updates should be created in the same order. - public bool? DeterministicUpdates { get; set; } - /// /// Token for accessing GitHub APIs. /// If no value is provided, calls to GitHub are not authenticated. diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index 9ecacde1..26dec2a2 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -65,6 +65,7 @@ "FeatureManagement": { "DebugAllJobs": false, - "DebugJobs": false + "DebugJobs": false, + "DeterministicUpdates": false } } From 254279037b1a053a24d3aef0db383f542170e8b7 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:37:44 +0300 Subject: [PATCH 09/24] Change WorkflowOptions.JobsApiUrl to Uri --- server/Tingle.Dependabot/Workflow/UpdateRunner.cs | 2 +- server/Tingle.Dependabot/Workflow/WorkflowOptions.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index a039f625..e63ad07a 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -243,7 +243,7 @@ internal async Task> CreateEnvironmentVariables(Proj ["DEPENDABOT_JOB_ID"] = job.Id!, ["DEPENDABOT_JOB_TOKEN"] = job.AuthKey!, ["DEPENDABOT_DEBUG"] = debugAllJobs.ToString().ToLower(), - ["DEPENDABOT_API_URL"] = options.JobsApiUrl!, + ["DEPENDABOT_API_URL"] = options.JobsApiUrl!.ToString(), ["DEPENDABOT_JOB_PATH"] = Path.Join(directory, JobDefinitionFileName), ["DEPENDABOT_OUTPUT_PATH"] = Path.Join(directory, "output"), // Setting DEPENDABOT_REPO_CONTENTS_PATH causes some issues, ignore till we can resolve diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index 72569cfc..32006580 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -12,7 +12,7 @@ public class WorkflowOptions public Uri? WebhookEndpoint { get; set; } /// URL on which to access the API from the jobs. - public string? JobsApiUrl { get; set; } + public Uri? JobsApiUrl { get; set; } /// Resource identifier for the resource group to create jobs in. /// /subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT From 5abe66508d3f2bc28e545c68707319610310967e Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:43:01 +0300 Subject: [PATCH 10/24] Update basic authentication to check credentials against database --- .../BasicUserValidationService.cs | 14 ++++++++------ .../Workflow/AzureDevOpsProvider.cs | 2 +- server/Tingle.Dependabot/appsettings.json | 5 ----- server/main.bicep | 1 - server/main.json | 4 ---- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/server/Tingle.Dependabot/BasicUserValidationService.cs b/server/Tingle.Dependabot/BasicUserValidationService.cs index ca50b7ad..81ace82a 100644 --- a/server/Tingle.Dependabot/BasicUserValidationService.cs +++ b/server/Tingle.Dependabot/BasicUserValidationService.cs @@ -1,19 +1,21 @@ using AspNetCore.Authentication.Basic; +using Microsoft.EntityFrameworkCore; +using Tingle.Dependabot.Models; namespace Tingle.Dependabot; internal class BasicUserValidationService : IBasicUserValidationService { - private readonly IConfiguration configuration; + private readonly MainDbContext dbContext; - public BasicUserValidationService(IConfiguration configuration) + public BasicUserValidationService(MainDbContext dbContext) { - this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this.dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); } - public Task IsValidAsync(string username, string password) + public async Task IsValidAsync(string username, string password) { - var expected = configuration.GetValue($"Authentication:Schemes:ServiceHooks:Credentials:{username}"); - return Task.FromResult(string.Equals(expected, password, StringComparison.Ordinal)); + var project = await dbContext.Projects.SingleOrDefaultAsync(p => p.Id == username); + return project is not null && string.Equals(project.Password, password, StringComparison.Ordinal); } } diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs index 0a95dd89..88fc1fe9 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs @@ -198,7 +198,7 @@ private static Dictionary MakeWebHooksConsumerInputs(Project pro ["detailedMessagesToSend"] = "none", ["messagesToSend"] = "none", ["url"] = webhookUrl.ToString(), - ["basicAuthUsername"] = "vsts", + ["basicAuthUsername"] = project.Id!, ["basicAuthPassword"] = project.Password!, }; } diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index 26dec2a2..cdec3200 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -22,11 +22,6 @@ "ValidAudiences": [ "http://localhost:3000" ] - }, - "ServiceHooks": { - "Credentials": { - "vsts": "AAAAAAAAAAA=" - } } } }, diff --git a/server/main.bicep b/server/main.bicep index 7ec9f7e5..468d0dc0 100644 --- a/server/main.bicep +++ b/server/main.bicep @@ -285,7 +285,6 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { { name: 'Authentication__Schemes__Management__Authority', value: '${environment().authentication.loginEndpoint}${subscription().tenantId}/v2.0' } { name: 'Authentication__Schemes__Management__ValidAudiences__0', value: 'https://${name}.${appEnvironment.properties.defaultDomain}' } - { name: 'Authentication__Schemes__ServiceHooks__Credentials__vsts', secretRef: 'notifications-password' } { name: 'EventBus__SelectedTransport', value: 'ServiceBus' } { name: 'EventBus__Transports__azure-service-bus__FullyQualifiedNamespace', value: split(split(serviceBusNamespace.properties.serviceBusEndpoint, '/')[2], ':')[0] } // manipulating https://{your-namespace}.servicebus.windows.net:443/ diff --git a/server/main.json b/server/main.json index 7706cf39..66a71321 100644 --- a/server/main.json +++ b/server/main.json @@ -467,10 +467,6 @@ "name": "Authentication__Schemes__Management__ValidAudiences__0", "value": "[format('https://{0}.{1}', parameters('name'), reference(resourceId('Microsoft.App/managedEnvironments', parameters('name')), '2023-05-01').defaultDomain)]" }, - { - "name": "Authentication__Schemes__ServiceHooks__Credentials__vsts", - "secretRef": "notifications-password" - }, { "name": "EventBus__SelectedTransport", "value": "ServiceBus" From 27161051ccd58a182ab75bb21e746d6d56a716f5 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 13:44:40 +0300 Subject: [PATCH 11/24] Do not expose Project.Token --- server/Tingle.Dependabot/Models/Management/Project.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index 0b57754b..e2e24615 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -32,6 +32,7 @@ public class Project /// Token for accessing the project with permissions for repositories, pull requests, and service hooks. /// [Required] + [JsonIgnore] // expose this once we know how to protect the values public string? Token { get; set; } /// Whether to set auto complete on created pull requests. From 14ee0598e71e4bdd079e1bcb25e98619e1350a07 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 16:51:55 +0300 Subject: [PATCH 12/24] Move ConfigurationFilePaths to AzureDevOpsProvider --- .../Workflow/AzureDevOpsProvider.cs | 13 +++++++++++-- .../Tingle.Dependabot/Workflow/WorkflowOptions.cs | 12 ------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs index 88fc1fe9..99a3ea77 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs @@ -14,6 +14,16 @@ namespace Tingle.Dependabot.Workflow; public class AzureDevOpsProvider { + // Possible/allowed paths for the configuration files in a repository. + private static readonly IReadOnlyList ConfigurationFilePaths = new[] { + // TODO: restore checks in .azuredevops folder once either the code can check that folder or we are passing ignore conditions via update_jobs API + //".azuredevops/dependabot.yml", + //".azuredevops/dependabot.yaml", + + ".github/dependabot.yml", + ".github/dependabot.yaml", + }; + private static readonly (string, string)[] SubscriptionEventTypes = { ("git.push", "1.0"), @@ -146,9 +156,8 @@ public async Task GetRepositoryAsync(Project project, string repo var connection = CreateVssConnection(url, project.Token!); // Try all known paths - var paths = options.ConfigurationFilePaths; var client = await connection.GetClientAsync(cancellationToken); - foreach (var path in paths) + foreach (var path in ConfigurationFilePaths) { try { diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index 32006580..5da25971 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -59,16 +59,4 @@ public class WorkflowOptions /// Location/region where to create new update jobs. public string? Location { get; set; } // using Azure.Core.Location does not work when binding from IConfiguration - - /// - /// Possible/allowed paths for the configuration files in a repository. - /// - public IReadOnlyList ConfigurationFilePaths { get; set; } = new[] { - // TODO: restore checks in .azuredevops folder once either the code can check that folder or we are passing ignore conditions via update_jobs API - //".azuredevops/dependabot.yml", - //".azuredevops/dependabot.yaml", - - ".github/dependabot.yml", - ".github/dependabot.yaml", - }; } From 2e2be775b30ca350c96287bfdeb3a71c8bead34d Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 17:00:01 +0300 Subject: [PATCH 13/24] Nest settings for auto complete and auto approve --- .../20230824083425_InitialCreate.Designer.cs | 60 +++++++++++++++---- .../20230824083425_InitialCreate.cs | 8 +-- .../Migrations/MainDbContextModelSnapshot.cs | 60 +++++++++++++++---- .../Tingle.Dependabot/Models/MainDbContext.cs | 46 +++++++------- .../Models/Management/Project.cs | 34 +++++++---- .../Workflow/UpdateRunner.cs | 8 +-- 6 files changed, 151 insertions(+), 65 deletions(-) diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index e872b662..e24d4f70 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -50,19 +50,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasMaxLength(50) .HasColumnType("nvarchar(50)"); - b.Property("AutoApprove") - .HasColumnType("bit"); - - b.Property("AutoComplete") - .HasColumnType("bit"); - - b.Property("AutoCompleteIgnoreConfigs") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("AutoCompleteMergeStrategy") - .HasColumnType("int"); - b.Property("Created") .HasColumnType("datetimeoffset"); @@ -258,6 +245,53 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoApprove", "AutoApprove", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoComplete", "AutoComplete", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.Property("IgnoreConfigs") + .HasColumnType("nvarchar(max)"); + + b1.Property("MergeStrategy") + .HasColumnType("int"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.Navigation("AutoApprove") + .IsRequired(); + + b.Navigation("AutoComplete") + .IsRequired(); + }); + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { b.HasOne("Tingle.Dependabot.Models.Management.Project", null) diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index dcfb7329..e6264d0c 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -36,10 +36,10 @@ protected override void Up(MigrationBuilder migrationBuilder) ProviderId = table.Column(type: "nvarchar(450)", nullable: false), Url = table.Column(type: "nvarchar(max)", nullable: false), Token = table.Column(type: "nvarchar(max)", nullable: false), - AutoComplete = table.Column(type: "bit", nullable: false), - AutoCompleteIgnoreConfigs = table.Column(type: "nvarchar(max)", nullable: false), - AutoCompleteMergeStrategy = table.Column(type: "int", nullable: true), - AutoApprove = table.Column(type: "bit", nullable: false), + AutoComplete_Enabled = table.Column(type: "bit", nullable: false), + AutoComplete_IgnoreConfigs = table.Column(type: "nvarchar(max)", nullable: true), + AutoComplete_MergeStrategy = table.Column(type: "int", nullable: true), + AutoApprove_Enabled = table.Column(type: "bit", nullable: false), Password = table.Column(type: "nvarchar(450)", nullable: false), Secrets = table.Column(type: "nvarchar(max)", nullable: false), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index 7c13dc90..2d205b6c 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -47,19 +47,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(50) .HasColumnType("nvarchar(50)"); - b.Property("AutoApprove") - .HasColumnType("bit"); - - b.Property("AutoComplete") - .HasColumnType("bit"); - - b.Property("AutoCompleteIgnoreConfigs") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("AutoCompleteMergeStrategy") - .HasColumnType("int"); - b.Property("Created") .HasColumnType("datetimeoffset"); @@ -255,6 +242,53 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Project", b => + { + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoApprove", "AutoApprove", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.OwnsOne("Tingle.Dependabot.Models.Management.ProjectAutoComplete", "AutoComplete", b1 => + { + b1.Property("ProjectId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Enabled") + .HasColumnType("bit"); + + b1.Property("IgnoreConfigs") + .HasColumnType("nvarchar(max)"); + + b1.Property("MergeStrategy") + .HasColumnType("int"); + + b1.HasKey("ProjectId"); + + b1.ToTable("Projects"); + + b1.WithOwner() + .HasForeignKey("ProjectId"); + }); + + b.Navigation("AutoApprove") + .IsRequired(); + + b.Navigation("AutoComplete") + .IsRequired(); + }); + modelBuilder.Entity("Tingle.Dependabot.Models.Management.Repository", b => { b.HasOne("Tingle.Dependabot.Models.Management.Project", null) diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index 7884c9cd..b246c9a7 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -18,37 +18,41 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); - modelBuilder.Entity(b => + modelBuilder.Entity(builder => { - b.Property(p => p.AutoCompleteIgnoreConfigs).HasJsonConversion(); - b.Property(p => p.Secrets).HasJsonConversion(); - - b.HasIndex(p => p.Created).IsDescending(); // faster filtering - b.HasIndex(p => p.ProviderId).IsUnique(); - b.HasIndex(p => p.Password).IsUnique(); // password should be unique per project + builder.OwnsOne(p => p.AutoApprove); + builder.OwnsOne(p => p.AutoComplete, ownedBuilder => + { + ownedBuilder.Property(p => p.IgnoreConfigs).HasJsonConversion(); + }); + builder.Property(p => p.Secrets).HasJsonConversion(); + + builder.HasIndex(p => p.Created).IsDescending(); // faster filtering + builder.HasIndex(p => p.ProviderId).IsUnique(); + builder.HasIndex(p => p.Password).IsUnique(); // password should be unique per project }); - modelBuilder.Entity(b => + modelBuilder.Entity(builder => { - b.Property(r => r.Updates).HasJsonConversion(); - b.Property(r => r.Registries).HasJsonConversion(); + builder.Property(r => r.Updates).HasJsonConversion(); + builder.Property(r => r.Registries).HasJsonConversion(); - b.HasIndex(r => r.Created).IsDescending(); // faster filtering - b.HasIndex(r => r.ProviderId).IsUnique(); + builder.HasIndex(r => r.Created).IsDescending(); // faster filtering + builder.HasIndex(r => r.ProviderId).IsUnique(); }); - modelBuilder.Entity(b => + modelBuilder.Entity(builder => { - b.Property(j => j.PackageEcosystem).IsRequired(); - b.Property(j => j.Error).HasJsonConversion(); + builder.Property(j => j.PackageEcosystem).IsRequired(); + builder.Property(j => j.Error).HasJsonConversion(); - b.HasIndex(j => j.Created).IsDescending(); // faster filtering - b.HasIndex(j => j.RepositoryId); - b.HasIndex(j => new { j.PackageEcosystem, j.Directory, }); // faster filtering - b.HasIndex(j => new { j.PackageEcosystem, j.Directory, j.EventBusId, }).IsUnique(); - b.HasIndex(j => j.AuthKey).IsUnique(); + builder.HasIndex(j => j.Created).IsDescending(); // faster filtering + builder.HasIndex(j => j.RepositoryId); + builder.HasIndex(j => new { j.PackageEcosystem, j.Directory, }); // faster filtering + builder.HasIndex(j => new { j.PackageEcosystem, j.Directory, j.EventBusId, }).IsUnique(); + builder.HasIndex(j => j.AuthKey).IsUnique(); - b.OwnsOne(j => j.Resources); + builder.OwnsOne(j => j.Resources); }); } } diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index e2e24615..72fc2430 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -35,17 +35,13 @@ public class Project [JsonIgnore] // expose this once we know how to protect the values public string? Token { get; set; } - /// Whether to set auto complete on created pull requests. - public bool AutoComplete { get; set; } - - /// Identifiers of configs to be ignored in auto complete. - public List AutoCompleteIgnoreConfigs { get; set; } = new(); - - /// Merge strategy to use when setting auto complete on created pull requests. - public MergeStrategy? AutoCompleteMergeStrategy { get; set; } + /// Auto complete settings. + [Required] + public ProjectAutoComplete AutoComplete { get; set; } = new(); - /// Whether to automatically approve created pull requests. - public bool AutoApprove { get; set; } + /// Auto approve settings. + [Required] + public ProjectAutoApprove AutoApprove { get; set; } = new(); /// Password for Webhooks, ServiceHooks, and Notifications from the provider. [Required] @@ -65,6 +61,24 @@ public class Project public byte[]? Etag { get; set; } } +public class ProjectAutoComplete +{ + /// Whether to set auto complete on created pull requests. + public bool Enabled { get; set; } + + /// Identifiers of configs to be ignored in auto complete. + public List? IgnoreConfigs { get; set; } + + /// Merge strategy to use when setting auto complete on created pull requests. + public MergeStrategy? MergeStrategy { get; set; } +} + +public class ProjectAutoApprove +{ + /// Whether to automatically approve created pull requests. + public bool Enabled { get; set; } +} + public enum ProjectType { Azure, diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index e63ad07a..774791d1 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -278,10 +278,10 @@ internal async Task> CreateEnvironmentVariables(Proj .AddIfNotDefault("AZURE_PROJECT", url.ProjectName) .AddIfNotDefault("AZURE_REPOSITORY", Uri.EscapeDataString(repository.Name!)) .AddIfNotDefault("AZURE_ACCESS_TOKEN", project.Token) - .AddIfNotDefault("AZURE_SET_AUTO_COMPLETE", project.AutoComplete.ToString().ToLowerInvariant()) - .AddIfNotDefault("AZURE_AUTO_COMPLETE_IGNORE_CONFIG_IDS", ToJson(project.AutoCompleteIgnoreConfigs ?? new())) - .AddIfNotDefault("AZURE_MERGE_STRATEGY", project.AutoCompleteMergeStrategy?.ToString()) - .AddIfNotDefault("AZURE_AUTO_APPROVE_PR", project.AutoApprove.ToString().ToLowerInvariant()); + .AddIfNotDefault("AZURE_SET_AUTO_COMPLETE", project.AutoComplete.Enabled.ToString().ToLowerInvariant()) + .AddIfNotDefault("AZURE_AUTO_COMPLETE_IGNORE_CONFIG_IDS", ToJson(project.AutoComplete.IgnoreConfigs ?? new())) + .AddIfNotDefault("AZURE_MERGE_STRATEGY", project.AutoComplete.MergeStrategy?.ToString()) + .AddIfNotDefault("AZURE_AUTO_APPROVE_PR", project.AutoApprove.Enabled.ToString().ToLowerInvariant()); return values; } From 7c5ed12ef23614852d1912743a1b25f49fef3b0b Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 17:07:46 +0300 Subject: [PATCH 14/24] Expand UpdateJob.Error so that we can search the error types --- .../20230824083425_InitialCreate.Designer.cs | 27 ++++++++++++++++--- .../20230824083425_InitialCreate.cs | 8 +++++- .../Migrations/MainDbContextModelSnapshot.cs | 27 ++++++++++++++++--- .../DependabotRecordUpdateJobErrorModel.cs | 4 ++- .../Tingle.Dependabot/Models/MainDbContext.cs | 8 ++++-- .../Models/Management/UpdateJob.cs | 1 + 6 files changed, 65 insertions(+), 10 deletions(-) diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index e24d4f70..a04ab817 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -191,9 +191,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("End") .HasColumnType("datetimeoffset"); - b.Property("Error") - .HasColumnType("nvarchar(max)"); - b.Property("Etag") .IsConcurrencyToken() .ValueGeneratedOnAddOrUpdate() @@ -303,6 +300,28 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => { + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobError", "Error", b1 => + { + b1.Property("UpdateJobId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Detail") + .HasColumnType("nvarchar(max)"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b1.HasKey("UpdateJobId"); + + b1.HasIndex("Type"); + + b1.ToTable("UpdateJobs"); + + b1.WithOwner() + .HasForeignKey("UpdateJobId"); + }); + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobResources", "Resources", b1 => { b1.Property("UpdateJobId") @@ -322,6 +341,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasForeignKey("UpdateJobId"); }); + b.Navigation("Error"); + b.Navigation("Resources") .IsRequired(); }); diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index e6264d0c..5bfd84ba 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -70,7 +70,8 @@ protected override void Up(MigrationBuilder migrationBuilder) End = table.Column(type: "datetimeoffset", nullable: true), Duration = table.Column(type: "bigint", nullable: true), Log = table.Column(type: "nvarchar(max)", nullable: true), - Error = table.Column(type: "nvarchar(max)", nullable: true), + Error_Type = table.Column(type: "nvarchar(450)", nullable: true), + Error_Detail = table.Column(type: "nvarchar(max)", nullable: true), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => @@ -154,6 +155,11 @@ protected override void Up(MigrationBuilder migrationBuilder) column: "Created", descending: new bool[0]); + migrationBuilder.CreateIndex( + name: "IX_UpdateJobs_Error_Type", + table: "UpdateJobs", + column: "Error_Type"); + migrationBuilder.CreateIndex( name: "IX_UpdateJobs_PackageEcosystem_Directory", table: "UpdateJobs", diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index 2d205b6c..f3f643e6 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -188,9 +188,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("End") .HasColumnType("datetimeoffset"); - b.Property("Error") - .HasColumnType("nvarchar(max)"); - b.Property("Etag") .IsConcurrencyToken() .ValueGeneratedOnAddOrUpdate() @@ -300,6 +297,28 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("Tingle.Dependabot.Models.Management.UpdateJob", b => { + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobError", "Error", b1 => + { + b1.Property("UpdateJobId") + .HasColumnType("nvarchar(50)"); + + b1.Property("Detail") + .HasColumnType("nvarchar(max)"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b1.HasKey("UpdateJobId"); + + b1.HasIndex("Type"); + + b1.ToTable("UpdateJobs"); + + b1.WithOwner() + .HasForeignKey("UpdateJobId"); + }); + b.OwnsOne("Tingle.Dependabot.Models.Management.UpdateJobResources", "Resources", b1 => { b1.Property("UpdateJobId") @@ -319,6 +338,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasForeignKey("UpdateJobId"); }); + b.Navigation("Error"); + b.Navigation("Resources") .IsRequired(); }); diff --git a/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs b/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs index d2b72cb6..fd261903 100644 --- a/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs +++ b/server/Tingle.Dependabot/Models/Dependabot/DependabotRecordUpdateJobErrorModel.cs @@ -1,10 +1,12 @@ -using System.Text.Json.Nodes; +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace Tingle.Dependabot.Models.Dependabot; public class DependabotRecordUpdateJobErrorModel { + [Required] [JsonPropertyName("error-type")] public string? ErrorType { get; set; } diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index b246c9a7..226ce7e7 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -23,7 +23,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) builder.OwnsOne(p => p.AutoApprove); builder.OwnsOne(p => p.AutoComplete, ownedBuilder => { - ownedBuilder.Property(p => p.IgnoreConfigs).HasJsonConversion(); + ownedBuilder.Property(ac => ac.IgnoreConfigs).HasJsonConversion(); }); builder.Property(p => p.Secrets).HasJsonConversion(); @@ -44,7 +44,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity(builder => { builder.Property(j => j.PackageEcosystem).IsRequired(); - builder.Property(j => j.Error).HasJsonConversion(); + builder.OwnsOne(j => j.Error, ownedBuilder => + { + ownedBuilder.Property(e => e.Detail).HasJsonConversion(); + ownedBuilder.HasIndex(e => e.Type); // faster filtering + }); builder.HasIndex(j => j.Created).IsDescending(); // faster filtering builder.HasIndex(j => j.RepositoryId); diff --git a/server/Tingle.Dependabot/Models/Management/UpdateJob.cs b/server/Tingle.Dependabot/Models/Management/UpdateJob.cs index ad99fdf3..554576de 100644 --- a/server/Tingle.Dependabot/Models/Management/UpdateJob.cs +++ b/server/Tingle.Dependabot/Models/Management/UpdateJob.cs @@ -81,6 +81,7 @@ public class UpdateJob public class UpdateJobError { + [Required] public string? Type { get; set; } public JsonNode? Detail { get; set; } } From 45c62790fd5981dd2d48a135b6b862fec9e424c5 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 17:21:28 +0300 Subject: [PATCH 15/24] Track project identifier in Application Insights --- .../DependabotTelemetryInitializer.cs | 27 +++++++++ .../InsightsFilteringProcessor.cs | 58 +++++++++++++++++++ .../InsightsShutdownFlushService.cs | 30 ++++++++++ .../Extensions/CollectionExtensions.cs | 50 ++++++++++++++++ .../IServiceCollectionExtensions.cs | 31 ++++++++++ .../Extensions/SystemExtensions.cs | 27 --------- server/Tingle.Dependabot/Program.cs | 3 +- .../Tingle.Dependabot.csproj | 1 + 8 files changed, 199 insertions(+), 28 deletions(-) create mode 100644 server/Tingle.Dependabot/ApplicationInsights/DependabotTelemetryInitializer.cs create mode 100644 server/Tingle.Dependabot/ApplicationInsights/InsightsFilteringProcessor.cs create mode 100644 server/Tingle.Dependabot/ApplicationInsights/InsightsShutdownFlushService.cs create mode 100644 server/Tingle.Dependabot/Extensions/CollectionExtensions.cs delete mode 100644 server/Tingle.Dependabot/Extensions/SystemExtensions.cs diff --git a/server/Tingle.Dependabot/ApplicationInsights/DependabotTelemetryInitializer.cs b/server/Tingle.Dependabot/ApplicationInsights/DependabotTelemetryInitializer.cs new file mode 100644 index 00000000..1512131d --- /dev/null +++ b/server/Tingle.Dependabot/ApplicationInsights/DependabotTelemetryInitializer.cs @@ -0,0 +1,27 @@ +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; + +namespace Tingle.Dependabot.ApplicationInsights; + +internal class DependabotTelemetryInitializer : ITelemetryInitializer +{ + private const string KeyProjectId = "ProjectId"; + + private readonly IHttpContextAccessor httpContextAccessor; + + public DependabotTelemetryInitializer(IHttpContextAccessor httpContextAccessor) + { + this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); + } + + public void Initialize(ITelemetry telemetry) + { + var context = httpContextAccessor.HttpContext; + if (context is null || telemetry is not RequestTelemetry rt) return; // ensure we have a context and the telemetry is for a request + + // add properties + var props = rt.Properties; + props.TryAddIfNotDefault(KeyProjectId, context.GetProjectId()); + } +} diff --git a/server/Tingle.Dependabot/ApplicationInsights/InsightsFilteringProcessor.cs b/server/Tingle.Dependabot/ApplicationInsights/InsightsFilteringProcessor.cs new file mode 100644 index 00000000..58eb4630 --- /dev/null +++ b/server/Tingle.Dependabot/ApplicationInsights/InsightsFilteringProcessor.cs @@ -0,0 +1,58 @@ +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; + +namespace Tingle.Dependabot.ApplicationInsights; + +/// +/// Implementation of that filters out unneeded telemetry. +/// +internal class InsightsFilteringProcessor : ITelemetryProcessor +{ + private static readonly string[] excludedRequestNames = + { + "ServiceBusReceiver.Receive", + "ServiceBusProcessor.ProcessMessage", + }; + + private readonly ITelemetryProcessor next; + + public InsightsFilteringProcessor(ITelemetryProcessor next) + { + this.next = next; + } + + /// + public void Process(ITelemetry item) + { + // Skip unneeded RequestTelemetry + if (item is RequestTelemetry rt) + { + // Skip known request names + if (rt.Name is not null && excludedRequestNames.Contains(rt.Name, StringComparer.OrdinalIgnoreCase)) + { + return; // terminate the processor pipeline + } + + // Skip requests for /health and /liveness because they are better diagnosed via logs + var path = rt.Url?.AbsolutePath; + if (string.Equals(path, "/health", StringComparison.OrdinalIgnoreCase) + || string.Equals(path, "/liveness", StringComparison.OrdinalIgnoreCase)) + { + return; // terminate the processor pipeline + } + } + + // Skip requests sent to the orchestrator to find the details of a pod/host + // Sometimes they fail, like when the service is starting up + if (item is DependencyTelemetry dt + && string.Equals("http", dt.Type, StringComparison.OrdinalIgnoreCase) + && string.Equals("10.0.0.1", dt.Target, StringComparison.OrdinalIgnoreCase)) + { + return; // terminate the processor pipeline + } + + // process all the others + next.Process(item); + } +} diff --git a/server/Tingle.Dependabot/ApplicationInsights/InsightsShutdownFlushService.cs b/server/Tingle.Dependabot/ApplicationInsights/InsightsShutdownFlushService.cs new file mode 100644 index 00000000..6b1de3be --- /dev/null +++ b/server/Tingle.Dependabot/ApplicationInsights/InsightsShutdownFlushService.cs @@ -0,0 +1,30 @@ +using Microsoft.ApplicationInsights; + +namespace Tingle.Dependabot.ApplicationInsights; + +// from https://medium.com/@asimmon/prevent-net-application-insights-telemetry-loss-d82a06c3673f +internal class InsightsShutdownFlushService : IHostedService +{ + private readonly TelemetryClient telemetryClient; + + public InsightsShutdownFlushService(TelemetryClient telemetryClient) + { + this.telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + } + + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public async Task StopAsync(CancellationToken cancellationToken) + { + // Flush the remaining telemetry data when application shutdown is requested. + // Using "CancellationToken.None" ensures that the application doesn't stop until the telemetry data is flushed. + // + // If you want to use the "cancellationToken" argument, make sure to configure "HostOptions.ShutdownTimeout" with a sufficiently large duration, + // and silence the eventual "OperationCanceledException" exception. Otherwise, you will still be at risk of loosing telemetry data. + var successfullyFlushed = await telemetryClient.FlushAsync(CancellationToken.None); + if (!successfullyFlushed) + { + // Here you can handle th case where transfer of telemetry data to the server has failed with HTTP status that cannot be retried. + } + } +} diff --git a/server/Tingle.Dependabot/Extensions/CollectionExtensions.cs b/server/Tingle.Dependabot/Extensions/CollectionExtensions.cs new file mode 100644 index 00000000..46d91f93 --- /dev/null +++ b/server/Tingle.Dependabot/Extensions/CollectionExtensions.cs @@ -0,0 +1,50 @@ +namespace System.Collections.Generic; + +internal static class CollectionExtensions +{ + /// + /// Adds an element with the provided key and value, + /// provided the value is not equal to the type's default value (or empty for strings). + /// + /// The type of keys in the dictionary. + /// The type of values in the dictionary. + /// The dictionary to use + /// The object to use as the key of the element to add. + /// The object to use as the value of the element to add. + /// key is null. + /// The dictionary is read-only. + /// + public static IDictionary AddIfNotDefault(this IDictionary dictionary, TKey key, TValue? value) + where TKey : notnull + { + if (value is not null || value is string s && !string.IsNullOrWhiteSpace(s)) + { + dictionary[key] = value; + } + + return dictionary; + } + + + /// + /// Tries to add an element with the provided key and value to the , + /// provided the value is not equal to the type's default value (or empty for strings). + /// + /// The type of keys in the dictionary. + /// The type of values in the dictionary. + /// The dictionary to use + /// The object to use as the key of the element to add. + /// The object to use as the value of the element to add. + /// key is null. + /// + public static bool TryAddIfNotDefault(this IDictionary dictionary, TKey key, TValue? value) + where TKey : notnull + { + if (value is not null || (value is string s && !string.IsNullOrWhiteSpace(s))) + { + return dictionary.TryAdd(key, value); + } + + return false; + } +} diff --git a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs index daedc413..1232322f 100644 --- a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs +++ b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs @@ -1,6 +1,9 @@ using Medallion.Threading; using Medallion.Threading.FileSystem; +using Microsoft.ApplicationInsights.DependencyCollector; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.FeatureManagement; +using Tingle.Dependabot.ApplicationInsights; using Tingle.Dependabot.FeatureManagement; using Tingle.Dependabot.Workflow; @@ -9,6 +12,34 @@ namespace Microsoft.Extensions.DependencyInjection; /// Extensions on . public static class IServiceCollectionExtensions { + /// Add standard Application Insights services. + /// The instance to add to. + /// The root configuration instance from which to pull settings. + public static IServiceCollection AddStandardApplicationInsights(this IServiceCollection services, IConfiguration configuration) + { + // Add the core services + services.AddApplicationInsightsTelemetry(configuration); + + // Add background service to flush telemetry on shutdown + services.AddHostedService(); + + // Add processors + services.AddApplicationInsightsTelemetryProcessor(); + + // Enrich the telemetry with various sources of information + services.AddHttpContextAccessor(); // Required to resolve the request from the HttpContext + // according to docs link below, this registration should be singleton + // https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-core#adding-telemetryinitializers + services.AddSingleton(); + services.AddApplicationInsightsTelemetryExtras(); // Add other extras + + services.AddActivitySourceDependencyCollector(new[] { + "Tingle.EventBus", + }); + + return services; + } + public static IServiceCollection AddDistributedLockProvider(this IServiceCollection services, IHostEnvironment environment, IConfiguration configuration) { var configKey = ConfigurationPath.Combine("DistributedLocking", "FilePath"); diff --git a/server/Tingle.Dependabot/Extensions/SystemExtensions.cs b/server/Tingle.Dependabot/Extensions/SystemExtensions.cs deleted file mode 100644 index b77eada8..00000000 --- a/server/Tingle.Dependabot/Extensions/SystemExtensions.cs +++ /dev/null @@ -1,27 +0,0 @@ -namespace System; - -internal static class SystemExtensions -{ - /// - /// Adds an element with the provided key and value, - /// provided the value is not equal to the type's default value (or empty for strings). - /// - /// The type of keys in the dictionary. - /// The type of values in the dictionary. - /// The dictionary to use - /// The object to use as the key of the element to add. - /// The object to use as the value of the element to add. - /// key is null. - /// The dictionary is read-only. - /// - public static IDictionary AddIfNotDefault(this IDictionary dictionary, TKey key, TValue? value) - where TKey : notnull - { - if (value is not null || value is string s && !string.IsNullOrWhiteSpace(s)) - { - dictionary[key] = value; - } - - return dictionary; - } -} diff --git a/server/Tingle.Dependabot/Program.cs b/server/Tingle.Dependabot/Program.cs index 3f7d8eee..c5a25dd6 100644 --- a/server/Tingle.Dependabot/Program.cs +++ b/server/Tingle.Dependabot/Program.cs @@ -32,7 +32,8 @@ }); }); -builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); +// Add Application Insights +builder.Services.AddStandardApplicationInsights(builder.Configuration); // Add DbContext builder.Services.AddDbContext(options => diff --git a/server/Tingle.Dependabot/Tingle.Dependabot.csproj b/server/Tingle.Dependabot/Tingle.Dependabot.csproj index b23b1b3e..e73f33a9 100644 --- a/server/Tingle.Dependabot/Tingle.Dependabot.csproj +++ b/server/Tingle.Dependabot/Tingle.Dependabot.csproj @@ -39,6 +39,7 @@ + From 7a68813098b9103650dfdf2518f5864266707187 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 18:43:12 +0300 Subject: [PATCH 16/24] Added support to setup projects on startup --- docs/server.md | 1 + server/Tingle.Dependabot/AppSetup.cs | 88 ++++++++++++++++--- .../IServiceCollectionExtensions.cs | 1 - .../Models/Management/Project.cs | 1 + .../Properties/launchSettings.json | 3 +- .../Workflow/AzureDevOpsProjectUrl.cs | 2 + .../Workflow/AzureDevOpsProvider.cs | 61 ++++++++++++- server/main.bicep | 6 ++ server/main.json | 15 ++++ 9 files changed, 163 insertions(+), 15 deletions(-) diff --git a/docs/server.md b/docs/server.md index 72893b04..4c13c318 100644 --- a/docs/server.md +++ b/docs/server.md @@ -55,6 +55,7 @@ The deployment exposes the following parameters that can be tuned to suit the se |name|The name of all resources.|No|`dependabot`| |synchronizeOnStartup|Whether to synchronize repositories on startup. This option is useful for initial deployments since the server synchronizes every 6 hours. Leaving it on has no harm, it actually helps you find out if the token works based on the logs.|No|false| |createOrUpdateWebhooksOnStartup|Whether to create or update Azure DevOps subscriptions on startup. This is required if you want configuration files to be picked up automatically and other event driven functionality.
When this is set to `true`, ensure the value provided for `projectToken` has permissions for service hooks and the owner is a Project Administrator. Leaving this on has no harm because the server will only create new subscription if there are no existing ones based on the URL.|No|false| +|projectSetups|A JSON array string representing the projects to be setup on startup. This is useful when running your own setup but may be removed in the future. Example: `[{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]`| |githubToken|Access token for authenticating requests to GitHub. Required for vulnerability checks and to avoid rate limiting on free requests|No|<empty>| |imageTag|The image tag to use when pulling the docker containers. A tag also defines the version. You should avoid using `latest`. Example: `1.1.0`|No|<version-downloaded>| |minReplicas|The minimum number of replicas to required for the deployment. Given that scheduling runs in process, this value cannot be less than `1`. This may change in the future.|No|1| diff --git a/server/Tingle.Dependabot/AppSetup.cs b/server/Tingle.Dependabot/AppSetup.cs index 6109f17e..323ba760 100644 --- a/server/Tingle.Dependabot/AppSetup.cs +++ b/server/Tingle.Dependabot/AppSetup.cs @@ -1,5 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; +using System.Security.Cryptography; +using System.Text.Json; using Tingle.Dependabot.Models; using Tingle.Dependabot.Workflow; @@ -7,6 +9,19 @@ namespace Tingle.Dependabot; internal static class AppSetup { + private class ProjectSetupInfo + { + public required Uri Url { get; set; } + public required string Token { get; set; } + public bool AutoComplete { get; set; } + public List? AutoCompleteIgnoreConfigs { get; set; } + public MergeStrategy? AutoCompleteMergeStrategy { get; set; } + public bool AutoApprove { get; set; } + public Dictionary Secrets { get; set; } = new(StringComparer.OrdinalIgnoreCase); + } + + private static readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + public static async Task SetupAsync(WebApplication app, CancellationToken cancellationToken = default) { using var scope = app.Services.CreateScope(); @@ -22,17 +37,68 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel } } + // add project if there are projects to be added + var adoProvider = provider.GetRequiredService(); var context = provider.GetRequiredService(); var projects = await context.Projects.ToListAsync(cancellationToken); + var setupsJson = app.Configuration.GetValue("PROJECT_SETUPS"); + if (!string.IsNullOrWhiteSpace(setupsJson)) + { + var setups = JsonSerializer.Deserialize>(setupsJson, serializerOptions)!; + foreach (var setup in setups) + { + var url = (AzureDevOpsProjectUrl)setup.Url; + var project = projects.SingleOrDefault(p => new Uri(p.Url!) == setup.Url); + if (project is null) + { + project = new Models.Management.Project + { + Id = Guid.NewGuid().ToString("n"), + Created = DateTimeOffset.UtcNow, + Password = GeneratePassword(32), + Url = setup.Url.ToString(), + Type = Models.Management.ProjectType.Azure, + }; + await context.Projects.AddAsync(project, cancellationToken); + } + + project.Token = setup.Token; + project.AutoComplete.Enabled = setup.AutoComplete; + project.AutoComplete.IgnoreConfigs = setup.AutoCompleteIgnoreConfigs; + project.AutoComplete.MergeStrategy = setup.AutoCompleteMergeStrategy; + project.AutoApprove.Enabled = setup.AutoApprove; + project.Secrets = setup.Secrets; + var tp = await adoProvider.GetProjectAsync(project, cancellationToken); + project.Name = tp.Name; + project.ProviderId = tp.Id.ToString(); + if (context.ChangeTracker.HasChanges()) + { + project.Updated = DateTimeOffset.UtcNow; + } + } + + // update databases + var updated = await context.SaveChangesAsync(cancellationToken); + + // update projects if we updated the db + projects = updated > 0 ? await context.Projects.ToListAsync(cancellationToken) : projects; + } var options = provider.GetRequiredService>().Value; - if (options.SynchronizeOnStartup) + var synchronizer = provider.GetRequiredService(); + foreach (var project in projects) { - var synchronizer = provider.GetRequiredService(); - foreach (var project in projects) + // synchronize project + if (options.SynchronizeOnStartup) { await synchronizer.SynchronizeAsync(project, false, cancellationToken); /* database sync should not trigger, just in case it's too many */ } + + // create or update webhooks/subscriptions + if (options.CreateOrUpdateWebhooksOnStartup) + { + await adoProvider.CreateOrUpdateSubscriptionsAsync(project, cancellationToken); + } } // skip loading schedules if told to @@ -45,15 +111,13 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel await scheduler.CreateOrUpdateAsync(repository, cancellationToken); } } + } - // create or update webhooks/subscriptions if asked to - if (options.CreateOrUpdateWebhooksOnStartup) - { - var adoProvider = provider.GetRequiredService(); - foreach (var project in projects) - { - await adoProvider.CreateOrUpdateSubscriptionsAsync(project, cancellationToken); - } - } + private static string GeneratePassword(int length = 32) + { + var data = new byte[length]; + using var rng = RandomNumberGenerator.Create(); + rng.GetBytes(data); + return Convert.ToBase64String(data); } } diff --git a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs index 1232322f..c98f0270 100644 --- a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs +++ b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs @@ -1,6 +1,5 @@ using Medallion.Threading; using Medallion.Threading.FileSystem; -using Microsoft.ApplicationInsights.DependencyCollector; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.FeatureManagement; using Tingle.Dependabot.ApplicationInsights; diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index 72fc2430..3deaff79 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -15,6 +15,7 @@ public class Project public ProjectType Type { get; set; } /// Name of the project as per provider. + [Required] public string? Name { get; set; } /// Identifier of the repository as per provider. diff --git a/server/Tingle.Dependabot/Properties/launchSettings.json b/server/Tingle.Dependabot/Properties/launchSettings.json index 55a49472..40bb847c 100644 --- a/server/Tingle.Dependabot/Properties/launchSettings.json +++ b/server/Tingle.Dependabot/Properties/launchSettings.json @@ -9,7 +9,8 @@ "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", "EFCORE_PERFORM_MIGRATIONS": "true", - "SKIP_LOAD_SCHEDULES": "true" + "SKIP_LOAD_SCHEDULES": "true", + "PROJECT_SETUPS_": "[{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]" } }, "Docker": { diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs index 73a7a69c..f765573d 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs @@ -14,6 +14,7 @@ public AzureDevOpsProjectUrl(string value) : this(new Uri(value)) { } public AzureDevOpsProjectUrl(Uri uri) { this.uri = uri ?? throw new ArgumentNullException(nameof(uri)); + Scheme = uri.Scheme; var host = Hostname = uri.Host; Port = uri switch { @@ -58,6 +59,7 @@ public static AzureDevOpsProjectUrl Create(string hostname, string organizationN return new(builder.Uri); } + public string Scheme { get; } public string Hostname { get; } public int? Port { get; } public string OrganizationName { get; } diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs index 99a3ea77..544f633c 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs @@ -6,13 +6,15 @@ using Microsoft.VisualStudio.Services.FormInput; using Microsoft.VisualStudio.Services.ServiceHooks.WebApi; using Microsoft.VisualStudio.Services.WebApi; +using System.Net.Http.Headers; using System.Security.Cryptography; using System.Text; +using System.Text.Json.Serialization; using Tingle.Dependabot.Models.Management; namespace Tingle.Dependabot.Workflow; -public class AzureDevOpsProvider +public class AzureDevOpsProvider // TODO: replace the Microsoft.(TeamFoundation|VisualStudio) libraries with direct usage of HttpClient { // Possible/allowed paths for the configuration files in a repository. private static readonly IReadOnlyList ConfigurationFilePaths = new[] { @@ -33,6 +35,7 @@ private static readonly (string, string)[] SubscriptionEventTypes = }; private readonly IMemoryCache cache; + private readonly HttpClient httpClient = new(); // TODO: consider injecting this for logging and tracing purposes private readonly WorkflowOptions options; public AzureDevOpsProvider(IMemoryCache cache, IOptions optionsAccessor) @@ -126,6 +129,28 @@ public async Task> CreateOrUpdateSubscriptionsAsync(Project project return ids; } + public async Task GetProjectAsync(Project project, CancellationToken cancellationToken) + { + //// get a connection to Azure DevOps + //var url = (AzureDevOpsProjectUrl)project.Url!; + //var connection = CreateVssConnection(url, project.Token!); + + //// get the project + //var client = await connection.GetClientAsync(cancellationToken); + //return await client.GetProject(id: url.ProjectIdOrName); + + var url = (AzureDevOpsProjectUrl)project.Url!; + var uri = new UriBuilder + { + Scheme = url.Scheme, + Host = url.Hostname, + Port = url.Port ?? -1, + Path = $"{url.OrganizationName}/_apis/projects/{url.ProjectIdOrName}", + }.Uri; + var request = new HttpRequestMessage(HttpMethod.Get, uri); + return (await SendAsync(project.Token!, request, cancellationToken))!; + } + public async Task> GetRepositoriesAsync(Project project, CancellationToken cancellationToken) { // get a connection to Azure DevOps @@ -234,4 +259,38 @@ static string hash(string v) return cache.Set(cacheKey, cached, TimeSpan.FromHours(1)); } + + private async Task SendAsync(string token, HttpRequestMessage request, CancellationToken cancellationToken) + { + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($":{token}"))); + + var response = await httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); + } +} + +public class AzdoProject +{ + [JsonPropertyName("id")] + public required string Id { get; set; } + + [JsonPropertyName("name")] + public required string Name { get; set; } + + [JsonPropertyName("visibility")] + public required AzdoProjectVisibility Visibility { get; set; } + + [JsonPropertyName("lastUpdateTime")] + public required DateTimeOffset LastUpdatedTime { get; set; } +} + +[JsonConverter(typeof(JsonStringEnumMemberConverter))] +public enum AzdoProjectVisibility +{ + Private, + Organization, + Public, + SystemPrivate, } diff --git a/server/main.bicep b/server/main.bicep index 468d0dc0..ead3c97c 100644 --- a/server/main.bicep +++ b/server/main.bicep @@ -10,6 +10,9 @@ param synchronizeOnStartup bool = false @description('Whether to create or update subscriptions on startup.') param createOrUpdateWebhooksOnStartup bool = false +@description('JSON array string fo projects to setup. E.g. [{"url":"https://dev.azure.com/tingle/dependabot","token":"dummy","AutoComplete":true}]') +param projectSetups string = '[]' + @description('Access token for authenticating requests to GitHub.') param githubToken string = '' @@ -227,6 +230,7 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { ingress: { external: true, targetPort: 80, traffic: [ { latestRevision: true, weight: 100 } ] } secrets: [ { name: 'connection-strings-application-insights', value: appInsights.properties.ConnectionString } + { name: 'project-setups', value: projectSetups } { name: 'connection-strings-sql' value: join([ @@ -258,6 +262,8 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { { name: 'ASPNETCORE_FORWARDEDHEADERS_ENABLED', value: 'true' } // Application is behind proxy { name: 'EFCORE_PERFORM_MIGRATIONS', value: 'true' } // Perform migrations on startup + { name: 'PROJECT_SETUPS', secretRef: 'project-setups' } + { name: 'AzureAppConfig__Endpoint', value: appConfiguration.properties.endpoint } { name: 'AzureAppConfig__Label', value: 'Production' } diff --git a/server/main.json b/server/main.json index 66a71321..460e0adf 100644 --- a/server/main.json +++ b/server/main.json @@ -30,6 +30,13 @@ "description": "Whether to create or update subscriptions on startup." } }, + "projectSetups": { + "type": "string", + "defaultValue": "[[]", + "metadata": { + "description": "JSON array string fo projects to setup. E.g. [{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]" + } + }, "githubToken": { "type": "string", "defaultValue": "", @@ -341,6 +348,10 @@ "name": "connection-strings-application-insights", "value": "[reference(resourceId('Microsoft.Insights/components', parameters('name')), '2020-02-02').ConnectionString]" }, + { + "name": "project-setups", + "value": "[parameters('projectSetups')]" + }, { "name": "connection-strings-sql", "value": "[join(createArray(format('Server=tcp:{0},1433', reference(resourceId('Microsoft.Sql/servers', format('{0}-{1}', parameters('name'), variables('collisionSuffix'))), '2022-05-01-preview').fullyQualifiedDomainName), format('Initial Catalog={0}', parameters('name')), format('User ID={0}', variables('sqlServerAdministratorLogin')), format('Password={0}', variables('sqlServerAdministratorLoginPassword')), 'Persist Security Info=False', 'MultipleActiveResultSets=False', 'Encrypt=True', 'TrustServerCertificate=False', 'Connection Timeout=30'), ';')]" @@ -379,6 +390,10 @@ "name": "EFCORE_PERFORM_MIGRATIONS", "value": "true" }, + { + "name": "PROJECT_SETUPS", + "secretRef": "project-setups" + }, { "name": "AzureAppConfig__Endpoint", "value": "[reference(resourceId('Microsoft.AppConfiguration/configurationStores', format('{0}-{1}', parameters('name'), variables('collisionSuffix'))), '2023-03-01').endpoint]" From a0dfbbee0d39df23765b2f9b6569a2e75f299d3d Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 18:52:46 +0300 Subject: [PATCH 17/24] synchronize and create/update subscription on startup if there are setups --- docs/server.md | 4 +- server/Tingle.Dependabot/AppSetup.cs | 88 +++++++++---------- .../Workflow/WorkflowOptions.cs | 6 -- server/Tingle.Dependabot/appsettings.json | 2 - server/main.bicep | 8 -- server/main.json | 22 ----- 6 files changed, 44 insertions(+), 86 deletions(-) diff --git a/docs/server.md b/docs/server.md index 4c13c318..7916a2b8 100644 --- a/docs/server.md +++ b/docs/server.md @@ -53,9 +53,7 @@ The deployment exposes the following parameters that can be tuned to suit the se |--|--|--|--| |location|Location to deploy the resources.|No|<resource-group-location>| |name|The name of all resources.|No|`dependabot`| -|synchronizeOnStartup|Whether to synchronize repositories on startup. This option is useful for initial deployments since the server synchronizes every 6 hours. Leaving it on has no harm, it actually helps you find out if the token works based on the logs.|No|false| -|createOrUpdateWebhooksOnStartup|Whether to create or update Azure DevOps subscriptions on startup. This is required if you want configuration files to be picked up automatically and other event driven functionality.
When this is set to `true`, ensure the value provided for `projectToken` has permissions for service hooks and the owner is a Project Administrator. Leaving this on has no harm because the server will only create new subscription if there are no existing ones based on the URL.|No|false| -|projectSetups|A JSON array string representing the projects to be setup on startup. This is useful when running your own setup but may be removed in the future. Example: `[{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]`| +|projectSetups|A JSON array string representing the projects to be setup on startup. This is useful when running your own setup. Example: `[{\"url\":\"https://dev.azure.com/tingle/dependabot\",\"token\":\"dummy\",\"AutoComplete\":true}]`| |githubToken|Access token for authenticating requests to GitHub. Required for vulnerability checks and to avoid rate limiting on free requests|No|<empty>| |imageTag|The image tag to use when pulling the docker containers. A tag also defines the version. You should avoid using `latest`. Example: `1.1.0`|No|<version-downloaded>| |minReplicas|The minimum number of replicas to required for the deployment. Given that scheduling runs in process, this value cannot be less than `1`. This may change in the future.|No|1| diff --git a/server/Tingle.Dependabot/AppSetup.cs b/server/Tingle.Dependabot/AppSetup.cs index 323ba760..29ff3655 100644 --- a/server/Tingle.Dependabot/AppSetup.cs +++ b/server/Tingle.Dependabot/AppSetup.cs @@ -37,66 +37,64 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel } } - // add project if there are projects to be added + // parse projects to be setup + var setupsJson = app.Configuration.GetValue("PROJECT_SETUPS"); + var setups = new List(); + if (!string.IsNullOrWhiteSpace(setupsJson)) + { + setups = JsonSerializer.Deserialize>(setupsJson, serializerOptions)!; + } + + // add projects if there are projects to be added var adoProvider = provider.GetRequiredService(); var context = provider.GetRequiredService(); var projects = await context.Projects.ToListAsync(cancellationToken); - var setupsJson = app.Configuration.GetValue("PROJECT_SETUPS"); - if (!string.IsNullOrWhiteSpace(setupsJson)) + foreach (var setup in setups) { - var setups = JsonSerializer.Deserialize>(setupsJson, serializerOptions)!; - foreach (var setup in setups) + var url = (AzureDevOpsProjectUrl)setup.Url; + var project = projects.SingleOrDefault(p => new Uri(p.Url!) == setup.Url); + if (project is null) { - var url = (AzureDevOpsProjectUrl)setup.Url; - var project = projects.SingleOrDefault(p => new Uri(p.Url!) == setup.Url); - if (project is null) + project = new Models.Management.Project { - project = new Models.Management.Project - { - Id = Guid.NewGuid().ToString("n"), - Created = DateTimeOffset.UtcNow, - Password = GeneratePassword(32), - Url = setup.Url.ToString(), - Type = Models.Management.ProjectType.Azure, - }; - await context.Projects.AddAsync(project, cancellationToken); - } - - project.Token = setup.Token; - project.AutoComplete.Enabled = setup.AutoComplete; - project.AutoComplete.IgnoreConfigs = setup.AutoCompleteIgnoreConfigs; - project.AutoComplete.MergeStrategy = setup.AutoCompleteMergeStrategy; - project.AutoApprove.Enabled = setup.AutoApprove; - project.Secrets = setup.Secrets; - var tp = await adoProvider.GetProjectAsync(project, cancellationToken); - project.Name = tp.Name; - project.ProviderId = tp.Id.ToString(); - if (context.ChangeTracker.HasChanges()) - { - project.Updated = DateTimeOffset.UtcNow; - } + Id = Guid.NewGuid().ToString("n"), + Created = DateTimeOffset.UtcNow, + Password = GeneratePassword(32), + Url = setup.Url.ToString(), + Type = Models.Management.ProjectType.Azure, + }; + await context.Projects.AddAsync(project, cancellationToken); } - // update databases - var updated = await context.SaveChangesAsync(cancellationToken); - - // update projects if we updated the db - projects = updated > 0 ? await context.Projects.ToListAsync(cancellationToken) : projects; + project.Token = setup.Token; + project.AutoComplete.Enabled = setup.AutoComplete; + project.AutoComplete.IgnoreConfigs = setup.AutoCompleteIgnoreConfigs; + project.AutoComplete.MergeStrategy = setup.AutoCompleteMergeStrategy; + project.AutoApprove.Enabled = setup.AutoApprove; + project.Secrets = setup.Secrets; + var tp = await adoProvider.GetProjectAsync(project, cancellationToken); + project.Name = tp.Name; + project.ProviderId = tp.Id.ToString(); + if (context.ChangeTracker.HasChanges()) + { + project.Updated = DateTimeOffset.UtcNow; + } } - var options = provider.GetRequiredService>().Value; + // update database and list of projects + var updated = await context.SaveChangesAsync(cancellationToken); + projects = updated > 0 ? await context.Projects.ToListAsync(cancellationToken) : projects; + + // synchronize and create/update subscriptions if we have setups var synchronizer = provider.GetRequiredService(); - foreach (var project in projects) + if (setups.Count > 0) { - // synchronize project - if (options.SynchronizeOnStartup) + foreach (var project in projects) { + // synchronize project await synchronizer.SynchronizeAsync(project, false, cancellationToken); /* database sync should not trigger, just in case it's too many */ - } - // create or update webhooks/subscriptions - if (options.CreateOrUpdateWebhooksOnStartup) - { + // create or update webhooks/subscriptions await adoProvider.CreateOrUpdateSubscriptionsAsync(project, cancellationToken); } } diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index 5da25971..e2e1d0ab 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -2,12 +2,6 @@ public class WorkflowOptions { - /// Whether to synchronize repositories on startup. - public bool SynchronizeOnStartup { get; set; } = true; - - /// Whether to create/update notifications on startup. - public bool CreateOrUpdateWebhooksOnStartup { get; set; } = true; - /// URL where subscription notifications shall be sent. public Uri? WebhookEndpoint { get; set; } diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index cdec3200..5ffe3566 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -45,8 +45,6 @@ }, "Workflow": { - "SynchronizeOnStartup": false, - "CreateOrUpdateWebhooksOnStartup": false, "WebhookEndpoint": "http://localhost:3000/webhooks/azure", "JobsApiUrl": "http://localhost:3000/", "ResourceGroupId": "/subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT", diff --git a/server/main.bicep b/server/main.bicep index ead3c97c..d98290bd 100644 --- a/server/main.bicep +++ b/server/main.bicep @@ -4,12 +4,6 @@ param location string = resourceGroup().location @description('Name of the resources') param name string = 'dependabot' -@description('Whether to synchronize repositories on startup.') -param synchronizeOnStartup bool = false - -@description('Whether to create or update subscriptions on startup.') -param createOrUpdateWebhooksOnStartup bool = false - @description('JSON array string fo projects to setup. E.g. [{"url":"https://dev.azure.com/tingle/dependabot","token":"dummy","AutoComplete":true}]') param projectSetups string = '[]' @@ -277,8 +271,6 @@ resource app 'Microsoft.App/containerApps@2023-05-01' = { { name: 'Logging__Seq__ServerUrl', value: '' } // set via AppConfig { name: 'Logging__Seq__ApiKey', value: '' } // set via AppConfig - { name: 'Workflow__SynchronizeOnStartup', value: synchronizeOnStartup ? 'true' : 'false' } - { name: 'Workflow__CreateOrUpdateWebhooksOnStartup', value: createOrUpdateWebhooksOnStartup ? 'true' : 'false' } { name: 'Workflow__JobsApiUrl', value: 'https://${name}.${appEnvironment.properties.defaultDomain}' } { name: 'Workflow__WorkingDirectory', value: '/mnt/dependabot' } { name: 'Workflow__WebhookEndpoint', value: 'https://${name}.${appEnvironment.properties.defaultDomain}/webhooks/azure' } diff --git a/server/main.json b/server/main.json index 460e0adf..d9f20455 100644 --- a/server/main.json +++ b/server/main.json @@ -16,20 +16,6 @@ "description": "Name of the resources" } }, - "synchronizeOnStartup": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to synchronize repositories on startup." - } - }, - "createOrUpdateWebhooksOnStartup": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Whether to create or update subscriptions on startup." - } - }, "projectSetups": { "type": "string", "defaultValue": "[[]", @@ -430,14 +416,6 @@ "name": "Logging__Seq__ApiKey", "value": "" }, - { - "name": "Workflow__SynchronizeOnStartup", - "value": "[if(parameters('synchronizeOnStartup'), 'true', 'false')]" - }, - { - "name": "Workflow__CreateOrUpdateWebhooksOnStartup", - "value": "[if(parameters('createOrUpdateWebhooksOnStartup'), 'true', 'false')]" - }, { "name": "Workflow__JobsApiUrl", "value": "[format('https://{0}.{1}', parameters('name'), reference(resourceId('Microsoft.App/managedEnvironments', parameters('name')), '2023-05-01').defaultDomain)]" From 1fcb13035c6db6b79ff0511196b820753e4aba49 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 18:59:34 +0300 Subject: [PATCH 18/24] Allow setting GitHubToken and Location per project --- .../20230824083425_InitialCreate.Designer.cs | 7 +++++++ .../Migrations/20230824083425_InitialCreate.cs | 4 +++- .../Migrations/MainDbContextModelSnapshot.cs | 7 +++++++ .../Models/Management/Project.cs | 15 +++++++++++++++ server/Tingle.Dependabot/Workflow/UpdateRunner.cs | 4 ++-- .../Tingle.Dependabot/Workflow/WorkflowOptions.cs | 1 + 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index a04ab817..efad93cf 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -58,7 +58,14 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .ValueGeneratedOnAddOrUpdate() .HasColumnType("rowversion"); + b.Property("GithubToken") + .HasColumnType("nvarchar(max)"); + + b.Property("Location") + .HasColumnType("nvarchar(max)"); + b.Property("Name") + .IsRequired() .HasColumnType("nvarchar(max)"); b.Property("Password") diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 5bfd84ba..b2f08b78 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -32,7 +32,7 @@ protected override void Up(MigrationBuilder migrationBuilder) Created = table.Column(type: "datetimeoffset", nullable: false), Updated = table.Column(type: "datetimeoffset", nullable: false), Type = table.Column(type: "int", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: true), + Name = table.Column(type: "nvarchar(max)", nullable: false), ProviderId = table.Column(type: "nvarchar(450)", nullable: false), Url = table.Column(type: "nvarchar(max)", nullable: false), Token = table.Column(type: "nvarchar(max)", nullable: false), @@ -42,6 +42,8 @@ protected override void Up(MigrationBuilder migrationBuilder) AutoApprove_Enabled = table.Column(type: "bit", nullable: false), Password = table.Column(type: "nvarchar(450)", nullable: false), Secrets = table.Column(type: "nvarchar(max)", nullable: false), + GithubToken = table.Column(type: "nvarchar(max)", nullable: true), + Location = table.Column(type: "nvarchar(max)", nullable: true), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index f3f643e6..b3b78d9b 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -55,7 +55,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAddOrUpdate() .HasColumnType("rowversion"); + b.Property("GithubToken") + .HasColumnType("nvarchar(max)"); + + b.Property("Location") + .HasColumnType("nvarchar(max)"); + b.Property("Name") + .IsRequired() .HasColumnType("nvarchar(max)"); b.Property("Password") diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index 3deaff79..4d3f2dc0 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -55,6 +55,21 @@ public class Project [JsonIgnore] // expose this once we know how to protect the values public Dictionary Secrets { get; set; } = new(StringComparer.OrdinalIgnoreCase); + /// + /// Token for accessing GitHub APIs. + /// If no value is provided, the a default token is used. + /// Providing a value avoids being rate limited in case when there + /// are many calls at the same time from the same IP. + /// When provided, it must have read access to public repositories. + /// + /// ghp_1234567890 + [JsonIgnore] // expose this once we know how to protect the values + public string? GithubToken { get; set; } + + /// Location/region where to create update jobs. + /// westeurope + public string? Location { get; set; } + [JsonIgnore] // only for internal use public List Repositories { get; set; } = new(); diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 774791d1..c5c1de26 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -78,7 +78,7 @@ public async Task CreateAsync(Project project, Repository repository, Repository foreach (var (key, value) in env) container.Env.Add(new ContainerAppEnvironmentVariable { Name = key, Value = value, }); // prepare the ContainerApp job - var data = new ContainerAppJobData(options.Location!) + var data = new ContainerAppJobData((project.Location ?? options.Location)!) { EnvironmentId = options.AppEnvironmentId, Configuration = new ContainerAppJobConfiguration(ContainerAppJobTriggerType.Manual, 1) @@ -260,7 +260,7 @@ internal async Task> CreateEnvironmentVariables(Proj }; // Add optional values - values.AddIfNotDefault("GITHUB_ACCESS_TOKEN", options.GithubToken) + values.AddIfNotDefault("GITHUB_ACCESS_TOKEN", project.GithubToken ?? options.GithubToken) .AddIfNotDefault("DEPENDABOT_REBASE_STRATEGY", update.RebaseStrategy) .AddIfNotDefault("DEPENDABOT_TARGET_BRANCH", update.TargetBranch) .AddIfNotDefault("DEPENDABOT_VENDOR", update.Vendor ? "true" : null) diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index e2e1d0ab..0008dd63 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -52,5 +52,6 @@ public class WorkflowOptions public string? GithubToken { get; set; } /// Location/region where to create new update jobs. + /// westeurope public string? Location { get; set; } // using Azure.Core.Location does not work when binding from IConfiguration } From 76392ccf23f50f8478859c0bd73ebc2a5375242f Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 19:40:03 +0300 Subject: [PATCH 19/24] Added ProjectId to the UpdateJob --- .../TriggerUpdateJobsEventConsumer.cs | 2 ++ .../Controllers/UpdateJobsController.cs | 25 +++++++++++++------ .../20230824083425_InitialCreate.Designer.cs | 6 +++++ .../20230824083425_InitialCreate.cs | 6 +++++ .../Migrations/MainDbContextModelSnapshot.cs | 6 +++++ .../Tingle.Dependabot/Models/MainDbContext.cs | 1 + .../Models/Management/UpdateJob.cs | 5 ++++ 7 files changed, 43 insertions(+), 8 deletions(-) diff --git a/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs b/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs index e3b1b896..dec830fd 100644 --- a/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs +++ b/server/Tingle.Dependabot/Consumers/TriggerUpdateJobsEventConsumer.cs @@ -89,6 +89,7 @@ public async Task ConsumeAsync(EventContext context, Can Status = UpdateJobStatus.Scheduled, Trigger = evt.Trigger, + ProjectId = project.Id, RepositoryId = repository.Id, RepositorySlug = repository.Slug, EventBusId = eventBusId, @@ -103,6 +104,7 @@ public async Task ConsumeAsync(EventContext context, Can End = null, Duration = null, Log = null, + Error = null, }; await dbContext.UpdateJobs.AddAsync(job, cancellationToken); diff --git a/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs b/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs index 60ef8e3a..af4aab2c 100644 --- a/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs +++ b/server/Tingle.Dependabot/Controllers/UpdateJobsController.cs @@ -33,7 +33,10 @@ public UpdateJobsController(MainDbContext dbContext, IEventPublisher publisher, [HttpPost("{id}/create_pull_request")] public async Task CreatePullRequestAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); + var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); + var project = await dbContext.Projects.SingleAsync(p => p.Id == job.ProjectId); + logger.LogInformation("Received request to create a pull request from job {JobId} but we did nothing.\r\n{ModelJson}", id, JsonSerializer.Serialize(model)); return Ok(); } @@ -41,7 +44,10 @@ public async Task CreatePullRequestAsync([FromRoute, Required] st [HttpPost("{id}/update_pull_request")] public async Task UpdatePullRequestAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); + var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); + var project = await dbContext.Projects.SingleAsync(p => p.Id == job.ProjectId); + logger.LogInformation("Received request to update a pull request from job {JobId} but we did nothing.\r\n{ModelJson}", id, JsonSerializer.Serialize(model)); return Ok(); } @@ -49,7 +55,10 @@ public async Task UpdatePullRequestAsync([FromRoute, Required] st [HttpPost("{id}/close_pull_request")] public async Task ClosePullRequestAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); + var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); + var project = await dbContext.Projects.SingleAsync(p => p.Id == job.ProjectId); + logger.LogInformation("Received request to close a pull request from job {JobId} but we did nothing.\r\n{ModelJson}", id, JsonSerializer.Serialize(model)); return Ok(); } @@ -57,7 +66,7 @@ public async Task ClosePullRequestAsync([FromRoute, Required] str [HttpPost("{id}/record_update_job_error")] public async Task RecordErrorAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); job.Error = new UpdateJobError { @@ -73,7 +82,7 @@ public async Task RecordErrorAsync([FromRoute, Required] string i [HttpPatch("{id}/mark_as_processed")] public async Task MarkAsProcessedAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); // publish event that will run update the job and collect logs var evt = new UpdateJobCheckStateEvent { JobId = id, }; @@ -85,7 +94,7 @@ public async Task MarkAsProcessedAsync([FromRoute, Required] stri [HttpPost("{id}/update_dependency_list")] public async Task UpdateDependencyListAsync([FromRoute, Required] string id, [FromBody] PayloadWithData model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); var repository = await dbContext.Repositories.SingleAsync(r => r.Id == job.RepositoryId); // update the database @@ -102,7 +111,7 @@ public async Task UpdateDependencyListAsync([FromRoute, Required] [HttpPost("{id}/record_ecosystem_versions")] public async Task RecordEcosystemVersionsAsync([FromRoute, Required] string id, [FromBody] JsonNode model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); logger.LogInformation("Received request to record ecosystem version from job {JobId} but we did nothing.\r\n{ModelJson}", id, model.ToJsonString()); return Ok(); } @@ -110,7 +119,7 @@ public async Task RecordEcosystemVersionsAsync([FromRoute, Requir [HttpPost("{id}/increment_metric")] public async Task IncrementMetricAsync([FromRoute, Required] string id, [FromBody] JsonNode model) { - var job = await dbContext.UpdateJobs.SingleAsync(p => p.Id == id); + var job = await dbContext.UpdateJobs.SingleAsync(j => j.Id == id); logger.LogInformation("Received metrics from job {JobId} but we did nothing with them.\r\n{ModelJson}", id, model.ToJsonString()); return Ok(); } diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index efad93cf..5d61393d 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -213,6 +213,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + b.Property("RepositoryId") .IsRequired() .HasColumnType("nvarchar(450)"); @@ -238,6 +242,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("RepositoryId"); b.HasIndex("PackageEcosystem", "Directory"); diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index b2f08b78..113dd204 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -59,6 +59,7 @@ protected override void Up(MigrationBuilder migrationBuilder) Created = table.Column(type: "datetimeoffset", nullable: false), Status = table.Column(type: "int", nullable: false), Trigger = table.Column(type: "int", nullable: false), + ProjectId = table.Column(type: "nvarchar(450)", nullable: false), RepositoryId = table.Column(type: "nvarchar(450)", nullable: false), RepositorySlug = table.Column(type: "nvarchar(max)", nullable: false), EventBusId = table.Column(type: "nvarchar(450)", nullable: true), @@ -174,6 +175,11 @@ protected override void Up(MigrationBuilder migrationBuilder) unique: true, filter: "[EventBusId] IS NOT NULL"); + migrationBuilder.CreateIndex( + name: "IX_UpdateJobs_ProjectId", + table: "UpdateJobs", + column: "ProjectId"); + migrationBuilder.CreateIndex( name: "IX_UpdateJobs_RepositoryId", table: "UpdateJobs", diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index b3b78d9b..9e538a30 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -210,6 +210,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("ProjectId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + b.Property("RepositoryId") .IsRequired() .HasColumnType("nvarchar(450)"); @@ -235,6 +239,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Created") .IsDescending(); + b.HasIndex("ProjectId"); + b.HasIndex("RepositoryId"); b.HasIndex("PackageEcosystem", "Directory"); diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index 226ce7e7..a56bb19c 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -51,6 +51,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) }); builder.HasIndex(j => j.Created).IsDescending(); // faster filtering + builder.HasIndex(j => j.ProjectId); builder.HasIndex(j => j.RepositoryId); builder.HasIndex(j => new { j.PackageEcosystem, j.Directory, }); // faster filtering builder.HasIndex(j => new { j.PackageEcosystem, j.Directory, j.EventBusId, }).IsUnique(); diff --git a/server/Tingle.Dependabot/Models/Management/UpdateJob.cs b/server/Tingle.Dependabot/Models/Management/UpdateJob.cs index 554576de..4971eecb 100644 --- a/server/Tingle.Dependabot/Models/Management/UpdateJob.cs +++ b/server/Tingle.Dependabot/Models/Management/UpdateJob.cs @@ -19,6 +19,11 @@ public class UpdateJob /// Trigger for the update job. public UpdateJobTrigger Trigger { get; set; } + /// Identifier of the project. + [Required] + [JsonIgnore] // only for internal use + public string? ProjectId { get; set; } + /// Identifier of the repository. [Required] [JsonIgnore] // only for internal use From 5d63aa585233b4d8e21943a07c45188d78d3c331 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 19:47:55 +0300 Subject: [PATCH 20/24] Update project during sync and skip sync if done within the last hour --- .../20230824083425_InitialCreate.Designer.cs | 3 +++ .../Migrations/20230824083425_InitialCreate.cs | 1 + .../Migrations/MainDbContextModelSnapshot.cs | 3 +++ .../Models/Management/Project.cs | 3 +++ .../Tingle.Dependabot/Workflow/Synchronizer.cs | 18 +++++++++++++++++- 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index 5d61393d..1874b3b3 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -80,6 +80,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(max)"); + b.Property("Synchronized") + .HasColumnType("datetimeoffset"); + b.Property("Token") .IsRequired() .HasColumnType("nvarchar(max)"); diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index 113dd204..ec4a5530 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -44,6 +44,7 @@ protected override void Up(MigrationBuilder migrationBuilder) Secrets = table.Column(type: "nvarchar(max)", nullable: false), GithubToken = table.Column(type: "nvarchar(max)", nullable: true), Location = table.Column(type: "nvarchar(max)", nullable: true), + Synchronized = table.Column(type: "datetimeoffset", nullable: true), Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index 9e538a30..792fc319 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -77,6 +77,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(max)"); + b.Property("Synchronized") + .HasColumnType("datetimeoffset"); + b.Property("Token") .IsRequired() .HasColumnType("nvarchar(max)"); diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index 4d3f2dc0..15d58382 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -73,6 +73,9 @@ public class Project [JsonIgnore] // only for internal use public List Repositories { get; set; } = new(); + /// Time at which the synchronization was last done for the project. + public DateTimeOffset? Synchronized { get; set; } + [Timestamp] public byte[]? Etag { get; set; } } diff --git a/server/Tingle.Dependabot/Workflow/Synchronizer.cs b/server/Tingle.Dependabot/Workflow/Synchronizer.cs index 92db2245..425933e0 100644 --- a/server/Tingle.Dependabot/Workflow/Synchronizer.cs +++ b/server/Tingle.Dependabot/Workflow/Synchronizer.cs @@ -33,7 +33,20 @@ public Synchronizer(MainDbContext dbContext, AzureDevOpsProvider adoProvider, IE public async Task SynchronizeAsync(Project project, bool trigger, CancellationToken cancellationToken = default) { - // TODO: skip if the project last synchronization is less than 1 hour ago + // skip if the project last synchronization is less than 1 hour ago + if ((DateTimeOffset.UtcNow - project.Synchronized) <= TimeSpan.FromHours(1)) + { + logger.LogInformation("Skipping synchronization for {ProjectUrl} since it last happened recently at {Synchronized}.", project.Url, project.Synchronized); + return; + } + + // update the project (it may have changed name or visibility) + var tp = await adoProvider.GetProjectAsync(project, cancellationToken); + if (!string.Equals(project.Name, tp.Name, StringComparison.Ordinal)) + { + project.Name = tp.Name; + project.Updated = DateTimeOffset.UtcNow; + } // track the synchronization pairs var syncPairs = new List<(SynchronizerConfigurationItem, Repository?)>(); @@ -83,6 +96,9 @@ public async Task SynchronizeAsync(Project project, bool trigger, CancellationTo { await SynchronizeAsync(project, repository, pi, trigger, cancellationToken); } + + // set the last synchronization time on the project + project.Synchronized = DateTimeOffset.UtcNow; } public async Task SynchronizeAsync(Project project, Repository repository, bool trigger, CancellationToken cancellationToken = default) From bd2cef5d57589e39d3b0d23959aaef0208ac7156 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 19:53:41 +0300 Subject: [PATCH 21/24] Save project visibility --- server/Tingle.Dependabot/AppSetup.cs | 9 +++++-- .../20230824083425_InitialCreate.Designer.cs | 3 +++ .../20230824083425_InitialCreate.cs | 1 + .../Migrations/MainDbContextModelSnapshot.cs | 3 +++ .../Models/Azure/AzdoProject.cs | 18 +++++++++++++ .../Models/Azure/AzdoProjectVisibility.cs | 12 +++++++++ .../Models/Management/Project.cs | 3 +++ .../Workflow/AzureDevOpsProvider.cs | 25 +------------------ .../Workflow/Synchronizer.cs | 4 ++- 9 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 server/Tingle.Dependabot/Models/Azure/AzdoProject.cs create mode 100644 server/Tingle.Dependabot/Models/Azure/AzdoProjectVisibility.cs diff --git a/server/Tingle.Dependabot/AppSetup.cs b/server/Tingle.Dependabot/AppSetup.cs index 29ff3655..a1ed866b 100644 --- a/server/Tingle.Dependabot/AppSetup.cs +++ b/server/Tingle.Dependabot/AppSetup.cs @@ -1,5 +1,4 @@ using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; using System.Security.Cryptography; using System.Text.Json; using Tingle.Dependabot.Models; @@ -66,15 +65,21 @@ public static async Task SetupAsync(WebApplication app, CancellationToken cancel await context.Projects.AddAsync(project, cancellationToken); } + // update project using values from the setup project.Token = setup.Token; project.AutoComplete.Enabled = setup.AutoComplete; project.AutoComplete.IgnoreConfigs = setup.AutoCompleteIgnoreConfigs; project.AutoComplete.MergeStrategy = setup.AutoCompleteMergeStrategy; project.AutoApprove.Enabled = setup.AutoApprove; project.Secrets = setup.Secrets; + + // update values from the project var tp = await adoProvider.GetProjectAsync(project, cancellationToken); - project.Name = tp.Name; project.ProviderId = tp.Id.ToString(); + project.Name = tp.Name; + project.Private = tp.Visibility is not Models.Azure.AzdoProjectVisibility.Public; + + // if there are changes, set the Updated field if (context.ChangeTracker.HasChanges()) { project.Updated = DateTimeOffset.UtcNow; diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index 1874b3b3..6ee3f156 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -72,6 +72,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("Private") + .HasColumnType("bit"); + b.Property("ProviderId") .IsRequired() .HasColumnType("nvarchar(450)"); diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index ec4a5530..b863c427 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -36,6 +36,7 @@ protected override void Up(MigrationBuilder migrationBuilder) ProviderId = table.Column(type: "nvarchar(450)", nullable: false), Url = table.Column(type: "nvarchar(max)", nullable: false), Token = table.Column(type: "nvarchar(max)", nullable: false), + Private = table.Column(type: "bit", nullable: false), AutoComplete_Enabled = table.Column(type: "bit", nullable: false), AutoComplete_IgnoreConfigs = table.Column(type: "nvarchar(max)", nullable: true), AutoComplete_MergeStrategy = table.Column(type: "int", nullable: true), diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index 792fc319..ce9bdeb6 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -69,6 +69,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(450)"); + b.Property("Private") + .HasColumnType("bit"); + b.Property("ProviderId") .IsRequired() .HasColumnType("nvarchar(450)"); diff --git a/server/Tingle.Dependabot/Models/Azure/AzdoProject.cs b/server/Tingle.Dependabot/Models/Azure/AzdoProject.cs new file mode 100644 index 00000000..5643de75 --- /dev/null +++ b/server/Tingle.Dependabot/Models/Azure/AzdoProject.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace Tingle.Dependabot.Models.Azure; + +public class AzdoProject +{ + [JsonPropertyName("id")] + public required string Id { get; set; } + + [JsonPropertyName("name")] + public required string Name { get; set; } + + [JsonPropertyName("visibility")] + public required AzdoProjectVisibility Visibility { get; set; } + + [JsonPropertyName("lastUpdateTime")] + public required DateTimeOffset LastUpdatedTime { get; set; } +} diff --git a/server/Tingle.Dependabot/Models/Azure/AzdoProjectVisibility.cs b/server/Tingle.Dependabot/Models/Azure/AzdoProjectVisibility.cs new file mode 100644 index 00000000..645c0ab5 --- /dev/null +++ b/server/Tingle.Dependabot/Models/Azure/AzdoProjectVisibility.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Tingle.Dependabot.Models.Azure; + +[JsonConverter(typeof(JsonStringEnumMemberConverter))] +public enum AzdoProjectVisibility +{ + Private, + Organization, + Public, + SystemPrivate, +} diff --git a/server/Tingle.Dependabot/Models/Management/Project.cs b/server/Tingle.Dependabot/Models/Management/Project.cs index 15d58382..eaf2a6e4 100644 --- a/server/Tingle.Dependabot/Models/Management/Project.cs +++ b/server/Tingle.Dependabot/Models/Management/Project.cs @@ -36,6 +36,9 @@ public class Project [JsonIgnore] // expose this once we know how to protect the values public string? Token { get; set; } + /// Whether the project is private. + public bool Private { get; set; } + /// Auto complete settings. [Required] public ProjectAutoComplete AutoComplete { get; set; } = new(); diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs index 544f633c..1441ab77 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs @@ -10,6 +10,7 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json.Serialization; +using Tingle.Dependabot.Models.Azure; using Tingle.Dependabot.Models.Management; namespace Tingle.Dependabot.Workflow; @@ -270,27 +271,3 @@ static string hash(string v) return await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); } } - -public class AzdoProject -{ - [JsonPropertyName("id")] - public required string Id { get; set; } - - [JsonPropertyName("name")] - public required string Name { get; set; } - - [JsonPropertyName("visibility")] - public required AzdoProjectVisibility Visibility { get; set; } - - [JsonPropertyName("lastUpdateTime")] - public required DateTimeOffset LastUpdatedTime { get; set; } -} - -[JsonConverter(typeof(JsonStringEnumMemberConverter))] -public enum AzdoProjectVisibility -{ - Private, - Organization, - Public, - SystemPrivate, -} diff --git a/server/Tingle.Dependabot/Workflow/Synchronizer.cs b/server/Tingle.Dependabot/Workflow/Synchronizer.cs index 425933e0..eeac35af 100644 --- a/server/Tingle.Dependabot/Workflow/Synchronizer.cs +++ b/server/Tingle.Dependabot/Workflow/Synchronizer.cs @@ -42,9 +42,11 @@ public async Task SynchronizeAsync(Project project, bool trigger, CancellationTo // update the project (it may have changed name or visibility) var tp = await adoProvider.GetProjectAsync(project, cancellationToken); - if (!string.Equals(project.Name, tp.Name, StringComparison.Ordinal)) + var @private = tp.Visibility is not Models.Azure.AzdoProjectVisibility.Public; + if (!string.Equals(project.Name, tp.Name, StringComparison.Ordinal) || @private != project.Private) { project.Name = tp.Name; + project.Private = @private; project.Updated = DateTimeOffset.UtcNow; } From 32c6d500bd5ffaf1789df24f1b4caf92f2fbb08f Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 20:02:23 +0300 Subject: [PATCH 22/24] Select V2 updater using feature flag --- server/Tingle.Dependabot/Constants.cs | 5 ++-- .../ProjectTargetingContextAccessor.cs | 13 +++------- .../Workflow/UpdateRunner.cs | 24 +++++++++++++++---- server/Tingle.Dependabot/appsettings.json | 3 ++- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/server/Tingle.Dependabot/Constants.cs b/server/Tingle.Dependabot/Constants.cs index c5fb0346..6b34147e 100644 --- a/server/Tingle.Dependabot/Constants.cs +++ b/server/Tingle.Dependabot/Constants.cs @@ -22,7 +22,8 @@ internal static class ErrorCodes internal static class FeatureNames { - internal const string DebugAllJobs = "DebugAllJobs"; - internal const string DebugJobs = "DebugJobs"; + internal const string DebugAllJobs = "DebugAllJobs"; // Whether to debug all jobs (controls environment variable value). + internal const string DebugJobs = "DebugJobs"; // Whether to debug jobs (controls value in job definition). internal const string DeterministicUpdates = "DeterministicUpdates"; // Whether updates should be created in the same order. + internal const string UpdaterV2 = "UpdaterV2"; // Whether to use V2 updater } diff --git a/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs b/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs index bbafcaa4..82af1ca6 100644 --- a/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs +++ b/server/Tingle.Dependabot/FeatureManagement/ProjectTargetingContextAccessor.cs @@ -20,16 +20,9 @@ public ValueTask GetContextAsync() { var httpContext = contextAccessor.HttpContext!; - // Prepare the groups - var groups = new List(); // where can we get the workspace groups? - - // Build targeting context based off workspace info - var workspaceId = httpContext.GetProjectId(); - var targetingContext = new TargetingContext - { - UserId = workspaceId, - Groups = groups - }; + // Build targeting context based off project info + var projectId = httpContext.GetProjectId(); + var targetingContext = new TargetingContext { UserId = projectId, }; return new ValueTask(targetingContext); } diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index c5c1de26..07713f7e 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -57,6 +57,10 @@ public async Task CreateAsync(Project project, Repository repository, Repository } catch (Azure.RequestFailedException rfe) when (rfe.Status is 404) { } + // check if V2 updater is enabled for the project via Feature Management + var fmc = MakeTargetingContext(project, job); + var useV2 = await featureManager.IsEnabledAsync(FeatureNames.UpdaterV2, fmc); + // prepare credentials with replaced secrets var secrets = new Dictionary(project.Secrets) { ["DEFAULT_TOKEN"] = project.Token!, }; var registries = update.Registries?.Select(r => repository.Registries[r]).ToList(); @@ -71,7 +75,7 @@ public async Task CreateAsync(Project project, Repository repository, Repository Name = UpdaterContainerName, Image = options.UpdaterContainerImageTemplate!.Replace("{{ecosystem}}", job.PackageEcosystem), Resources = job.Resources!, - Args = { "update_script", }, + Args = { useV2 ? "update_files" : "update_script", }, VolumeMounts = { new ContainerAppVolumeMount { VolumeName = volumeName, MountPath = options.WorkingDirectory, }, }, }; var env = await CreateEnvironmentVariables(project, repository, update, job, directory, credentials, cancellationToken); @@ -231,8 +235,8 @@ internal async Task> CreateEnvironmentVariables(Proj [return: NotNullIfNotNull(nameof(value))] static string? ToJson(T? value) => value is null ? null : JsonSerializer.Serialize(value, serializerOptions); // null ensures we do not add to the values - // check if debug is enabled for the project via Feature Management - var fmc = new TargetingContext { Groups = new[] { $"provider:{project.Type.ToString().ToLower()}", $"project:{project.Id}", $"ecosystem:{job.PackageEcosystem}", }, }; + // check if debug and determinism is enabled for the project via Feature Management + var fmc = MakeTargetingContext(project, job); var debugAllJobs = await featureManager.IsEnabledAsync(FeatureNames.DebugAllJobs, fmc); var deterministic = await featureManager.IsEnabledAsync(FeatureNames.DeterministicUpdates, fmc); @@ -300,7 +304,7 @@ internal async Task WriteJobDefinitionAsync(Project project, var credentialsMetadata = MakeCredentialsMetadata(credentials); // check if debug is enabled for the project via Feature Management - var fmc = new TargetingContext { Groups = new[] { $"provider:{project.Type.ToString().ToLower()}", $"project:{project.Id}", $"ecosystem:{job.PackageEcosystem}", }, }; + var fmc = MakeTargetingContext(project, job); var debug = await featureManager.IsEnabledAsync(FeatureNames.DebugJobs, fmc); var definition = new JsonObject @@ -350,6 +354,18 @@ internal async Task WriteJobDefinitionAsync(Project project, return path; } + internal static TargetingContext MakeTargetingContext(Project project, UpdateJob job) + { + return new TargetingContext + { + Groups = new[] + { + $"provider:{project.Type.ToString().ToLower()}", + $"project:{project.Id}", + $"ecosystem:{job.PackageEcosystem}", + }, + }; + } internal static IList> MakeCredentialsMetadata(IList> credentials) { return credentials.Select(cred => diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index 5ffe3566..7d057074 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -59,6 +59,7 @@ "FeatureManagement": { "DebugAllJobs": false, "DebugJobs": false, - "DeterministicUpdates": false + "DeterministicUpdates": false, + "UpdaterV2": false } } From 365d2266a574112f0eb719ff7c986fa1e5d044f3 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 20:12:46 +0300 Subject: [PATCH 23/24] Update unit tests --- .../MissedTriggerCheckerTaskTests.cs | 12 ++++++ .../PeriodicTasks/SynchronizationTaskTests.cs | 27 +++++++++++++- .../UpdateJobsCleanerTaskTests.cs | 18 +++++++++ .../WebhooksControllerIntegrationTests.cs | 37 +++++++++++-------- .../Workflow/AzureDevOpsProvider.cs | 1 - 5 files changed, 77 insertions(+), 18 deletions(-) diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs index d4810ec7..df2285f2 100644 --- a/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/MissedTriggerCheckerTaskTests.cs @@ -16,6 +16,7 @@ namespace Tingle.Dependabot.Tests.PeriodicTasks; public class MissedTriggerCheckerTaskTests { + private const string ProjectId = "prj_1234567890"; private const string RepositoryId = "repo_1234567890"; private const int UpdateId1 = 1; @@ -103,9 +104,20 @@ private async Task TestAsync(DateTimeOffset? lastUpdate0, DateTimeOffset? lastUp var context = provider.GetRequiredService(); await context.Database.EnsureCreatedAsync(); + await context.Projects.AddAsync(new Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); await context.Repositories.AddAsync(new Repository { Id = RepositoryId, + ProjectId = ProjectId, + ProviderId = Guid.NewGuid().ToString(), Name = "test-repo", ConfigFileContents = "", Updates = new List diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs index 37a256e3..105f57c5 100644 --- a/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/SynchronizationTaskTests.cs @@ -1,7 +1,10 @@ -using Microsoft.Extensions.DependencyInjection; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Tingle.Dependabot.Events; +using Tingle.Dependabot.Models; +using Tingle.Dependabot.Models.Management; using Tingle.Dependabot.Workflow; using Tingle.EventBus; using Tingle.EventBus.Transports.InMemory; @@ -12,6 +15,8 @@ namespace Tingle.Dependabot.Tests.PeriodicTasks; public class SynchronizationTaskTests { + private const string ProjectId = "prj_1234567890"; + private readonly ITestOutputHelper outputHelper; public SynchronizationTaskTests(ITestOutputHelper outputHelper) @@ -42,6 +47,12 @@ private async Task TestAsync(Func builder.AddXUnit(outputHelper)) .ConfigureServices((context, services) => { + var dbName = Guid.NewGuid().ToString(); + services.AddDbContext(options => + { + options.UseInMemoryDatabase(dbName, o => o.EnableNullChecks()); + options.EnableDetailedErrors(); + }); services.AddEventBus(builder => builder.AddInMemoryTransport().AddInMemoryTestHarness()); }) .Build(); @@ -49,6 +60,20 @@ private async Task TestAsync(Func(); + await context.Database.EnsureCreatedAsync(); + + await context.Projects.AddAsync(new Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); + await context.SaveChangesAsync(); + var harness = provider.GetRequiredService(); await harness.StartAsync(); diff --git a/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs b/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs index 997dafcf..241ac77f 100644 --- a/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs +++ b/server/Tingle.Dependabot.Tests/PeriodicTasks/UpdateJobsCleanerTaskTests.cs @@ -16,6 +16,7 @@ namespace Tingle.Dependabot.Tests.PeriodicTasks; public class UpdateJobsCleanerTaskTests { + private const string ProjectId = "prj_1234567890"; private const string RepositoryId = "repo_1234567890"; private readonly ITestOutputHelper outputHelper; @@ -34,6 +35,7 @@ await TestAsync(async (harness, context, pt) => await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddMinutes(-19), @@ -46,6 +48,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddHours(-100), @@ -58,6 +61,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = targetId, + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddMinutes(-30), @@ -87,6 +91,7 @@ await TestAsync(async (harness, context, pt) => await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddDays(-80), @@ -98,6 +103,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddDays(-100), @@ -109,6 +115,7 @@ await context.UpdateJobs.AddAsync(new UpdateJob await context.UpdateJobs.AddAsync(new UpdateJob { Id = Guid.NewGuid().ToString(), + ProjectId = ProjectId, RepositoryId = RepositoryId, RepositorySlug = "test-repo", Created = DateTimeOffset.UtcNow.AddDays(-120), @@ -146,9 +153,20 @@ private async Task TestAsync(Func(); await context.Database.EnsureCreatedAsync(); + await context.Projects.AddAsync(new Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); await context.Repositories.AddAsync(new Repository { Id = RepositoryId, + ProjectId = ProjectId, + ProviderId = Guid.NewGuid().ToString(), Name = "test-repo", ConfigFileContents = "", Updates = new List diff --git a/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs b/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs index 4570f07f..640c6a65 100644 --- a/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs +++ b/server/Tingle.Dependabot.Tests/WebhooksControllerIntegrationTests.cs @@ -3,7 +3,6 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System.Net; @@ -20,6 +19,8 @@ namespace Tingle.Dependabot.Tests; public class WebhooksControllerIntegrationTests { + private const string ProjectId = "prj_1234567890"; + private readonly ITestOutputHelper outputHelper; public WebhooksControllerIntegrationTests(ITestOutputHelper outputHelper) @@ -41,7 +42,7 @@ await TestAsync(async (harness, client) => // password does not match what is on record request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump5"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump5"))); response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); Assert.Empty(await response.Content.ReadAsStringAsync()); @@ -55,7 +56,7 @@ public async Task Returns_BadRequest_NoBody() await TestAsync(async (harness, client) => { var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StringContent("", Encoding.UTF8, "application/json"); var response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); @@ -74,7 +75,7 @@ public async Task Returns_BadRequest_MissingValues() await TestAsync(async (harness, client) => { var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StringContent("{}", Encoding.UTF8, "application/json"); var response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); @@ -96,7 +97,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestUpdated1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); var response = await client.SendAsync(request); Assert.Equal(HttpStatusCode.UnsupportedMediaType, response.StatusCode); @@ -115,7 +116,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsGitPush1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -139,7 +140,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestUpdated1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -156,7 +157,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestMerged1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -173,7 +174,7 @@ await TestAsync(async (harness, client) => { var stream = TestSamples.GetAzureDevOpsPullRequestCommentEvent1(); var request = new HttpRequestMessage(HttpMethod.Post, "/webhooks/azure"); - request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes("vsts:burp-bump"))); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{ProjectId}:burp-bump"))); request.Content = new StreamContent(stream); request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json", "utf-8"); var response = await client.SendAsync(request); @@ -188,13 +189,6 @@ private async Task TestAsync(Func execute // Arrange var builder = new WebHostBuilder() .ConfigureLogging(builder => builder.AddXUnit(outputHelper)) - .ConfigureAppConfiguration(builder => - { - builder.AddInMemoryCollection(new Dictionary - { - ["Authentication:Schemes:ServiceHooks:Credentials:vsts"] = "burp-bump", - }); - }) .ConfigureServices((context, services) => { services.AddControllers() @@ -247,6 +241,17 @@ private async Task TestAsync(Func execute var context = provider.GetRequiredService(); await context.Database.EnsureCreatedAsync(); + await context.Projects.AddAsync(new Dependabot.Models.Management.Project + { + Id = ProjectId, + Url = "https://dev.azure.com/dependabot/dependabot", + Token = "token", + Name = "dependabot", + ProviderId = "6ce954b1-ce1f-45d1-b94d-e6bf2464ba2c", + Password = "burp-bump", + }); + await context.SaveChangesAsync(); + var harness = provider.GetRequiredService(); await harness.StartAsync(); diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs index 1441ab77..735b0431 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProvider.cs @@ -9,7 +9,6 @@ using System.Net.Http.Headers; using System.Security.Cryptography; using System.Text; -using System.Text.Json.Serialization; using Tingle.Dependabot.Models.Azure; using Tingle.Dependabot.Models.Management; From ff2b86112f2f4608482acf6e1ddc6de177f5c6d6 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Wed, 20 Sep 2023 21:34:38 +0300 Subject: [PATCH 24/24] Fix issue with app insights --- .../Extensions/IServiceCollectionExtensions.cs | 8 ++++---- server/Tingle.Dependabot/Tingle.Dependabot.csproj | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs index c98f0270..1e3d6e5a 100644 --- a/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs +++ b/server/Tingle.Dependabot/Extensions/IServiceCollectionExtensions.cs @@ -30,11 +30,11 @@ public static IServiceCollection AddStandardApplicationInsights(this IServiceCol // according to docs link below, this registration should be singleton // https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-core#adding-telemetryinitializers services.AddSingleton(); - services.AddApplicationInsightsTelemetryExtras(); // Add other extras + // services.AddApplicationInsightsTelemetryExtras(); // Add other extras - services.AddActivitySourceDependencyCollector(new[] { - "Tingle.EventBus", - }); + // services.AddActivitySourceDependencyCollector(new[] { + // "Tingle.EventBus", + // }); return services; } diff --git a/server/Tingle.Dependabot/Tingle.Dependabot.csproj b/server/Tingle.Dependabot/Tingle.Dependabot.csproj index e73f33a9..11cf3eee 100644 --- a/server/Tingle.Dependabot/Tingle.Dependabot.csproj +++ b/server/Tingle.Dependabot/Tingle.Dependabot.csproj @@ -39,7 +39,7 @@ - +