From 879e9e3afebc90d2fdd44c91ba2e96db40be24ab Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Mon, 11 Sep 2023 13:22:05 +0300 Subject: [PATCH] Filter relevant registries after validating the configuration (#781) --- extension/task/IDependabotConfig.ts | 16 ++-- extension/task/index.ts | 12 ++- .../task/utils/getAzureDevOpsAccessToken.ts | 2 +- extension/task/utils/parseConfigFile.ts | 46 ++++++++-- extension/tests/utils/dependabot.yml | 42 +++++++++ extension/tests/utils/parseConfigFile.test.ts | 91 ++++++++++++++++--- .../AzureDevOpsEventHandlerTests.cs | 2 - .../Models/DependabotConfigurationTests.cs | 74 ++++++++++++++- .../Samples/dependabot.yml | 7 +- .../Workflow/UpdateRunnerTests.cs | 2 +- .../AzureDevOpsEventHandler.cs | 4 +- .../20230824083425_InitialCreate.Designer.cs | 74 +-------------- .../20230824083425_InitialCreate.cs | 4 +- .../Migrations/MainDbContextModelSnapshot.cs | 74 +-------------- .../Models/DependabotConfiguration.cs | 28 +++++- .../Tingle.Dependabot/Models/MainDbContext.cs | 31 ++++++- server/Tingle.Dependabot/Models/Repository.cs | 2 +- .../Workflow/Synchronizer.cs | 2 +- .../Workflow/UpdateRunner.cs | 3 +- 19 files changed, 322 insertions(+), 194 deletions(-) create mode 100644 extension/tests/utils/dependabot.yml diff --git a/extension/task/IDependabotConfig.ts b/extension/task/IDependabotConfig.ts index 58896c32..e8c52f92 100644 --- a/extension/task/IDependabotConfig.ts +++ b/extension/task/IDependabotConfig.ts @@ -16,18 +16,18 @@ export interface IDependabotConfig { /** * Optional. Specify authentication details to access private package registries. */ - registries?: IDependabotRegistry[]; + registries?: Record; } export interface IDependabotUpdate { - /** - * Location of package manifests. - * */ - directory: string; /** * Package manager to use. * */ packageEcosystem: string; + /** + * Location of package manifests. + * */ + directory: string; schedule?: IDependabotUpdateSchedule; /** * Customize which updates are allowed. @@ -56,11 +56,15 @@ export interface IDependabotUpdate { /** * Whether to reject external code */ - rejectExternalCode: boolean; + insecureExternalCodeExecution?: string; /** * Limit number of open pull requests for version updates. */ openPullRequestsLimit?: number; + /** + * Registries configured for this update. + */ + registries: string[]; /** * Branch to create pull requests against. */ diff --git a/extension/task/index.ts b/extension/task/index.ts index a5d7def3..5b828060 100644 --- a/extension/task/index.ts +++ b/extension/task/index.ts @@ -1,6 +1,6 @@ import * as tl from "azure-pipelines-task-lib/task" import { ToolRunner } from "azure-pipelines-task-lib/toolrunner" -import { IDependabotConfig, IDependabotUpdate } from "./IDependabotConfig"; +import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from "./IDependabotConfig"; import getSharedVariables from "./utils/getSharedVariables"; import { parseConfigFile } from "./utils/parseConfigFile"; @@ -88,7 +88,7 @@ async function run() { } // Set exception behaviour if true - if (update.rejectExternalCode === true) { + if (update.insecureExternalCodeExecution === "deny") { dockerRunner.arg(["-e", 'DEPENDABOT_REJECT_EXTERNAL_CODE=true']); } @@ -130,8 +130,12 @@ async function run() { } // Set the extra credentials - if (config.registries != undefined && config.registries.length > 0) { - let extraCredentials = JSON.stringify(config.registries, (k, v) => v === null ? undefined : v); + if (config.registries != undefined && Object.keys(config.registries).length > 0) { + let selectedRegistries: IDependabotRegistry[] = []; + for (const reg of update.registries) { + selectedRegistries.push(config.registries[reg]); + } + let extraCredentials = JSON.stringify(selectedRegistries, (k, v) => v === null ? undefined : v); dockerRunner.arg(["-e", `DEPENDABOT_EXTRA_CREDENTIALS=${extraCredentials}`]); } diff --git a/extension/task/utils/getAzureDevOpsAccessToken.ts b/extension/task/utils/getAzureDevOpsAccessToken.ts index a71a103a..81b373bc 100644 --- a/extension/task/utils/getAzureDevOpsAccessToken.ts +++ b/extension/task/utils/getAzureDevOpsAccessToken.ts @@ -24,7 +24,7 @@ export default function getAzureDevOpsAccessToken() { debug(`Loading authorization for service connection ${serviceConnectionName}`); return getEndpointAuthorizationParameter(serviceConnectionName, "AccessToken", false); } - + debug("No custom token provided. The SystemVssConnection's AccessToken shall be used."); return getEndpointAuthorizationParameter( "SystemVssConnection", diff --git a/extension/task/utils/parseConfigFile.ts b/extension/task/utils/parseConfigFile.ts index c4e36a57..52c10092 100644 --- a/extension/task/utils/parseConfigFile.ts +++ b/extension/task/utils/parseConfigFile.ts @@ -133,13 +133,15 @@ async function parseConfigFile(variables: ISharedVariables): Promise { + var registries: Record = {}; var rawRegistries = config["registries"]; @@ -243,7 +246,7 @@ function parseRegistries(config: any): IDependabotRegistry[] { var type = rawType?.replace("-", "_"); var parsed: IDependabotRegistry = { type: type, }; - registries.push(parsed); + registries[registryConfigKey] = parsed; // handle special fields for 'hex-organization' types if (type === 'hex_organization') { @@ -311,6 +314,29 @@ function parseRegistries(config: any): IDependabotRegistry[] { return registries; } +function validateConfiguration(updates: IDependabotUpdate[], registries: Record) { + const configured = Object.keys(registries); + const referenced: string[] = []; + for (const u of updates) referenced.push(...u.registries); + + // ensure there are no configured registries that have not been referenced + const missingConfiguration = referenced.filter((el) => !configured.includes(el)); + if (missingConfiguration.length > 0) { + throw new Error( + `Referenced registries: '${missingConfiguration.join(',')}' have not been configured in the root of dependabot.yml` + ); + } + + // ensure there are no registries referenced but not configured + const missingReferences = configured.filter((el) => !referenced.includes(el)); + if (missingReferences.length > 0) + { + throw new Error( + `Registries: '${missingReferences.join(',')}' have not been referenced by any update` + ); + } +} + const KnownRegistryTypes = [ "composer-repository", "docker-registry", @@ -325,4 +351,4 @@ const KnownRegistryTypes = [ "terraform-registry", ]; -export { parseConfigFile, parseUpdates, parseRegistries, }; +export { parseConfigFile, parseUpdates, parseRegistries, validateConfiguration, }; diff --git a/extension/tests/utils/dependabot.yml b/extension/tests/utils/dependabot.yml new file mode 100644 index 00000000..f10d0673 --- /dev/null +++ b/extension/tests/utils/dependabot.yml @@ -0,0 +1,42 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: 'docker' # See documentation for possible values + directory: '/' # Location of package manifests + schedule: + interval: 'weekly' + time: '03:00' + day: 'sunday' + open-pull-requests-limit: 10 + - package-ecosystem: 'npm' # See documentation for possible values + directory: '/client' # Location of package manifests + schedule: + interval: 'daily' + time: '03:15' + open-pull-requests-limit: 10 + registries: + - reg1 + - reg2 + insecure-external-code-execution: 'deny' + ignore: + - dependency-name: 'react' + update-types: ['version-update:semver-major'] + - dependency-name: 'react-dom' + update-types: ['version-update:semver-major'] + - dependency-name: '@types/react' + update-types: ['version-update:semver-major'] + - dependency-name: '@types/react-dom' + update-types: ['version-update:semver-major'] +registries: + reg1: + type: nuget-feed + url: 'https://pkgs.dev.azure.com/dependabot/_packaging/dependabot/nuget/v3/index.json' + token: ':${{DEFAULT_TOKEN}}' + reg2: + type: npm-registry + url: 'https://pkgs.dev.azure.com/dependabot/_packaging/dependabot-npm/npm/registry/' + token: 'tingle-npm:${{DEFAULT_TOKEN}}' diff --git a/extension/tests/utils/parseConfigFile.test.ts b/extension/tests/utils/parseConfigFile.test.ts index 87e55ce9..64e7448e 100644 --- a/extension/tests/utils/parseConfigFile.test.ts +++ b/extension/tests/utils/parseConfigFile.test.ts @@ -1,16 +1,39 @@ import { load } from "js-yaml"; import * as fs from "fs"; import * as path from "path"; -import { parseRegistries } from "../../task/utils/parseConfigFile"; +import { parseRegistries, parseUpdates, validateConfiguration } from "../../task/utils/parseConfigFile"; +import { IDependabotRegistry, IDependabotUpdate } from "../../task/IDependabotConfig"; + +describe("Parse configuration file", () => { + it("Parsing works as expected", () => { + let config: any = load(fs.readFileSync('tests/utils/dependabot.yml', "utf-8")); + let updates = parseUpdates(config); + expect(updates.length).toBe(2); + + // first + const first = updates[0]; + expect(first.directory).toBe('/'); + expect(first.packageEcosystem).toBe('docker'); + expect(first.insecureExternalCodeExecution).toBe(undefined); + expect(first.registries).toEqual([]); + + // second + const second = updates[1]; + expect(second.directory).toBe('/client'); + expect(second.packageEcosystem).toBe('npm'); + expect(second.insecureExternalCodeExecution).toBe('deny'); + expect(second.registries).toEqual(['reg1', 'reg2']); + }); +}); describe("Parse registries", () => { it("Parsing works as expected", () => { let config: any = load(fs.readFileSync('tests/utils/sample-registries.yml', "utf-8")); let registries = parseRegistries(config); - expect(registries.length).toBe(11); + expect(Object.keys(registries).length).toBe(11); // composer-repository - var registry = registries[0]; + var registry = registries['composer']; expect(registry.type).toBe('composer_repository'); expect(registry.url).toBe('https://repo.packagist.com/example-company/'); expect(registry["index-url"]).toBe(undefined); @@ -27,7 +50,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(undefined); // docker-registry - registry = registries[1]; + registry = registries['dockerhub']; expect(registry.type).toBe('docker_registry'); expect(registry.url).toBe(undefined); expect(registry["index-url"]).toBe(undefined); @@ -44,7 +67,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(true); // git - registry = registries[2]; + registry = registries['github-octocat']; expect(registry.type).toBe('git'); expect(registry.url).toBe('https://github.com'); expect(registry["index-url"]).toBe(undefined); @@ -61,7 +84,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(undefined); // hex-organization - registry = registries[3]; + registry = registries['github-hex-org']; expect(registry.type).toBe('hex_organization'); expect(registry.url).toBe(undefined); expect(registry["index-url"]).toBe(undefined); @@ -78,7 +101,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(undefined); // hex-repository - registry = registries[4]; + registry = registries['github-hex-repository']; expect(registry.type).toBe('hex_repository'); expect(registry.url).toBe('https://private-repo.example.com'); expect(registry.registry).toBe(undefined); @@ -94,7 +117,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(undefined); // maven-repository - registry = registries[5]; + registry = registries['maven-artifactory']; expect(registry.type).toBe('maven_repository'); expect(registry.url).toBe('https://artifactory.example.com'); expect(registry["index-url"]).toBe(undefined); @@ -111,7 +134,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(true); // npm-registry - registry = registries[6]; + registry = registries['npm-github']; expect(registry.type).toBe('npm_registry'); expect(registry.url).toBe(undefined); expect(registry["index-url"]).toBe(undefined); @@ -128,7 +151,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(true); // nuget-feed - registry = registries[7]; + registry = registries['nuget-azure-devops']; expect(registry.type).toBe('nuget_feed'); expect(registry.url).toBe('https://pkgs.dev.azure.com/contoso/_packaging/My_Feed/nuget/v3/index.json'); expect(registry["index-url"]).toBe(undefined); @@ -145,7 +168,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(undefined); // python-index - registry = registries[8]; + registry = registries['python-azure']; expect(registry.type).toBe('python_index'); expect(registry.url).toBe(undefined); expect(registry["index-url"]).toBe('https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example'); @@ -162,7 +185,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(true); // rubygems-server - registry = registries[9]; + registry = registries['ruby-github']; expect(registry.type).toBe('rubygems_server'); expect(registry.url).toBe('https://rubygems.pkg.github.com/octocat/github_api'); expect(registry["index-url"]).toBe(undefined); @@ -179,7 +202,7 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(false); // terraform-registry - registry = registries[10]; + registry = registries['terraform-example']; expect(registry.type).toBe('terraform_registry'); expect(registry.url).toBe(undefined); expect(registry["index-url"]).toBe(undefined); @@ -196,3 +219,45 @@ describe("Parse registries", () => { expect(registry["replaces-base"]).toBe(undefined); }); }); + +describe("Validate registries", () => { + it("Validation works as expected", () => { + // let config: any = load(fs.readFileSync('tests/utils/dependabot.yml', "utf-8")); + // let updates = parseUpdates(config); + // expect(updates.length).toBe(2); + + var updates: IDependabotUpdate[] = [ + { + packageEcosystem: "npm", + directory: "/", + registries: ["dummy1", "dummy2"], + }, + ]; + + var registries: Record = { + 'dummy1': { + type: 'nuget', + url: "https://pkgs.dev.azure.com/contoso/_packaging/My_Feed/nuget/v3/index.json", + token: "pwd_1234567890", + }, + 'dummy2': { + type: "python-index", + url: "https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example", + username: "octocat@example.com", + password: "pwd_1234567890", + "replaces-base": true, + }, + }; + + // works as expected + validateConfiguration(updates, registries); + + // fails: registry not referenced + updates[0].registries = []; + expect(() => validateConfiguration(updates, registries)).toThrow(`Registries: 'dummy1,dummy2' have not been referenced by any update`); + + // fails: registrynot configured + updates[0].registries = ["dummy1", "dummy2", "dummy3",]; + expect(() => validateConfiguration(updates, registries)).toThrow(`Referenced registries: 'dummy3' have not been configured in the root of dependabot.yml`); + }); +}); diff --git a/server/Tingle.Dependabot.Tests/AzureDevOpsEventHandlerTests.cs b/server/Tingle.Dependabot.Tests/AzureDevOpsEventHandlerTests.cs index e11b4704..190595e6 100644 --- a/server/Tingle.Dependabot.Tests/AzureDevOpsEventHandlerTests.cs +++ b/server/Tingle.Dependabot.Tests/AzureDevOpsEventHandlerTests.cs @@ -1,13 +1,11 @@ using AspNetCore.Authentication.Basic; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http.Json; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using System.Net; using System.Text; using Tingle.Dependabot.Events; diff --git a/server/Tingle.Dependabot.Tests/Models/DependabotConfigurationTests.cs b/server/Tingle.Dependabot.Tests/Models/DependabotConfigurationTests.cs index 33a8e274..c9dc68ba 100644 --- a/server/Tingle.Dependabot.Tests/Models/DependabotConfigurationTests.cs +++ b/server/Tingle.Dependabot.Tests/Models/DependabotConfigurationTests.cs @@ -1,4 +1,5 @@ -using Tingle.Dependabot.Models; +using System.ComponentModel.DataAnnotations; +using Tingle.Dependabot.Models; using Xunit; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -19,9 +20,9 @@ public void Deserialization_Works() var configuration = deserializer.Deserialize(reader); Assert.NotNull(configuration); - Assert.Equal(2, configuration!.Version); - Assert.NotNull(configuration.Updates!); - Assert.Equal(2, configuration.Updates!.Count); + Assert.Equal(2, configuration.Version); + Assert.NotNull(configuration.Updates); + Assert.Equal(2, configuration.Updates.Count); var first = configuration.Updates[0]; Assert.Equal("/", first.Directory); @@ -29,7 +30,9 @@ public void Deserialization_Works() Assert.Equal(DependabotScheduleInterval.Weekly, first.Schedule?.Interval); Assert.Equal(new TimeOnly(3, 0), first.Schedule?.Time); Assert.Equal(DependabotScheduleDay.Sunday, first.Schedule?.Day); + Assert.Equal("Etc/UTC", first.Schedule?.Timezone); Assert.Null(first.InsecureExternalCodeExecution); + Assert.Null(first.Registries); var second = configuration.Updates[1]; Assert.Equal("/client", second.Directory); @@ -37,6 +40,69 @@ public void Deserialization_Works() Assert.Equal(DependabotScheduleInterval.Daily, second.Schedule?.Interval); Assert.Equal(new TimeOnly(3, 15), second.Schedule?.Time); Assert.Equal(DependabotScheduleDay.Monday, second.Schedule?.Day); + Assert.Equal("Etc/UTC", second.Schedule?.Timezone); Assert.Equal("deny", second.InsecureExternalCodeExecution); + Assert.Equal(new[] { "reg1", "reg2", }, second.Registries); + } + + [Fact] + public void Validation_Works() + { + var configuration = new DependabotConfiguration + { + Version = 2, + Updates = new List + { + new DependabotUpdate + { + PackageEcosystem = "npm", + Directory = "/", + Registries = new List { "dummy1", "dummy2", }, + }, + }, + Registries = new Dictionary + { + ["dummy1"] = new DependabotRegistry + { + Type = "nuget", + Url = "https://pkgs.dev.azure.com/contoso/_packaging/My_Feed/nuget/v3/index.json", + Token = "pwd_1234567890", + }, + ["dummy2"] = new DependabotRegistry + { + Type = "python-index", + Url = "https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example", + Username = "octocat@example.com", + Password = "pwd_1234567890", + ReplacesBase = true, + }, + }, + }; + + // works as expected + var results = new List(); + var actual = RecursiveValidator.TryValidateObject(configuration, results); + Assert.True(actual); + Assert.Empty(results); + + // fails: registry not referenced + configuration.Updates[0].Registries?.Clear(); + results = new List(); + actual = RecursiveValidator.TryValidateObject(configuration, results); + Assert.False(actual); + var val = Assert.Single(results); + Assert.Empty(val.MemberNames); + Assert.NotNull(val.ErrorMessage); + Assert.Equal("Registries: 'dummy1,dummy2' have not been referenced by any update", val.ErrorMessage); + + // fails: registrynot configured + configuration.Updates[0].Registries?.AddRange(new[] { "dummy1", "dummy2", "dummy3" }); + results = new List(); + actual = RecursiveValidator.TryValidateObject(configuration, results); + Assert.False(actual); + val = Assert.Single(results); + Assert.Empty(val.MemberNames); + Assert.NotNull(val.ErrorMessage); + Assert.Equal("Referenced registries: 'dummy3' have not been configured in the root of dependabot.yml", val.ErrorMessage); } } diff --git a/server/Tingle.Dependabot.Tests/Samples/dependabot.yml b/server/Tingle.Dependabot.Tests/Samples/dependabot.yml index 02411b5a..f10d0673 100644 --- a/server/Tingle.Dependabot.Tests/Samples/dependabot.yml +++ b/server/Tingle.Dependabot.Tests/Samples/dependabot.yml @@ -18,6 +18,9 @@ updates: interval: 'daily' time: '03:15' open-pull-requests-limit: 10 + registries: + - reg1 + - reg2 insecure-external-code-execution: 'deny' ignore: - dependency-name: 'react' @@ -29,11 +32,11 @@ updates: - dependency-name: '@types/react-dom' update-types: ['version-update:semver-major'] registries: - tingle: + reg1: type: nuget-feed url: 'https://pkgs.dev.azure.com/dependabot/_packaging/dependabot/nuget/v3/index.json' token: ':${{DEFAULT_TOKEN}}' - tingle-npm: + reg2: type: npm-registry url: 'https://pkgs.dev.azure.com/dependabot/_packaging/dependabot-npm/npm/registry/' token: 'tingle-npm:${{DEFAULT_TOKEN}}' diff --git a/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs b/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs index 062f70bd..40f76bcc 100644 --- a/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs +++ b/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs @@ -27,7 +27,7 @@ public void MakeExtraCredentials_Works_1() .Build(); var configuration = deserializer.Deserialize(reader); - Assert.NotNull(configuration?.Registries); + Assert.NotNull(configuration); var registries = UpdateRunner.MakeExtraCredentials(configuration.Registries.Values, new Dictionary()); Assert.NotNull(registries); Assert.Equal(11, registries.Count); diff --git a/server/Tingle.Dependabot/AzureDevOpsEventHandler.cs b/server/Tingle.Dependabot/AzureDevOpsEventHandler.cs index 86e4b28e..22a07d3f 100644 --- a/server/Tingle.Dependabot/AzureDevOpsEventHandler.cs +++ b/server/Tingle.Dependabot/AzureDevOpsEventHandler.cs @@ -1,6 +1,4 @@ -using Microsoft.AspNetCore.Http.Json; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; +using Microsoft.EntityFrameworkCore; using System.Text.Json; using Tingle.Dependabot.Events; using Tingle.EventBus; diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs index 1f30d2fd..9348d3e6 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.Designer.cs @@ -72,6 +72,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("ProviderId") .HasColumnType("nvarchar(450)"); + b.Property("Registries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + b.Property("Slug") .HasColumnType("nvarchar(max)"); @@ -175,76 +179,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.Repository", b => - { - b.OwnsMany("Tingle.Dependabot.Models.DependabotRegistry", "Registries", b1 => - { - b1.Property("RepositoryId") - .HasColumnType("nvarchar(50)"); - - b1.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("int"); - - b1.Property("AuthKey") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "auth-key"); - - b1.Property("Key") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "key"); - - b1.Property("Organization") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "organization"); - - b1.Property("Password") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "password"); - - b1.Property("PublicKeyFingerprint") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "public-key-fingerprint"); - - b1.Property("ReplacesBase") - .HasColumnType("bit") - .HasAnnotation("Relational:JsonPropertyName", "replaces-base"); - - b1.Property("Repo") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "repo"); - - b1.Property("Token") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "token"); - - b1.Property("Type") - .IsRequired() - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "type"); - - b1.Property("Url") - .IsRequired() - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "url"); - - b1.Property("Username") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "username"); - - b1.HasKey("RepositoryId", "Id"); - - b1.ToTable("Repositories"); - - b1.ToJson("Registries"); - - b1.WithOwner() - .HasForeignKey("RepositoryId"); - }); - - b.Navigation("Registries"); - }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => { b.OwnsOne("Tingle.Dependabot.Models.UpdateJobResources", "Resources", b1 => diff --git a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs index db19a647..046f356c 100644 --- a/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs +++ b/server/Tingle.Dependabot/Migrations/20230824083425_InitialCreate.cs @@ -38,8 +38,8 @@ protected override void Up(MigrationBuilder migrationBuilder) ConfigFileContents = table.Column(type: "nvarchar(max)", nullable: false), SyncException = table.Column(type: "nvarchar(max)", nullable: true), Updates = table.Column(type: "nvarchar(max)", nullable: false), - Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true), - Registries = table.Column(type: "nvarchar(max)", nullable: true) + Registries = table.Column(type: "nvarchar(max)", nullable: false), + Etag = table.Column(type: "rowversion", rowVersion: true, nullable: true) }, constraints: table => { diff --git a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs index 3e2c94b3..b45f8aa5 100644 --- a/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs +++ b/server/Tingle.Dependabot/Migrations/MainDbContextModelSnapshot.cs @@ -69,6 +69,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("ProviderId") .HasColumnType("nvarchar(450)"); + b.Property("Registries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + b.Property("Slug") .HasColumnType("nvarchar(max)"); @@ -172,76 +176,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("UpdateJobs"); }); - modelBuilder.Entity("Tingle.Dependabot.Models.Repository", b => - { - b.OwnsMany("Tingle.Dependabot.Models.DependabotRegistry", "Registries", b1 => - { - b1.Property("RepositoryId") - .HasColumnType("nvarchar(50)"); - - b1.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("int"); - - b1.Property("AuthKey") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "auth-key"); - - b1.Property("Key") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "key"); - - b1.Property("Organization") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "organization"); - - b1.Property("Password") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "password"); - - b1.Property("PublicKeyFingerprint") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "public-key-fingerprint"); - - b1.Property("ReplacesBase") - .HasColumnType("bit") - .HasAnnotation("Relational:JsonPropertyName", "replaces-base"); - - b1.Property("Repo") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "repo"); - - b1.Property("Token") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "token"); - - b1.Property("Type") - .IsRequired() - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "type"); - - b1.Property("Url") - .IsRequired() - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "url"); - - b1.Property("Username") - .HasColumnType("nvarchar(max)") - .HasAnnotation("Relational:JsonPropertyName", "username"); - - b1.HasKey("RepositoryId", "Id"); - - b1.ToTable("Repositories"); - - b1.ToJson("Registries"); - - b1.WithOwner() - .HasForeignKey("RepositoryId"); - }); - - b.Navigation("Registries"); - }); - modelBuilder.Entity("Tingle.Dependabot.Models.UpdateJob", b => { b.OwnsOne("Tingle.Dependabot.Models.UpdateJobResources", "Resources", b1 => diff --git a/server/Tingle.Dependabot/Models/DependabotConfiguration.cs b/server/Tingle.Dependabot/Models/DependabotConfiguration.cs index 4a8e8a57..e3a3690e 100644 --- a/server/Tingle.Dependabot/Models/DependabotConfiguration.cs +++ b/server/Tingle.Dependabot/Models/DependabotConfiguration.cs @@ -3,7 +3,7 @@ namespace Tingle.Dependabot.Models; -public class DependabotConfiguration +public class DependabotConfiguration : IValidatableObject { [Required, AllowedValues(2)] [JsonPropertyName("version")] @@ -14,7 +14,28 @@ public class DependabotConfiguration public List? Updates { get; set; } [JsonPropertyName("registries")] - public Dictionary? Registries { get; set; } + public Dictionary Registries { get; set; } = new(); + + public IEnumerable Validate(ValidationContext validationContext) + { + var updates = Updates ?? new(); + var configured = Registries.Keys; + var referenced = updates.SelectMany(r => r.Registries ?? new()).ToList(); + + // ensure there are no configured registries that have not been referenced + var missingConfiguration = referenced.Except(configured).ToList(); + if (missingConfiguration.Count > 0) + { + yield return new ValidationResult($"Referenced registries: '{string.Join(",", missingConfiguration)}' have not been configured in the root of dependabot.yml"); ; + } + + // ensure there are no registries referenced but not configured + var missingReferences = configured.Except(referenced).ToList(); + if (missingReferences.Count > 0) + { + yield return new ValidationResult($"Registries: '{string.Join(",", missingReferences)}' have not been referenced by any update"); + } + } } public record DependabotUpdate @@ -36,6 +57,9 @@ public record DependabotUpdate [JsonPropertyName("open-pull-requests-limit")] public int? OpenPullRequestsLimit { get; set; } = 5; + [JsonPropertyName("registries")] + public List? Registries { get; set; } + [JsonPropertyName("allow")] public List? Allow { get; set; } [JsonPropertyName("labels")] diff --git a/server/Tingle.Dependabot/Models/MainDbContext.cs b/server/Tingle.Dependabot/Models/MainDbContext.cs index 12cff2f6..77300a1a 100644 --- a/server/Tingle.Dependabot/Models/MainDbContext.cs +++ b/server/Tingle.Dependabot/Models/MainDbContext.cs @@ -1,5 +1,9 @@ using Microsoft.AspNetCore.DataProtection.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using System.Text.Json; namespace Tingle.Dependabot.Models; @@ -19,7 +23,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity(b => { b.Property(r => r.Updates).HasJsonConversion(); - b.OwnsMany(r => r.Registries).ToJson(); + HasJsonConversion(b.Property(r => r.Registries)); b.HasIndex(r => r.Created).IsDescending(); // faster filtering b.HasIndex(r => r.ProviderId).IsUnique(); @@ -38,4 +42,29 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.OwnsOne(j => j.Resources); }); } + + static PropertyBuilder HasJsonConversion(PropertyBuilder propertyBuilder, JsonSerializerOptions? serializerOptions = null) + { + ArgumentNullException.ThrowIfNull(propertyBuilder); + +#pragma warning disable CS8603 // Possible null reference return. + var converter = new ValueConverter( + convertToProviderExpression: v => ConvertToJson(v, serializerOptions), + convertFromProviderExpression: v => ConvertFromJson(v, serializerOptions)); + + var comparer = new ValueComparer( + equalsExpression: (l, r) => ConvertToJson(l, serializerOptions) == ConvertToJson(r, serializerOptions), + hashCodeExpression: v => v == null ? 0 : ConvertToJson(v, serializerOptions).GetHashCode(), + snapshotExpression: v => ConvertFromJson(ConvertToJson(v, serializerOptions), serializerOptions)); +#pragma warning restore CS8603 // Possible null reference return. + + propertyBuilder.HasConversion(converter); + propertyBuilder.Metadata.SetValueConverter(converter); + propertyBuilder.Metadata.SetValueComparer(comparer); + + return propertyBuilder; + } + + private static string ConvertToJson(T value, JsonSerializerOptions? serializerOptions) => JsonSerializer.Serialize(value, serializerOptions); + private static T? ConvertFromJson(string? value, JsonSerializerOptions? serializerOptions) => value is null ? default : JsonSerializer.Deserialize(value, serializerOptions); } diff --git a/server/Tingle.Dependabot/Models/Repository.cs b/server/Tingle.Dependabot/Models/Repository.cs index 586f0dca..678af7c6 100644 --- a/server/Tingle.Dependabot/Models/Repository.cs +++ b/server/Tingle.Dependabot/Models/Repository.cs @@ -48,7 +48,7 @@ public class Repository /// When null or empty, there was a parsing exception. /// [JsonIgnore] // only for internal use - public List Registries { get; set; } = new List(); + public Dictionary Registries { get; set; } = new(); [Timestamp] public byte[]? Etag { get; set; } diff --git a/server/Tingle.Dependabot/Workflow/Synchronizer.cs b/server/Tingle.Dependabot/Workflow/Synchronizer.cs index d752337f..bc34f5ad 100644 --- a/server/Tingle.Dependabot/Workflow/Synchronizer.cs +++ b/server/Tingle.Dependabot/Workflow/Synchronizer.cs @@ -199,7 +199,7 @@ internal async Task SynchronizeAsync(Repository? repository, RecursiveValidator.ValidateObjectRecursive(configuration); // set the registries - repository.Registries = configuration.Registries?.Values.ToList() ?? new List(); + repository.Registries = configuration.Registries; // set the updates a fresh var updates = configuration.Updates!; diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 376d1744..a9749430 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -237,7 +237,8 @@ internal IDictionary CreateVariables(Repository repository, Repo .AddIfNotDefault("AZURE_AUTO_APPROVE_PR", (options.AutoApprove ?? false).ToString().ToLowerInvariant()); // Add extra credentials with replaced secrets - values.AddIfNotDefault("DEPENDABOT_EXTRA_CREDENTIALS", ToJson(MakeExtraCredentials(repository.Registries, secrets))); + var registries = update.Registries?.Select(r => repository.Registries[r]).ToList(); + values.AddIfNotDefault("DEPENDABOT_EXTRA_CREDENTIALS", ToJson(MakeExtraCredentials(registries, secrets))); return values; }