Preliminary new updater setup (#794)

* Setup pulling of files from dependabot updater
* Update ENV variables passed to the updater to match the official one

copy-updater-files.ps1

@@ -0,0 +1,20 @@
+Param(
+    [string] $tag = "v0.230.0"
+)
+
+$hash = [ordered]@{
+    "updater/lib/dependabot/environment.rb"       = "lib/dependabot/environment.rb"
+    "updater/spec/dependabot/environment_spec.rb" = "spec/dependabot/environment_spec.rb"
+    # "updater/spec/spec_helper.rb"                 = "spec/spec_helper.rb"
+}
+
+$baseUrl = "https://raw.githubusercontent.com/dependabot/dependabot-core"
+$destinationFolder = Join-Path -Path '.' -ChildPath 'updater'
+
+foreach ($h in $hash.GetEnumerator()) {
+    $sourceUrl = "$baseUrl/$tag/$($h.Name)"
+    $destinationPath = Join-Path -Path "$destinationFolder" -ChildPath "$($h.Value)"
+    Write-Host "`Downloading $($h.Name) ..."
+    [System.IO.Directory]::CreateDirectory("$(Split-Path -Path "$destinationPath")") | Out-Null
+    Invoke-WebRequest -Uri $sourceUrl -OutFile $destinationPath
+}

server/Tingle.Dependabot/Models/UpdateJobResponse.cs

@@ -58,6 +58,9 @@ public UpdateJobAttributes(UpdateJob job) : this()
 
     [JsonPropertyName("security-updates-only")]
     public bool? SecurityUpdatesOnly { get; set; }
+
+    [JsonPropertyName("debug")]
+    public bool? Debug { get; set; }
 }
 
 public sealed record UpdateJobAttributesSource()

server/Tingle.Dependabot/Workflow/UpdateRunner.cs

@@ -54,15 +54,20 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up
         catch (Azure.RequestFailedException rfe) when (rfe.Status is 404) { }
 
         // prepare the container
+        var fileShareName = options.FileShareName;
+        var volumeName = "working-dir";
         var image = options.UpdaterContainerImageTemplate!.Replace("{{ecosystem}}", job.PackageEcosystem);
         var container = new ContainerInstanceContainer(UpdaterContainerName, image, new(job.Resources!));
         var env = CreateVariables(repository, update, job);
         foreach (var (key, value) in env) container.EnvironmentVariables.Add(new ContainerEnvironmentVariable(key) { Value = value, });
 
-        // prepare the container command/entrypoint (this is what seems to work)
+        // set the container command/entrypoint (this is what seems to work)
         container.Command.Add("/bin/bash");
         container.Command.Add("bin/run.sh");
         container.Command.Add("update-script");
+
+        // add volume mounts
+        container.VolumeMounts.Add(new ContainerVolumeMount(volumeName, "/mnt/dependabot"));
 
         // prepare the container group
         var data = new ContainerGroupData(options.Location!, new[] { container, }, ContainerInstanceOperatingSystemType.Linux)
@@ -81,6 +86,12 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up
             data.ImageRegistryCredentials.Add(new ContainerGroupImageRegistryCredential(registry) { Identity = options.ManagedIdentityId, });
         }
 
+        // add volumes
+        data.Volumes.Add(new ContainerVolume(volumeName)
+        {
+            AzureFile = new(fileShareName, options.StorageAccountName) { StorageAccountKey = options.StorageAccountKey, },
+        });
+
         // add tags to the data for tracing purposes
         data.Tags["purpose"] = "dependabot";
         data.Tags.AddIfNotDefault("ecosystem", job.PackageEcosystem)
@@ -130,6 +141,13 @@ public async Task DeleteAsync(UpdateJob job, CancellationToken cancellationToken
 
             // there is no state for jobs that are running
             if (status is UpdateJobStatus.Running) return null;
+
+            // delete the job directory f it exists
+            var jobDirectory = Path.Join(options.WorkingDirectory, job.Id);
+            if (Directory.Exists(jobDirectory))
+            {
+                Directory.Delete(jobDirectory);
+            }
 
             // get the period
             var currentState = resource.Data.Containers.Single(c => c.Name == UpdaterContainerName).InstanceView?.CurrentState;
@@ -193,15 +211,27 @@ internal IDictionary<string, string> CreateVariables(Repository repository, Repo
     {
         static string? ToJson<T>(T? entries) => entries is null ? null : JsonSerializer.Serialize(entries, serializerOptions); // null ensures we do not add to the values
 
+        var jobDirectory = Path.Join(options.WorkingDirectory, job.Id);
+
         // Add compulsory values
         var values = new Dictionary<string, string>
         {
+            ["DEPENDABOT_JOB_ID"] = job.Id!,
+            ["DEPENDABOT_JOB_TOKEN"] = job.AuthKey!,
+            ["DEPENDABOT_JOB_PATH"] = Path.Join(jobDirectory, "job.json"),
+            ["DEPENDABOT_OUTPUT_PATH"] = Path.Join(jobDirectory, "output"),
+
             ["DEPENDABOT_PACKAGE_MANAGER"] = job.PackageEcosystem!,
             ["DEPENDABOT_DIRECTORY"] = update.Directory!,
             ["DEPENDABOT_OPEN_PULL_REQUESTS_LIMIT"] = update.OpenPullRequestsLimit!.Value.ToString(),
         };
 
         // Add optional values
+        values.AddIfNotDefault("DEPENDABOT_DEBUG", options.DebugJobs?.ToString().ToLower())
+              .AddIfNotDefault("DEPENDABOT_API_URL", options.JobsApiUrl)
+              .AddIfNotDefault("DEPENDABOT_REPO_CONTENTS_PATH", Path.Join(jobDirectory, "repo"))
+              .AddIfNotDefault("UPDATER_DETERMINISTIC", options.DeterministicUpdates?.ToString().ToLower());
+
         values.AddIfNotDefault("GITHUB_ACCESS_TOKEN", options.GithubToken)
               .AddIfNotDefault("DEPENDABOT_REBASE_STRATEGY", update.RebaseStrategy)
               .AddIfNotDefault("DEPENDABOT_TARGET_BRANCH", update.TargetBranch)

server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs

@@ -63,11 +63,31 @@ public ValidateOptionsResult Validate(string? name, WorkflowOptions options)
             return ValidateOptionsResult.Fail($"'{nameof(options.ManagedIdentityId)}' cannot be null or whitespace");
         }
 
+        if (string.IsNullOrWhiteSpace(options.WorkingDirectory))
+        {
+            return ValidateOptionsResult.Fail($"'{nameof(options.WorkingDirectory)}' cannot be null or whitespace");
+        }
+
         if (string.IsNullOrWhiteSpace(options.Location))
         {
             return ValidateOptionsResult.Fail($"'{nameof(options.Location)}' cannot be null or whitespace");
         }
 
+        if (string.IsNullOrWhiteSpace(options.StorageAccountName))
+        {
+            return ValidateOptionsResult.Fail($"'{nameof(options.StorageAccountName)}' cannot be null or whitespace");
+        }
+
+        if (string.IsNullOrWhiteSpace(options.StorageAccountKey))
+        {
+            return ValidateOptionsResult.Fail($"'{nameof(options.StorageAccountKey)}' cannot be null or whitespace");
+        }
+
+        if (string.IsNullOrWhiteSpace(options.FileShareName))
+        {
+            return ValidateOptionsResult.Fail($"'{nameof(options.FileShareName)}' cannot be null or whitespace");
+        }
+
         return ValidateOptionsResult.Success;
     }
 }

server/Tingle.Dependabot/Workflow/WorkflowOptions.cs

@@ -51,6 +51,27 @@ public class WorkflowOptions
     /// <summary>Authentication token for accessing the project.</summary>
     public string? ProjectToken { get; set; }
 
+    /// <summary>Whether to debug all jobs.</summary>
+    public bool? DebugJobs { get; set; }
+
+    /// <summary>URL on which to access the API from the jobs.</summary>
+    /// <example>https://dependabot.dummy-123.westeurope.azurecontainerapps.io</example>
+    public string? JobsApiUrl { get; set; }
+
+    /// <summary>
+    /// Root working directory where file are written during job scheduling and execution.
+    /// This directory is the root for all jobs.
+    /// Subdirectories are created for each job and further for each usage type.
+    /// For example, if this value is set to <c>/mnt/dependabot</c>,
+    /// A job identified as <c>123456789</c> will have files written at <c>/mnt/dependabot/123456789</c>
+    /// and some nested directories in it such as <c>/mnt/dependabot/123456789/repo</c>.
+    /// </summary>
+    /// <example>/mnt/dependabot</example>
+    public string? WorkingDirectory { get; set; }
+
+    /// <summary>Whether updates should be created in the same order.</summary>
+    public bool? DeterministicUpdates { get; set; }
+
     /// <summary>Whether update jobs should fail when an exception occurs.</summary>
     public bool FailOnException { get; set; }
 
@@ -82,6 +103,17 @@ public class WorkflowOptions
     /// <summary>Location/region where to create new update jobs.</summary>
     public string? Location { get; set; } // using Azure.Core.Location does not work when binding from IConfiguration
 
+    /// <summary>Name of the storage account.</summary>
+    /// <example>dependabot-1234567890</example>
+    public string? StorageAccountName { get; set; } // only used with ContainerInstances
+
+    /// <summary>Access key for the storage account.</summary>
+    public string? StorageAccountKey { get; set; } // only used with ContainerInstances
+
+    /// <summary>Name of the file share for the working directory</summary>
+    /// <example>working-dir</example>
+    public string? FileShareName { get; set; } // only used with ContainerInstances
+
     /// <summary>
     /// Possible/allowed paths for the configuration files in a repository.
     /// </summary>

server/Tingle.Dependabot/appsettings.json

@@ -67,7 +67,11 @@
     "UpdaterContainerImageTemplate": "ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:1.20.0-ci.37",
     "ProjectUrl": "https://dev.azure.com/fabrikam/DefaultCollection",
     "ProjectToken": "<my-pat-here>",
+    "WorkingDirectory": "work",
     "GithubToken": "<github-pat-here>",
-    "Location": "westeurope"
+    "Location": "westeurope",
+    "StorageAccountName": "dependabot-1234567890",
+    "StorageAccountKey": "<key-here>",
+    "FileShareName": "working-dir"
   }
 }

server/main.bicep

@@ -16,6 +16,9 @@ param synchronizeOnStartup bool = false
 @description('Whether to create or update subscriptions on startup.')
 param createOrUpdateWebhooksOnStartup bool = false
 
+@description('Whether to debug all jobs.')
+param debugAllJobs bool = false
+
 @description('Access token for authenticating requests to GitHub.')
 param githubToken string = ''
 
@@ -64,6 +67,7 @@ var sqlServerAdministratorLogin = uniqueString(resourceGroup().id) // e.g. zecnx
 var sqlServerAdministratorLoginPassword = '${skip(uniqueString(resourceGroup().id), 5)}%${uniqueString('sql-password', resourceGroup().id)}' // e.g. abcde%zecnx476et7xm (19 characters)
 // avoid conflicts across multiple deployments for resources that generate FQDN based on the name
 var collisionSuffix = uniqueString(resourceGroup().id) // e.g. zecnx476et7xm (13 characters)
+var fileShareName = 'working-dir'
 
 /* Managed Identities */
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
@@ -101,6 +105,12 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = {
       defaultAction: 'Allow'
     }
   }
+
+  resource fileServices 'fileServices' existing = {
+    name: 'default'
+
+    resource workingDir 'shares' = { name: fileShareName }
+  }
 }
 
 /* SQL Server */
@@ -168,7 +178,7 @@ resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10
 }
 
 /* Container App Environment */
-resource appEnvironment 'Microsoft.App/managedEnvironments@2022-10-01' = {
+resource appEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = {
   name: name
   location: location
   properties: {
@@ -180,6 +190,18 @@ resource appEnvironment 'Microsoft.App/managedEnvironments@2022-10-01' = {
       }
     }
   }
+
+  resource workingDir 'storages' = {
+    name: fileShareName
+    properties: {
+      azureFile: {
+        accessMode: 'ReadWrite'
+        shareName: fileShareName
+        accountName: storageAccount.name
+        accountKey: storageAccount.listKeys().keys[0].value
+      }
+    }
+  }
 }
 
 /* Application Insights */
@@ -194,7 +216,7 @@ resource appInsights 'Microsoft.Insights/components@2020-02-02' = {
 }
 
 /* Container App */
-resource app 'Microsoft.App/containerApps@2022-10-01' = {
+resource app 'Microsoft.App/containerApps@2023-05-01' = {
   name: name
   location: location
   properties: {
@@ -232,13 +254,18 @@ resource app 'Microsoft.App/containerApps@2022-10-01' = {
           name: 'log-analytics-workspace-key'
           value: logAnalyticsWorkspace.listKeys().primarySharedKey
         }
+        {
+          name: 'storage-account-key'
+          value: storageAccount.listKeys().keys[0].value
+        }
       ]
     }
     template: {
       containers: [
         {
           image: 'ghcr.io/tinglesoftware/dependabot-server:${imageTag}'
           name: 'dependabot'
+          volumeMounts: [ { mountPath: '/mnt/dependabot', volumeName: fileShareName } ]
           env: [
             { name: 'AZURE_CLIENT_ID', value: managedIdentity.properties.clientId } // Specifies the User-Assigned Managed Identity to use. Without this, the app attempt to use the system assigned one.
             { name: 'ASPNETCORE_FORWARDEDHEADERS_ENABLED', value: 'true' } // Application is behind proxy
@@ -252,6 +279,9 @@ resource app 'Microsoft.App/containerApps@2022-10-01' = {
             { name: 'Workflow__CreateOrUpdateWebhooksOnStartup', value: createOrUpdateWebhooksOnStartup ? 'true' : 'false' }
             { name: 'Workflow__ProjectUrl', value: projectUrl }
             { name: 'Workflow__ProjectToken', secretRef: 'project-token' }
+            { name: 'Workflow__DebugJobs', value: '${debugAllJobs}' }
+            { name: 'Workflow__JobsApiUrl', value: 'https://${name}.${appEnvironment.properties.defaultDomain}' }
+            { name: 'Workflow__WorkingDirectory', value: '/mnt/dependabot' }
             {
               name: 'Workflow__WebhookEndpoint'
               value: 'https://${name}.${appEnvironment.properties.defaultDomain}/webhooks/azure'
@@ -276,6 +306,9 @@ resource app 'Microsoft.App/containerApps@2022-10-01' = {
             { name: 'Workflow__AutoApprove', value: autoApprove ? 'true' : 'false' }
             { name: 'Workflow__GithubToken', value: githubToken }
             { name: 'Workflow__Location', value: location }
+            { name: 'Workflow__StorageAccountName', value: storageAccount.name }
+            { name: 'Workflow__StorageAccountKey', secretRef: 'storage-account-key' }
+            { name: 'Workflow__FileShareName', value: fileShareName }
 
             {
               name: 'Authentication__Schemes__Management__Authority'
@@ -311,6 +344,7 @@ resource app 'Microsoft.App/containerApps@2022-10-01' = {
           ]
         }
       ]
+      volumes: [ { name: fileShareName, storageName: fileShareName, storageType: 'AzureFile' } ]
       scale: {
         minReplicas: minReplicas
         maxReplicas: maxReplicas