Request non-concurrent target workspace if engine is non-concurrent (#339)

This optimizes workspace allocation on GitLab when the engine being used doesn't support or allow concurrency. It also adds a new environment variable UET_USE_EXCLUSIVE_NODES_FOR_NONCONCURRENT_ENGINE which will request tags 'buildgraph-exclusive-windows' and 'buildgraph-exclusive-mac' (GitLab runners with this tag should be configured to only run one job at a time per node). This ensures that our concurrent GitLab runners don't idle waiting for an engine or target workspace to unlock due to non-concurrency.

Author: hach-que

UET/Redpoint.Uet.BuildPipeline.Executors.GitLab/GitLabBuildExecutor.cs

@@ -34,6 +34,7 @@ protected override async Task ExecuteBuildServerSpecificPipelineAsync(
             BuildSpecification buildSpecification,
             BuildServerPipeline buildServerPipeline)
         {
+            ArgumentNullException.ThrowIfNull(buildSpecification);
             ArgumentNullException.ThrowIfNull(buildServerPipeline);
 
             if (string.IsNullOrWhiteSpace(_buildServerOutputFilePath))
@@ -67,11 +68,27 @@ protected override async Task ExecuteBuildServerSpecificPipelineAsync(
 
                 if (sourceJob.Agent.Platform == BuildServerJobPlatform.Windows)
                 {
-                    job.Tags = new List<string> { "buildgraph-windows" };
+                    if (buildSpecification.Engine.IsNonConcurrent &&
+                        System.Environment.GetEnvironmentVariable("UET_USE_EXCLUSIVE_NODES_FOR_NONCONCURRENT_ENGINE") == "1")
+                    {
+                        job.Tags = new List<string> { "buildgraph-exclusive-windows" };
+                    }
+                    else
+                    {
+                        job.Tags = new List<string> { "buildgraph-windows" };
+                    }
                 }
                 else if (sourceJob.Agent.Platform == BuildServerJobPlatform.Mac)
                 {
-                    job.Tags = new List<string> { "buildgraph-mac" };
+                    if (buildSpecification.Engine.IsNonConcurrent &&
+                        System.Environment.GetEnvironmentVariable("UET_USE_EXCLUSIVE_NODES_FOR_NONCONCURRENT_ENGINE") == "1")
+                    {
+                        job.Tags = new List<string> { "buildgraph-exclusive-mac" };
+                    }
+                    else
+                    {
+                        job.Tags = new List<string> { "buildgraph-mac" };
+                    }
                 }
                 else if (sourceJob.Agent.Platform == BuildServerJobPlatform.Meta)
                 {

UET/Redpoint.Uet.BuildPipeline.Executors.GitLab/GitLabBuildNodeExecutor.cs

@@ -16,6 +16,7 @@
     using System.Threading.Tasks;
     using Redpoint.Concurrency;
     using Redpoint.Uet.BuildPipeline.BuildGraph.PreBuild;
+    using System.Collections.Specialized;
 
     public class GitLabBuildNodeExecutor : IBuildNodeExecutor
     {
@@ -60,6 +61,42 @@ private class NodeNameExecutionState
             public string? NodeName;
         }
 
+        private async Task<IWorkspace> GetGitWorkspaceAsync(
+            string repository,
+            string commit,
+            string branch,
+            IReadOnlyList<string> nodeNames,
+            BuildSpecification buildSpecification,
+            CancellationToken cancellationToken)
+        {
+            NameValueCollection? queryString = null;
+            if (buildSpecification.Engine.IsNonConcurrent)
+            {
+                _logger.LogInformation("Requesting non-concurrent Git workspace, because the engine is non-concurrent.");
+                queryString = new NameValueCollection
+                {
+                    { "concurrent", "false" }
+                };
+            }
+
+            return await _workspaceProvider.GetWorkspaceAsync(
+                new GitWorkspaceDescriptor
+                {
+                    RepositoryUrl = repository,
+                    RepositoryCommitOrRef = commit,
+                    RepositoryBranchForReservationParameters = branch,
+                    AdditionalFolderLayers = Array.Empty<string>(),
+                    AdditionalFolderZips = Array.Empty<string>(),
+                    WorkspaceDisambiguators = nodeNames,
+                    ProjectFolderName = buildSpecification.ProjectFolderName,
+                    BuildType = GitWorkspaceDescriptorBuildType.Generic,
+                    WindowsSharedGitCachePath = null,
+                    MacSharedGitCachePath = null,
+                    QueryString = queryString,
+                },
+                cancellationToken).ConfigureAwait(false);
+        }
+
         public async Task<int> ExecuteBuildNodesAsync(
             BuildSpecification buildSpecification,
             BuildConfigDynamic<BuildConfigPluginDistribution, IPrepareProvider>[]? preparePlugin,
@@ -247,20 +284,12 @@ async Task<int> ExecuteNodesInWorkspaceAsync(
                     _logger.LogTrace($"Executing build with engine build type of 'CurrentWorkspace', obtaining single workspace and using it as the engine directory as well.");
 
                     _logger.LogTrace($"Obtaining workspace for build.");
-                    await using ((await _workspaceProvider.GetWorkspaceAsync(
-                        new GitWorkspaceDescriptor
-                        {
-                            RepositoryUrl = repository,
-                            RepositoryCommitOrRef = commit,
-                            RepositoryBranchForReservationParameters = branch,
-                            AdditionalFolderLayers = Array.Empty<string>(),
-                            AdditionalFolderZips = Array.Empty<string>(),
-                            WorkspaceDisambiguators = nodeNames,
-                            ProjectFolderName = buildSpecification.ProjectFolderName,
-                            BuildType = GitWorkspaceDescriptorBuildType.Generic,
-                            WindowsSharedGitCachePath = null,
-                            MacSharedGitCachePath = null,
-                        },
+                    await using ((await GetGitWorkspaceAsync(
+                        repository,
+                        commit,
+                        branch,
+                        nodeNames,
+                        buildSpecification,
                         cancellationToken).ConfigureAwait(false)).AsAsyncDisposable(out var targetWorkspace).ConfigureAwait(false))
                     {
                         _logger.LogTrace($"Calling ExecuteNodesInWorkspaceAsync inside allocated workspace.");
@@ -291,20 +320,12 @@ async Task<int> ExecuteNodesInWorkspaceAsync(
                             _logger.LogTrace($"Executing build with engine build type of 'None', obtained engine workspace and obtaining target workspace separately.");
 
                             _logger.LogTrace($"Obtaining workspace for build.");
-                            await using ((await _workspaceProvider.GetWorkspaceAsync(
-                                new GitWorkspaceDescriptor
-                                {
-                                    RepositoryUrl = repository,
-                                    RepositoryCommitOrRef = commit,
-                                    RepositoryBranchForReservationParameters = branch,
-                                    AdditionalFolderLayers = Array.Empty<string>(),
-                                    AdditionalFolderZips = Array.Empty<string>(),
-                                    WorkspaceDisambiguators = nodeNames,
-                                    ProjectFolderName = buildSpecification.ProjectFolderName,
-                                    BuildType = GitWorkspaceDescriptorBuildType.Generic,
-                                    WindowsSharedGitCachePath = null,
-                                    MacSharedGitCachePath = null,
-                                },
+                            await using ((await GetGitWorkspaceAsync(
+                                repository,
+                                commit,
+                                branch,
+                                nodeNames,
+                                buildSpecification,
                                 cancellationToken).ConfigureAwait(false)).AsAsyncDisposable(out var targetWorkspace).ConfigureAwait(false))
                             {
                                 _logger.LogTrace($"Calling ExecuteNodesInWorkspaceAsync inside allocated workspace.");

UET/Redpoint.Uet.BuildPipeline/Executors/BuildEngineSpecification.cs

@@ -172,5 +172,32 @@ public string ToReparsableString()
                 throw new NotSupportedException();
             }
         }
+
+        public bool IsNonConcurrent
+        {
+            get
+            {
+                if (!string.IsNullOrWhiteSpace(_engineVersion))
+                {
+                    return true;
+                }
+
+                if (!string.IsNullOrWhiteSpace(_enginePath))
+                {
+                    return true;
+                }
+
+                if (!string.IsNullOrWhiteSpace(_gitUrl))
+                {
+                    if (_queryString != null &&
+                        _queryString["concurrent"] == "false")
+                    {
+                        return true;
+                    }
+                }
+
+                return false;
+            }
+        }
     }
 }