diff --git a/src/Agent.Listener/CommandLine/ConfigureAgent.cs b/src/Agent.Listener/CommandLine/ConfigureAgent.cs index c9b772fdb7..1f8439dc36 100644 --- a/src/Agent.Listener/CommandLine/ConfigureAgent.cs +++ b/src/Agent.Listener/CommandLine/ConfigureAgent.cs @@ -60,6 +60,9 @@ public class ConfigureAgent : ConfigureOrRemoveBase [Option(Constants.Agent.CommandLine.Flags.DisableLogUploads)] public bool DisableLogUploads { get; set; } + [Option(Constants.Agent.CommandLine.Flags.ReStreamLogsToFiles)] + public bool ReStreamLogsToFiles { get; set; } + [Option(Constants.Agent.CommandLine.Flags.MachineGroup)] public bool MachineGroup { get; set; } diff --git a/src/Agent.Listener/CommandSettings.cs b/src/Agent.Listener/CommandSettings.cs index 0407937c37..c2e75952f7 100644 --- a/src/Agent.Listener/CommandSettings.cs +++ b/src/Agent.Listener/CommandSettings.cs @@ -571,6 +571,11 @@ public bool GetDisableLogUploads() return TestFlag(Configure?.DisableLogUploads, Constants.Agent.CommandLine.Flags.DisableLogUploads); } + public bool GetReStreamLogsToFiles() + { + return TestFlag(Configure?.ReStreamLogsToFiles, Constants.Agent.CommandLine.Flags.ReStreamLogsToFiles); + } + public bool Unattended() { if (TestFlag(GetConfigureOrRemoveBase()?.Unattended, Constants.Agent.CommandLine.Flags.Unattended)) diff --git a/src/Agent.Listener/Configuration/ConfigurationManager.cs b/src/Agent.Listener/Configuration/ConfigurationManager.cs index e2f7b3c2cf..7bb61ae64e 100644 --- a/src/Agent.Listener/Configuration/ConfigurationManager.cs +++ b/src/Agent.Listener/Configuration/ConfigurationManager.cs @@ -398,6 +398,12 @@ public async Task ConfigureAsync(CommandSettings command) agentSettings.NotificationSocketAddress = command.GetNotificationSocketAddress(); agentSettings.DisableLogUploads = command.GetDisableLogUploads(); + agentSettings.ReStreamLogsToFiles = command.GetReStreamLogsToFiles(); + + if (agentSettings.DisableLogUploads && agentSettings.ReStreamLogsToFiles) + { + throw new NotSupportedException(StringUtil.Loc("ReStreamLogsToFilesError")); + } agentSettings.AlwaysExtractTask = command.GetAlwaysExtractTask(); @@ -736,7 +742,7 @@ private void CheckAgentRootDirectorySecure() // Get info about root folder DirectoryInfo dirInfo = new DirectoryInfo(rootDirPath); - // Get directory access control list + // Get directory access control list DirectorySecurity directorySecurityInfo = dirInfo.GetAccessControl(); AuthorizationRuleCollection dirAccessRules = directorySecurityInfo.GetAccessRules(true, true, typeof(NTAccount)); diff --git a/src/Agent.Worker/ExecutionContext.cs b/src/Agent.Worker/ExecutionContext.cs index 571c512e17..d38702e216 100644 --- a/src/Agent.Worker/ExecutionContext.cs +++ b/src/Agent.Worker/ExecutionContext.cs @@ -117,7 +117,7 @@ public sealed class ExecutionContext : AgentService, IExecutionContext, IDisposa private bool _throttlingReported = false; private ExecutionTargetInfo _defaultStepTarget; private ExecutionTargetInfo _currentStepTarget; - private bool _disableLogUploads; + private LogsStreamingOptions _logsStreamingOptions; private string _buildLogsFolderPath; private string _buildLogsFile; private FileStream _buildLogsData; @@ -179,9 +179,21 @@ public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); - _disableLogUploads = HostContext.GetService().GetSettings().DisableLogUploads; + var agentSettings = HostContext.GetService().GetSettings(); - if (_disableLogUploads) + + _logsStreamingOptions = LogsStreamingOptions.StreamToServer; + if (agentSettings.ReStreamLogsToFiles) + { + _logsStreamingOptions |= LogsStreamingOptions.StreamToFiles; + } + else if (agentSettings.DisableLogUploads) + { + _logsStreamingOptions = LogsStreamingOptions.StreamToFiles; + } + Trace.Info($"Logs streaming mode: {_logsStreamingOptions}"); + + if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles)) { _buildLogsFolderPath = Path.Combine(hostContext.GetDiagDirectory(), _buildLogsFolderName); Directory.CreateDirectory(_buildLogsFolderPath); @@ -264,7 +276,7 @@ public void Start(string currentOperation = null) _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); - if (_disableLogUploads) + if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles)) { var buildLogsJobFolder = Path.Combine(_buildLogsFolderPath, _mainTimelineId.ToString()); Directory.CreateDirectory(buildLogsJobFolder); @@ -276,7 +288,14 @@ public void Start(string currentOperation = null) _buildLogsData = new FileStream(_buildLogsFile, FileMode.CreateNew); _buildLogsWriter = new StreamWriter(_buildLogsData, System.Text.Encoding.UTF8); - _logger.Write(StringUtil.Loc("BuildLogsMessage", _buildLogsFile)); + if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToServerAndFiles)) + { + _logger.Write(StringUtil.Loc("LogOutputMessage", _buildLogsFile)); + } + else + { + _logger.Write(StringUtil.Loc("BuildLogsMessage", _buildLogsFile)); + } } } @@ -287,7 +306,7 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = Result = result; } - if (_disableLogUploads) + if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles)) { _buildLogsWriter.Flush(); _buildLogsData.Flush(); @@ -717,21 +736,21 @@ public long Write(string tag, string inputMessage, bool canMaskSecrets = true) { totalLines = _logger.TotalLines + 1; - if (_disableLogUploads) + if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToServer)) { - _buildLogsWriter.WriteLine(message); + _logger.Write(message); } - else + if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles)) { - _logger.Write(message); + //Add date time stamp to log line + _buildLogsWriter.WriteLine("{0:O} {1}", DateTime.UtcNow, message); } } - if (!_disableLogUploads) + if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToServer)) { // write to job level execution context's log file. - var parentContext = _parentExecutionContext as ExecutionContext; - if (parentContext != null) + if (_parentExecutionContext is ExecutionContext parentContext) { lock (parentContext._loggerLock) { @@ -951,7 +970,7 @@ private void PublishTelemetry( publishTelemetryCmd.ProcessCommand(this, cmd); } - public void PublishTaskRunnerTelemetry(Dictionary taskRunnerData) + public void PublishTaskRunnerTelemetry(Dictionary taskRunnerData) { PublishTelemetry(taskRunnerData, IsAgentTelemetry: true); } @@ -966,6 +985,15 @@ public void Dispose() _buildLogsData?.Dispose(); _buildLogsData = null; } + + [Flags] + private enum LogsStreamingOptions + { + None = 0, + StreamToServer = 1, + StreamToFiles = 2, + StreamToServerAndFiles = StreamToServer | StreamToFiles + } } // The Error/Warning/etc methods are created as extension methods to simplify unit testing. diff --git a/src/Microsoft.VisualStudio.Services.Agent/ConfigurationStore.cs b/src/Microsoft.VisualStudio.Services.Agent/ConfigurationStore.cs index 92d6f6c2a8..5001b0818e 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/ConfigurationStore.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/ConfigurationStore.cs @@ -89,6 +89,9 @@ public string Fingerprint [DataMember(EmitDefaultValue = false)] public bool DisableLogUploads { get; set; } + [DataMember(EmitDefaultValue = false)] + public bool ReStreamLogsToFiles { get; set; } + [DataMember(EmitDefaultValue = false)] public int PoolId { get; set; } diff --git a/src/Microsoft.VisualStudio.Services.Agent/Constants.cs b/src/Microsoft.VisualStudio.Services.Agent/Constants.cs index 2842370879..970b54e939 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/Constants.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/Constants.cs @@ -220,6 +220,7 @@ public static class Flags public const string GitUseSChannel = "gituseschannel"; public const string Help = "help"; public const string DisableLogUploads = "disableloguploads"; + public const string ReStreamLogsToFiles = "restreamlogstofiles"; public const string MachineGroup = "machinegroup"; public const string Replace = "replace"; public const string NoRestart = "norestart"; diff --git a/src/Misc/layoutbin/en-US/strings.json b/src/Misc/layoutbin/en-US/strings.json index 1c3496a54e..f96afaa75c 100644 --- a/src/Misc/layoutbin/en-US/strings.json +++ b/src/Misc/layoutbin/en-US/strings.json @@ -137,6 +137,8 @@ " --acceptTeeEula macOS and Linux only. Accept the TEE end user license agreement.", " --gitUseSChannel Windows only. Tell Git to use Windows' native cert store.", " --alwaysExtractTask Perform an unzip for tasks for each pipeline step.", + " --disableLogUploads Don't stream or send console log output to the server. Instead, you may retrieve them from the agent host's filesystem after the job completes. NOTE: Cannot be used with --reStreamLogsToFiles, it will cause an error.", + " --reStreamLogsToFiles Stream or send console log output to the server as well as a log file on the agent host's filesystem. NOTE: Cannot be used with --disableLogUploads, it will cause an error.", "", "CLI-WIDTH-OPTIONS-(35-CHARS)-------CLI-WIDTH-DESCRIPTION-(70-CHARS)--------------------------------------", "Startup options (Windows only):", @@ -394,6 +396,7 @@ "ListenForJobs": "{0:u}: Listening for Jobs", "LocalClockSkewed": "The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again.", "LocalSystemAccountNotFound": "Cannot find local system account", + "LogOutputMessage": "The agent has enabled uploading logs as well as saving log to file. After the job completes, you can retrieve this step's logs at {0} on the agent.", "Maintenance": "Maintenance", "MaxHierarchyLevelReached": "Hierarchy level is more than supported limit {0}, truncating lower hierarchy.", "MaxSubResultLimitReached": "Number of subresults in test case '{0}' is more than the supported limit of {1}, truncating remaining ones.", @@ -503,6 +506,7 @@ "ResourceUtilizationWarningsIsDisabled": "Resource Utilization warnings is disabled, switch \"DISABLE_RESOURCE_UTILIZATION_WARNINGS\" variable to \"true\" if you want to enable it", "RestartIn15SecMessage": "Restarting the machine in 15 seconds...", "RestartMessage": "Restart the machine to launch agent and for autologon settings to take effect.", + "ReStreamLogsToFilesError": "You cannot use --disableloguploads and --reStreamLogsToFiles at the same time!", "RetryCountLimitExceeded": "The maximum allowed number of attempts is {0} but got {1}. Retry attempts count will be decreased to {0}.", "RMApiFailure": "Api {0} failed with an error code {1}", "RMArtifactContainerDetailsInvalidError": "The artifact does not have valid container details: {0}",