Removes exception throwing during shutdown of dynamic instrumentation (#7375)

## Summary of changes

Removes exception throwing in "success" path, even when DI is disabled

## Reason for change

#7304 introduced a bunch of changes, but the use of
`CancellationTokenSource` resulted in exceptions being thrown in the
"happy" shutdown path, which can cause crashes in buggy versions of .NET
(i.e. all of them, currently)

## Implementation details

Replace usages of `TaskCompletionSource<bool>` with
`CancellationTokenSource`

## Test coverage

Covered by existing - checked the execution tests to make sure the
exception count has gone back own.

## Other details

Discovered some additional issues that need to be addressed I think.
Most importantly, `SafeDisposal` looks like a band-aid due to unclear
lifetime management. We should refactor the code to not require it i.e.
`Disposing` types should be safe and should not throw (regardless of
whether we catch the exception).

Additionally, this looks like it does quite a lot of work even when DI
is disabled. I would suggest we refactor it so that it doesn't do a
bunch of work in cases where it's never enabled.

Author: andrewlock

tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs

@@ -38,19 +38,16 @@ internal class DebuggerManager
 
         private static volatile bool _discoveryServiceReady;
         private readonly SemaphoreSlim _semaphore;
-        private readonly CancellationTokenSource _cancellationToken;
-        private volatile bool _isShuttingDown;
+        private readonly TaskCompletionSource<bool> _processExit = new(TaskCreationOptions.RunContinuationsAsynchronously);
         private int _initialized;
 
         private DebuggerManager(DebuggerSettings debuggerSettings, ExceptionReplaySettings exceptionReplaySettings)
         {
             _initialized = 0;
             _discoveryServiceReady = false;
-            _isShuttingDown = false;
             _semaphore = new SemaphoreSlim(1, 1);
             DebuggerSettings = debuggerSettings;
             ExceptionReplaySettings = exceptionReplaySettings;
-            _cancellationToken = new CancellationTokenSource();
             ServiceName = string.Empty;
         }
 
@@ -70,7 +67,7 @@ private DebuggerManager(DebuggerSettings debuggerSettings, ExceptionReplaySettin
 
         internal string ServiceName { get; private set; }
 
-        private async Task<bool> WaitForDiscoveryServiceAsync(IDiscoveryService discoveryService, CancellationToken cancellationToken)
+        private async Task<bool> WaitForDiscoveryServiceAsync(IDiscoveryService discoveryService, Task shutdownTask)
         {
             if (_discoveryServiceReady)
             {
@@ -79,10 +76,10 @@ private async Task<bool> WaitForDiscoveryServiceAsync(IDiscoveryService discover
 
             var tc = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
 
-            using var registration = cancellationToken.Register(() => tc.TrySetResult(false));
-
             discoveryService.SubscribeToChanges(Callback);
-            _discoveryServiceReady = await tc.Task.ConfigureAwait(false);
+            var completedTask = await Task.WhenAny(shutdownTask, tc.Task).ConfigureAwait(false);
+
+            _discoveryServiceReady = completedTask == tc.Task;
             return _discoveryServiceReady;
 
             void Callback(AgentConfiguration x)
@@ -209,7 +206,7 @@ private async Task SetDynamicInstrumentationState(TracerSettings tracerSettings)
                 var sw = Stopwatch.StartNew();
                 if (!_discoveryServiceReady)
                 {
-                    var isDiscoverySuccessful = await WaitForDiscoveryServiceAsync(discoveryService, _cancellationToken.Token).ConfigureAwait(false);
+                    var isDiscoverySuccessful = await WaitForDiscoveryServiceAsync(discoveryService, _processExit.Task).ConfigureAwait(false);
                     if (!isDiscoverySuccessful)
                     {
                         Log.Warning("Discovery service is not ready, Dynamic Instrumentation will not be initialized.");
@@ -238,7 +235,7 @@ internal Task UpdateConfiguration(TracerSettings tracerSettings, DebuggerSetting
 
         private async Task UpdateProductsState(TracerSettings tracerSettings, DebuggerSettings newDebuggerSettings)
         {
-            if (_isShuttingDown)
+            if (_processExit.Task.IsCompleted)
             {
                 return;
             }
@@ -248,8 +245,20 @@ private async Task UpdateProductsState(TracerSettings tracerSettings, DebuggerSe
             bool semaphoreAcquired = false;
             try
             {
-                semaphoreAcquired = await _semaphore.WaitAsync(TimeSpan.FromSeconds(30), _cancellationToken.Token).ConfigureAwait(false);
-                if (!semaphoreAcquired || _isShuttingDown)
+                var attemptsRemaining = 6; // 5*6 = 30s timeout
+                while (!_processExit.Task.IsCompleted && !semaphoreAcquired && attemptsRemaining > 0)
+                {
+                    attemptsRemaining--;
+                    semaphoreAcquired = await _semaphore.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+                }
+
+                if (_processExit.Task.IsCompleted)
+                {
+                    Log.Debug("Skipping update debugger state due to process exit");
+                    return;
+                }
+
+                if (!semaphoreAcquired)
                 {
                     Log.Debug("Skipping update debugger state due to semaphore timed out");
                     return;
@@ -345,19 +354,22 @@ private async Task InitializeSymbolUploader(TracerSettings tracerSettings)
 
         private void ShutdownTasks(Exception? ex)
         {
-            _isShuttingDown = true;
+            if (_processExit.Task.IsCompleted)
+            {
+                return;
+            }
+
+            _processExit.TrySetResult(true);
 
             if (ex != null)
             {
                 Log.Debug(ex, "Shutdown task for DebuggerManager is running with exception");
             }
 
             SafeDisposal.New()
-                        .Execute(() => _cancellationToken.Cancel(), "cancelling DebuggerManager operations")
                         .Add(DynamicInstrumentation)
                         .Add(ExceptionReplay)
                         .Add(SymbolsUploader)
-                        .Add(_cancellationToken)
                         .Add(_semaphore)
                         .DisposeAll();
         }

tracer/src/Datadog.Trace/Debugger/Symbols/SymbolsUploader.cs

@@ -42,13 +42,13 @@ internal class SymbolsUploader : IDebuggerUploader
         private readonly ImmutableHashSet<string> _symDb3rdPartyIncludes;
         private readonly ImmutableHashSet<string> _symDb3rdPartyExcludes;
         private readonly long _thresholdInBytes;
-        private readonly CancellationTokenSource _cancellationToken;
+        private readonly TaskCompletionSource<bool> _processExit = new(TaskCreationOptions.RunContinuationsAsynchronously);
         private readonly IBatchUploadApi _api;
         private readonly IRcmSubscriptionManager _subscriptionManager;
         private readonly ISubscription _subscription;
         private readonly object _disposeLock = new();
+        private readonly IDiscoveryService _discoveryService;
         private volatile bool _disposed = false;
-        private IDiscoveryService? _discoveryService;
         private byte[]? _payload;
         private string? _symDbEndpoint;
         private bool _isSymDbEnabled;
@@ -74,7 +74,6 @@ private SymbolsUploader(
             _thresholdInBytes = settings.SymbolDatabaseBatchSizeInBytes;
             _symDb3rdPartyIncludes = settings.SymDbThirdPartyDetectionIncludes;
             _symDb3rdPartyExcludes = settings.SymDbThirdPartyDetectionExcludes;
-            _cancellationToken = new CancellationTokenSource();
             _jsonSerializerSettings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore };
             _discoveryService.SubscribeToChanges(ConfigurationChanged);
             _subscription = new Subscription(Callback, RcmProducts.LiveDebuggingSymbolDb);
@@ -102,7 +101,7 @@ into rawFile
             {
                 _isSymDbEnabled = false;
                 UnRegisterToAssemblyLoadEvent();
-                _cancellationToken.Cancel(false);
+                _processExit.TrySetResult(true);
             }
 
             return Array.Empty<ApplyDetails>();
@@ -118,8 +117,7 @@ private void ConfigurationChanged(AgentConfiguration configuration)
 
             _symDbEndpoint = configuration.SymbolDbEndpoint;
             _discoveryServiceSemaphore.Release(1);
-            _discoveryService!.RemoveSubscription(ConfigurationChanged);
-            _discoveryService = null;
+            _discoveryService.RemoveSubscription(ConfigurationChanged);
         }
 
         public static IDebuggerUploader Create(IBatchUploadApi api, IDiscoveryService discoveryService, IRcmSubscriptionManager remoteConfigurationManager, TracerSettings tracerSettings, DebuggerSettings settings, string serviceName)
@@ -167,10 +165,13 @@ private async Task ProcessItemAsync(Assembly assembly)
             bool semaphoreAcquired = false;
             try
             {
-                await _assemblySemaphore.WaitAsync(_cancellationToken.Token).ConfigureAwait(false);
-                semaphoreAcquired = true;
+                var acquireTimeout = TimeSpan.FromSeconds(5);
+                while (!_processExit.Task.IsCompleted && !semaphoreAcquired)
+                {
+                    semaphoreAcquired = await _assemblySemaphore.WaitAsync(acquireTimeout).ConfigureAwait(false);
+                }
 
-                if (!_isSymDbEnabled || _cancellationToken.IsCancellationRequested || _disposed)
+                if (!_isSymDbEnabled || _processExit.Task.IsCompleted || _disposed)
                 {
                     return;
                 }
@@ -392,7 +393,6 @@ public async Task StartFlushingAsync()
 
             if (await WaitForEnablementAsync().ConfigureAwait(false) == false)
             {
-                Log.Information("This can happen when the service is shut down");
                 return;
             }
 
@@ -413,7 +413,7 @@ private async Task<bool> WaitForDiscoveryServiceAsync()
                 return true;
             }
 
-            if (_disposed || _cancellationToken.IsCancellationRequested)
+            if (_disposed || _processExit.Task.IsCompleted)
             {
                 return false;
             }
@@ -422,10 +422,12 @@ private async Task<bool> WaitForDiscoveryServiceAsync()
 
             try
             {
-                await _discoveryServiceSemaphore.WaitAsync(_cancellationToken.Token).ConfigureAwait(false);
-                _discoveryServiceSemaphore.Dispose();
+                var completedTask = await Task.WhenAny(
+                                                   _discoveryServiceSemaphore.WaitAsync(),
+                                                   _processExit.Task)
+                                              .ConfigureAwait(false);
 
-                if (_cancellationToken.IsCancellationRequested || _disposed)
+                if (completedTask == _processExit.Task || _disposed)
                 {
                     return false;
                 }
@@ -444,7 +446,7 @@ private async Task<bool> WaitForDiscoveryServiceAsync()
 
         private async Task<bool> WaitForEnablementAsync()
         {
-            if (_disposed || _cancellationToken.IsCancellationRequested)
+            if (_disposed || _processExit.Task.IsCompleted)
             {
                 return false;
             }
@@ -453,8 +455,12 @@ private async Task<bool> WaitForEnablementAsync()
 
             try
             {
-                await _enablementSemaphore.WaitAsync(_cancellationToken.Token).ConfigureAwait(false);
-                if (_cancellationToken.IsCancellationRequested || _disposed)
+                var completedTask = await Task.WhenAny(
+                                                   _enablementSemaphore.WaitAsync(),
+                                                   _processExit.Task)
+                                              .ConfigureAwait(false);
+
+                if (completedTask == _processExit.Task || _disposed)
                 {
                     return false;
                 }
@@ -488,19 +494,7 @@ public void Dispose()
                 _disposed = true;
                 _subscriptionManager.Unsubscribe(_subscription);
 
-                try
-                {
-                    if (!_cancellationToken.IsCancellationRequested)
-                    {
-                        _cancellationToken.Cancel();
-                    }
-                }
-                catch (ObjectDisposedException)
-                {
-                    // Already disposed, ignore
-                }
-
-                _cancellationToken.Dispose();
+                _processExit.TrySetResult(true);
                 _assemblySemaphore.Dispose();
                 _enablementSemaphore.Dispose();
                 _discoveryServiceSemaphore.Dispose();