WIP - InteractionProfilerGrain

OrleansDashboard.Core/DashboardClient.cs

@@ -68,5 +68,10 @@ public async Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrai
         {
             return await dashboardGrain.TopGrainMethods().ConfigureAwait(false);
         }
+
+        public async Task<string> GetInteractionsGraph()
+        {
+            return await dashboardGrain.GetInteractionsGraph().ConfigureAwait(false);
+        }
     }
 }

OrleansDashboard.Core/IDashboardClient.cs

@@ -18,5 +18,6 @@ public interface IDashboardClient
         Task<Immutable<StatCounter[]>> GetCounters(string siloAddress);
         Task<Immutable<Dictionary<string, Dictionary<string, GrainTraceEntry>>>> GrainStats(string grainName);
         Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrainMethods();
+        Task<string> GetInteractionsGraph();
     }
 }

OrleansDashboard.Core/IDashboardGrain.cs

@@ -14,6 +14,9 @@ public interface IDashboardGrain : IGrainWithIntegerKey
 
         [OneWay]
         Task SubmitTracing(string siloAddress, Immutable<SiloGrainTraceEntry[]> grainCallTime);
+
+        [OneWay]
+        Task SubmitGrainInteraction(string interactionsGraph);
 
         Task<Immutable<DashboardCounters>> GetCounters();
 
@@ -24,5 +27,7 @@ public interface IDashboardGrain : IGrainWithIntegerKey
         Task<Immutable<Dictionary<string, GrainTraceEntry>>> GetSiloTracing(string address);
 
         Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrainMethods();
+
+        Task<string> GetInteractionsGraph();
     }
 }

OrleansDashboard.Core/Metrics/IGrainInteractionProfiler.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using Orleans;
+using Orleans.Concurrency;
+using OrleansDashboard.Model;
+
+namespace OrleansDashboard.Metrics
+{
+    public interface IInteractionProfiler : IGrainWithIntegerKey
+    {
+        [OneWay]
+        Task Track(GrainInteractionInfoEntry entry);
+    }
+}

OrleansDashboard.Core/Model/DashboardCounters.cs

@@ -26,5 +26,6 @@ public DashboardCounters()
         public SimpleGrainStatisticCounter[] SimpleGrainStats { get; set; }
         public int TotalActivationCount { get; set; }
         public ImmutableQueue<int> TotalActivationCountHistory { get; set; }
+        public string InteractionsGraph { get; set; }
     }
 }

OrleansDashboard.Core/Model/GrainInteractionInfoEntry.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace OrleansDashboard.Model
+{
+    [Serializable]
+    public class GrainInteractionInfoEntry
+    {
+        public string Grain { get; set; }
+        public string TargetGrain { get; set; }
+        public string Method { get; set; }
+        public uint Count { get; set; } = 1;
+
+        public string Key => Grain + ":" + (TargetGrain ?? string.Empty) + ":" + Method;
+    }
+}

OrleansDashboard/Implementation/GrainInteractionFilter.cs

@@ -0,0 +1,101 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Orleans;
+using Orleans.Runtime;
+using OrleansDashboard.Metrics;
+using OrleansDashboard.Model;
+
+namespace OrleansDashboard.Implementation
+{
+    public sealed class GrainInteractionFilter : IIncomingGrainCallFilter, IOutgoingGrainCallFilter
+    {
+        private readonly IGrainFactory grainFactory;
+        private IInteractionProfiler grainInteractionProfiler;
+
+        public GrainInteractionFilter(IGrainFactory grainFactory)
+        {
+            this.grainFactory = grainFactory;
+        }
+
+        public async Task Invoke(IIncomingGrainCallContext context)
+        {
+            try
+            {
+                var call = GetGrainMethod(context);
+                TrackBeginInvoke(call.Grain, call.Method);
+                await context.Invoke();
+            }
+            finally
+            {
+                await TrackEndInvoke();
+            }
+        }
+
+        public async Task Invoke(IOutgoingGrainCallContext context)
+        {
+            try
+            {
+                var call = GetGrainMethod(context);
+                TrackBeginInvoke(call.Grain, call.Method);
+                await context.Invoke();
+            }
+            finally
+            {
+                await TrackEndInvoke();
+            }
+        }
+
+        private Stack<GrainInteractionInfoEntry> GetCallStack()
+        {
+            return RequestContext.Get(nameof(GrainInteractionFilter)) as Stack<GrainInteractionInfoEntry> ?? new Stack<GrainInteractionInfoEntry>();
+        }
+
+        private void SaveCallStack(Stack<GrainInteractionInfoEntry> stack)
+        {
+            if (stack.Count == 0)
+            {
+                RequestContext.Remove(nameof(GrainInteractionFilter));
+            }
+            else
+            {
+                RequestContext.Set(nameof(GrainInteractionFilter), stack);
+            }
+        }
+
+        private void TrackBeginInvoke(string grain, string method)
+        {
+            var stack = GetCallStack();
+            if (stack.TryPeek(out var info))
+            {
+                info.TargetGrain = grain;
+            }
+
+            stack.Push(new GrainInteractionInfoEntry
+            {
+                Grain = grain,
+                Method = method
+            });
+            SaveCallStack(stack);
+        }
+
+        private async Task TrackEndInvoke()
+        {
+            var stack = GetCallStack();
+            var info = stack.Pop();
+
+            grainInteractionProfiler ??= grainFactory.GetGrain<IInteractionProfiler>(0);
+            await grainInteractionProfiler.Track(info);
+            SaveCallStack(stack);
+        }
+
+        private (string Grain, string Method) GetGrainMethod(IIncomingGrainCallContext context)
+        {
+            return (context.InterfaceMethod.ReflectedType.Name, context.InterfaceMethod.Name);
+        }
+
+        private (string Grain, string Method) GetGrainMethod(IOutgoingGrainCallContext context)
+        {
+            return (context.InterfaceMethod.ReflectedType.Name, context.InterfaceMethod.Name);
+        }
+    }
+}

OrleansDashboard/Implementation/Grains/DashboardGrain.cs

@@ -113,7 +113,7 @@ public override Task OnActivateAsync()
 
             return base.OnActivateAsync();
         }
-
+        
         public async Task<Immutable<DashboardCounters>> GetCounters()
         {
             await EnsureCountersAreUpToDate();
@@ -156,7 +156,12 @@ public async Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrai
                 { "errors", values.Where(x => x.ExceptionCount > 0 && x.Count > 0).OrderByDescending(x => x.ExceptionCount / x.Count).Take(numberOfResultsToReturn).ToArray() },
             }.AsImmutable();
         }
-
+
+        public Task<string> GetInteractionsGraph()
+        {
+            return Task.FromResult(counters.InteractionsGraph);
+        }
+
         public Task Init()
         {
             // just used to activate the grain
@@ -169,5 +174,11 @@ public Task SubmitTracing(string siloAddress, Immutable<SiloGrainTraceEntry[]> g
 
             return Task.CompletedTask;
         }
+
+        public Task SubmitGrainInteraction(string interactionsGraph)
+        {
+            counters.InteractionsGraph = interactionsGraph;
+            return Task.CompletedTask;
+        }
     }
 }

OrleansDashboard/Implementation/Grains/InteractionProfilerGrain.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+using Orleans;
+using Orleans.Concurrency;
+using OrleansDashboard.Model;
+
+namespace OrleansDashboard.Metrics.Grains
+{
+    [Reentrant]
+    public class InteractionProfilerGrain : Grain, IInteractionProfiler
+    {
+        private const int DefaultTimerIntervalMs = 1000; // 1 second
+        private readonly Dictionary<string, Dictionary<string, GrainInteractionInfoEntry>> interaction = new();
+        private readonly DashboardOptions options;
+        private IDisposable timer;
+
+        public InteractionProfilerGrain(IOptions<DashboardOptions> options)
+        {
+            this.options = options.Value;
+        }
+
+        public Task Track(GrainInteractionInfoEntry entry)
+        {
+            if (interaction.TryGetValue(entry.Grain, out var existing))
+            {
+                if (existing.TryGetValue(entry.Key, out var existingEntry))
+                {
+                    existingEntry.Count++;
+                }
+                else
+                {
+                    existing[entry.Key] = entry;
+                }
+            }
+            else
+            {
+                interaction.Add(entry.Grain, new Dictionary<string, GrainInteractionInfoEntry>
+                {
+                    [entry.Key] = entry
+                });
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public override async Task OnActivateAsync()
+        {
+            var updateInterval = TimeSpan.FromMilliseconds(Math.Max(options.CounterUpdateIntervalMs, DefaultTimerIntervalMs));
+
+            try
+            {
+                timer = RegisterTimer(x => CollectStatistics((bool)x), true, updateInterval, updateInterval);
+            }
+            catch (InvalidOperationException)
+            {
+                Debug.WriteLine("Not running in Orleans runtime");
+            }
+
+            await base.OnActivateAsync();
+        }
+
+        private async Task CollectStatistics(bool canDeactivate)
+        {
+            var dashboardGrain = GrainFactory.GetGrain<IDashboardGrain>(0);
+            try
+            {
+                await dashboardGrain.SubmitGrainInteraction(BuildGraph());
+            }
+            catch (Exception)
+            {
+                // we can't get the silo stats, it's probably dead, so kill the grain
+                if (canDeactivate)
+                {
+                    timer?.Dispose();
+                    timer = null;
+
+                    DeactivateOnIdle();
+                }
+            }
+        }
+
+        private string BuildGraph()
+        {
+            var content = string.Join("\n    ", interaction.Values.SelectMany(s => s)
+                /*.Where(w=>!string.IsNullOrEmpty(w.To))*/
+                .Select(s => $"{s.Value.Grain} -> {s.Value.TargetGrain ?? s.Value.Grain+"_self"} [ label = \"{s.Value.Method}\", color=\"0.650 0.700 0.700\" ];"));
+
+            var colors = string.Join("\n", interaction.Values.SelectMany(s => s)
+                .Select(s => s.Value.Grain)
+                .Distinct()
+                .Select(s => $"{s} [color=\"0.628 0.227 1.000\"];"));
+
+            var graphCode = @$"
+digraph finite_state_machine {{
+rankdir=LR;
+ratio = fill;
+node [style=filled];
+    {content}
+
+{colors}
+}}";
+            return graphCode;
+        }
+    }
+}

OrleansDashboard/ServiceCollectionExtensions.cs

@@ -26,6 +26,8 @@ public static ISiloHostBuilder UseDashboard(this ISiloHostBuilder builder,
         {
             builder.ConfigureApplicationParts(parts => parts.AddDashboardParts());
             builder.ConfigureServices(services => services.AddDashboard(configurator));
+            builder.AddIncomingGrainCallFilter<GrainInteractionFilter>();
+            builder.AddOutgoingGrainCallFilter<GrainInteractionFilter>();
             builder.AddStartupTask<Dashboard>();
 
             return builder;