Skip to content

illustrate all iterations for gcperfsim #5248

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VincentBu wants to merge 2 commits into
base: main
Choose a base branch
from
Open

illustrate all iterations for gcperfsim #5248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8f057b2
illustrate all iterations for gcperfsim
VincentBu Jun 30, 2026
f08e991
update code based on copilot reviews
VincentBu Jun 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions ...rks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ public GCTraceMetricComparisonResult(IEnumerable<GCTraceMetrics> baselines, IEnu
OutliersFreeComparandMetricCollection = API.Statistics.RemoveOutliers(OriginalComparandMetricCollection);

// Calculate averaged metrics
OriginalAveragedBaselineMetric = API.GoodLinq.Average(
OriginalBaselineMetricCollection.ToList().Where(r => !double.IsNaN(r)), r => r);
OriginalAveragedComparandMetric = API.GoodLinq.Average(
OriginalComparandMetricCollection.ToList().Where(r => !double.IsNaN(r)), r => r);
AveragedBaselineMetric = API.GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r);
AveragedComparandMetric = API.GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r);
}
Expand All @@ -87,9 +91,30 @@ public GCTraceMetricComparisonResult(IEnumerable<GCTraceMetrics> baselines, IEnu
public IEnumerable<double> OutliersFreeBaselineMetricCollection { get; }
public IEnumerable<double> OutliersFreeComparandMetricCollection { get; }

public double OriginalAveragedBaselineMetric { get; }
public double OriginalAveragedComparandMetric { get; }
public double AveragedBaselineMetric { get; }
public double AveragedComparandMetric { get; }
public double OriginalDelta => OriginalAveragedComparandMetric - OriginalAveragedBaselineMetric;
public double Delta => AveragedComparandMetric - AveragedBaselineMetric;
public double OriginalPercentageDelta
{
get
{
if (OriginalAveragedBaselineMetric == 0)
{
if (OriginalAveragedComparandMetric == 0)
{
return 0;
}
else
{
return double.NaN;
}
}
return (OriginalDelta / OriginalAveragedBaselineMetric) * 100.0;
}
}
public double PercentageDelta
{
get
Expand All @@ -114,6 +139,7 @@ public double PercentageDelta

// Regression-oriented delta used for classification: positive => regression, negative => improvement,
// regardless of whether the metric is higher-is-better (e.g. Speed_MBPerMSec) or lower-is-better.
public double OriginalRegressionPercentageDelta => MetricDirection.GetRegressionDelta(MetricName, OriginalPercentageDelta);
public double RegressionPercentageDelta => MetricDirection.GetRegressionDelta(MetricName, PercentageDelta);
}
}
74 changes: 55 additions & 19 deletions ...benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/GCPerfSim/Markdown.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public static void GenerateComparisonTable(ResultItem baseResultItem, ResultItem
}

public static void GenerateForAnalyzeCommand(GCPerfSimConfiguration configuration,
IEnumerable<GCTraceMetricComparisonResults> comparisonResultsCollection,
IReadOnlyCollection<GCTraceMetricComparisonResults> comparisonResultsCollection,
Dictionary<string, ProcessExecutionDetails> executionDetails,
string path)
{
Expand Down Expand Up @@ -239,63 +239,99 @@ public static void GenerateForAnalyzeCommand(GCPerfSimConfiguration configuratio
sw.WriteLine();
sw.AddReproSection(executionDetails);
sw.WriteLine();
sw.AddDetailsOfSingleComparison(configuration, comparisonResults);
sw.AddDetailsOfSingleRun(configuration, comparisonResults);
}
}
}

internal static void AddDetailsOfSingleComparison(this StreamWriter sw,
GCPerfSimConfiguration configuration,
GCTraceMetricComparisonResults comparisonResult)
internal static void AddDetailsOfSingleRun(this StreamWriter sw,
GCPerfSimConfiguration configuration,
GCTraceMetricComparisonResults comparisonResult)
{
// Large Regressions
sw.WriteLine($"### Large Regressions (>= 20%): {comparisonResult.LargeRegressions.Count()} \n");
sw.AddTableForSingleCriteria(comparisonResult.LargeRegressions);
sw.AddTablesForSingleCriteria(comparisonResult.LargeRegressions);
sw.WriteLine("\n");

// Large Improvements
sw.WriteLine($"### Large Improvements (<= -20%): {comparisonResult.LargeImprovements.Count()} \n");
sw.AddTableForSingleCriteria(comparisonResult.LargeImprovements);
sw.AddTablesForSingleCriteria(comparisonResult.LargeImprovements);
sw.WriteLine("\n");

// Regressions
sw.WriteLine($"### Regressions (>= 5% and < 20%): {comparisonResult.Regressions.Count()} \n");
sw.AddTableForSingleCriteria(comparisonResult.Regressions);
sw.AddTablesForSingleCriteria(comparisonResult.Regressions);
sw.WriteLine("\n");

// Improvements
sw.WriteLine($"### Improvements (> -20% and <= -5%): {comparisonResult.Improvements.Count()} \n");
sw.AddTableForSingleCriteria(comparisonResult.Improvements);
sw.AddTablesForSingleCriteria(comparisonResult.Improvements);
sw.WriteLine("\n");

// Stale Regressions
sw.WriteLine($"### Stale Regressions (> 0% and < 5%): {comparisonResult.StaleRegressions.Count()} \n");
sw.AddTableForSingleCriteria(comparisonResult.StaleRegressions);
sw.AddTablesForSingleCriteria(comparisonResult.StaleRegressions);
sw.WriteLine("\n");

// Stale Improvements
sw.WriteLine($"### Stale Improvements (> -5% and <= 0%): {comparisonResult.StaleImprovements.Count()} \n");
sw.AddTableForSingleCriteria(comparisonResult.StaleImprovements);
sw.AddTablesForSingleCriteria(comparisonResult.StaleImprovements);
sw.WriteLine("\n\n");
}

internal static void AddTableForSingleCriteria(this StreamWriter sw,
IEnumerable<GCTraceMetricComparisonResult> comparisons)
internal static void AddTablesForSingleCriteria(this StreamWriter sw,
IEnumerable<GCTraceMetricComparisonResult> comparisons)
{
if (comparisons.ToList().Count == 0)
var comparisonList = comparisons.ToList();
if (comparisonList.Count == 0)
{
sw.WriteLine("No metrics in this category.\n");
return;
}
var runName = comparisons.FirstOrDefault()?.RunName;
sw.WriteLine($" | Metric | Base | {runName} | Δ% | Δ |");

foreach (var comparison in comparisonList)
{
sw.AddTableForSingleMetric(comparison);
sw.WriteLine("\n");
}
}

internal static void AddTableForSingleMetric(this StreamWriter sw,
GCTraceMetricComparisonResult comparison)
{
var runName = comparison.RunName;
var metricName = comparison.MetricName;
sw.WriteLine($" | {metricName} | Base | {runName} | Δ% | Δ |");
sw.WriteLine($" | ----- | ---- | ------ | --- | --- |");
foreach (var r in comparisons)
int maxIterations = Math.Max(comparison.OriginalBaselineMetricCollection.Count(), comparison.OriginalComparandMetricCollection.Count());
for (int idx = 0; idx < maxIterations; idx++)
{
sw.WriteLine($"| {r.MetricName} | {r.AveragedBaselineMetric:N2} | {r.AveragedComparandMetric:N2} | {r.PercentageDelta:N2} | {r.Delta:N2} |");
string baseRow = $"| iteration.{idx} ";
var baselineValue = comparison.OriginalBaselineMetricCollection?.ElementAtOrDefault(idx) ??double.NaN;
var comparandValue = comparison.OriginalComparandMetricCollection?.ElementAtOrDefault(idx) ?? double.NaN;
var baselineStr = double.IsNaN(baselineValue) || !(comparison.OutliersFreeBaselineMetricCollection?.Contains(baselineValue) ?? false) ? $"**~~{Math.Round(baselineValue, 2)}~~**" : Math.Round(baselineValue, 2).ToString();
var comparandStr = double.IsNaN(comparandValue) || !(comparison.OutliersFreeComparandMetricCollection?.Contains(comparandValue) ?? false) ? $"**~~{Math.Round(comparandValue, 2)}~~**" : Math.Round(comparandValue, 2).ToString();
baseRow += $"| {baselineStr} | {comparandStr} | | |";
sw.WriteLine(baseRow);
}
Comment thread
VincentBu marked this conversation as resolved.

// Add Original Averaged, Diff and DiffPerc
var originalAveragedBaselineValue = comparison.OriginalAveragedBaselineMetric;
var originalAveragedComparandValue = comparison.OriginalAveragedComparandMetric;
var originalDiff = comparison.OriginalDelta;
var originalDiffPerc = comparison.OriginalRegressionPercentageDelta;
string originalAveragedRow = $"| Average | {Math.Round(originalAveragedBaselineValue, 2)} | {Math.Round(originalAveragedComparandValue, 2)} | {Math.Round(originalDiffPerc, 2)} | {Math.Round(originalDiff, 2)} |";
sw.WriteLine(originalAveragedRow);

// Add Corrective Averaged, Diff and DiffPerc
var averagedBaselineValue = comparison.AveragedBaselineMetric;
var averagedComparandValue = comparison.AveragedComparandMetric;
var diff = comparison.Delta;
var diffPerc = comparison.RegressionPercentageDelta;
string averagedRow = $"| Average (Corrected) | {Math.Round(averagedBaselineValue, 2)} | {Math.Round(averagedComparandValue, 2)} | {Math.Round(diffPerc, 2)} | {Math.Round(diff, 2)} |";
sw.WriteLine(averagedRow);
}


internal static GCTraceMetricComparisonResult?
GetGCTraceMetricComparisonResultByRunNameAndMetricName(IEnumerable<GCTraceMetricComparisonResults> comparisonResultsCollection,
string runName,
Expand Down
2 changes: 1 addition & 1 deletion ...arks/gc/GC.Infrastructure/GC.Infrastructure/Commands/GCPerfSim/GCPerfSimAnalyzeCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public static IReadOnlyCollection<GCTraceMetricComparisonResults> ExecuteAnalysi
}

public static void Present(GCPerfSimConfiguration configuration,
IEnumerable<GCTraceMetricComparisonResults> comparisonResultGroupedByRunName,
IReadOnlyCollection<GCTraceMetricComparisonResults> comparisonResultGroupedByRunName,
Dictionary<string, ProcessExecutionDetails> executionDetails)
{
foreach (var format in configuration.Output.Formats)
Expand Down