diff --git a/Algorithm/QCAlgorithm.Plotting.cs b/Algorithm/QCAlgorithm.Plotting.cs index 23abbae02a4f..a8c68999dd4b 100644 --- a/Algorithm/QCAlgorithm.Plotting.cs +++ b/Algorithm/QCAlgorithm.Plotting.cs @@ -497,6 +497,12 @@ public void SetRuntimeStatistic(string name, double value) [DocumentationAttribute(StatisticsTag)] public void SetSummaryStatistic(string name, string value) { + if (int.TryParse(name, NumberStyles.Integer, CultureInfo.InvariantCulture, out var intName) && + intName >= 0 && intName <= 100) + { + throw new ArgumentException($"'{name}' is a reserved statistic name."); + } + _statisticsService.SetSummaryStatistic(name, value); } @@ -508,7 +514,7 @@ public void SetSummaryStatistic(string name, string value) [DocumentationAttribute(StatisticsTag)] public void SetSummaryStatistic(string name, int value) { - _statisticsService.SetSummaryStatistic(name, value.ToStringInvariant()); + SetSummaryStatistic(name, value.ToStringInvariant()); } /// @@ -519,7 +525,7 @@ public void SetSummaryStatistic(string name, int value) [DocumentationAttribute(StatisticsTag)] public void SetSummaryStatistic(string name, double value) { - _statisticsService.SetSummaryStatistic(name, value.ToStringInvariant()); + SetSummaryStatistic(name, value.ToStringInvariant()); } /// @@ -530,7 +536,7 @@ public void SetSummaryStatistic(string name, double value) [DocumentationAttribute(StatisticsTag)] public void SetSummaryStatistic(string name, decimal value) { - _statisticsService.SetSummaryStatistic(name, value.ToStringInvariant()); + SetSummaryStatistic(name, value.ToStringInvariant()); } /// diff --git a/Common/Api/OptimizationBacktestJsonConverter.cs b/Common/Api/OptimizationBacktestJsonConverter.cs index a2fea19201a8..f1acac5ee418 100644 --- a/Common/Api/OptimizationBacktestJsonConverter.cs +++ b/Common/Api/OptimizationBacktestJsonConverter.cs @@ -15,8 +15,8 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using QuantConnect.Optimizer.Parameters; @@ -30,6 +30,39 @@ namespace QuantConnect.Api /// public class OptimizationBacktestJsonConverter : JsonConverter { + private static string[] StatisticNames = + [ + PerformanceMetrics.Alpha, + PerformanceMetrics.AnnualStandardDeviation, + PerformanceMetrics.AnnualVariance, + PerformanceMetrics.AverageLoss, + PerformanceMetrics.AverageWin, + PerformanceMetrics.Beta, + PerformanceMetrics.CompoundingAnnualReturn, + PerformanceMetrics.Drawdown, + PerformanceMetrics.EstimatedStrategyCapacity, + PerformanceMetrics.Expectancy, + PerformanceMetrics.InformationRatio, + PerformanceMetrics.LossRate, + PerformanceMetrics.NetProfit, + PerformanceMetrics.ProbabilisticSharpeRatio, + PerformanceMetrics.ProfitLossRatio, + PerformanceMetrics.SharpeRatio, + PerformanceMetrics.TotalFees, + PerformanceMetrics.TotalOrders, + PerformanceMetrics.TrackingError, + PerformanceMetrics.TreynorRatio, + PerformanceMetrics.WinRate, + PerformanceMetrics.SortinoRatio, + PerformanceMetrics.StartEquity, + PerformanceMetrics.EndEquity, + PerformanceMetrics.PortfolioTurnover, + PerformanceMetrics.DrawdownRecovery, + ]; + + // Only 21 Lean statistics where supported when the serialized statistics where a json array + private static int ArrayStatisticsCount = 21; + /// /// Determines whether this instance can convert the specified object type. /// @@ -97,25 +130,27 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s if (!optimizationBacktest.Statistics.IsNullOrEmpty()) { writer.WritePropertyName("statistics"); - writer.WriteStartArray(); - foreach (var keyValuePair in optimizationBacktest.Statistics.OrderBy(pair => pair.Key)) + writer.WriteStartObject(); + + var customStatisticsNames = new HashSet(); + int idx; + foreach (var (name, statisticValue, index) in optimizationBacktest.Statistics + .Select(kvp => ( + Name: kvp.Key, + kvp.Value, + Index: (idx = Array.IndexOf(StatisticNames, kvp.Key)) != -1 ? idx : int.MaxValue + )) + .OrderBy(t => t.Index) + .ThenByDescending(t => t.Name)) { - switch (keyValuePair.Key) - { - case PerformanceMetrics.PortfolioTurnover: - case PerformanceMetrics.SortinoRatio: - case PerformanceMetrics.StartEquity: - case PerformanceMetrics.EndEquity: - case PerformanceMetrics.DrawdownRecovery: - continue; - } - var statistic = keyValuePair.Value.Replace("%", string.Empty); + var statistic = statisticValue.Replace("%", string.Empty, StringComparison.InvariantCulture); if (Currencies.TryParse(statistic, out var result)) { + writer.WritePropertyName(index < StatisticNames.Length? index.ToStringInvariant() : name); writer.WriteValue(result); } } - writer.WriteEndArray(); + writer.WriteEndObject(); } if (optimizationBacktest.ParameterSet != null) @@ -164,34 +199,25 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist Dictionary statistics = default; if (jStatistics != null) { - statistics = new Dictionary + if (jStatistics.Type == JTokenType.Array) + { + var statsCount = Math.Min(ArrayStatisticsCount, (jStatistics as JArray).Count); + statistics = new Dictionary(StatisticNames + .Take(statsCount) + .Select((x, i) => KeyValuePair.Create(x, jStatistics[i].Value())) + .Where(kvp => kvp.Value != null)); + } + else { - { PerformanceMetrics.Alpha, jStatistics[0].Value() }, - { PerformanceMetrics.AnnualStandardDeviation, jStatistics[1].Value() }, - { PerformanceMetrics.AnnualVariance, jStatistics[2].Value() }, - { PerformanceMetrics.AverageLoss, jStatistics[3].Value() }, - { PerformanceMetrics.AverageWin, jStatistics[4].Value() }, - { PerformanceMetrics.Beta, jStatistics[5].Value() }, - { PerformanceMetrics.CompoundingAnnualReturn, jStatistics[6].Value() }, - { PerformanceMetrics.Drawdown, jStatistics[7].Value() }, - { PerformanceMetrics.EstimatedStrategyCapacity, jStatistics[8].Value() }, - { PerformanceMetrics.Expectancy, jStatistics[9].Value() }, - { PerformanceMetrics.InformationRatio, jStatistics[10].Value() }, - { PerformanceMetrics.LossRate, jStatistics[11].Value() }, - { PerformanceMetrics.NetProfit, jStatistics[12].Value() }, - { PerformanceMetrics.ProbabilisticSharpeRatio, jStatistics[13].Value() }, - { PerformanceMetrics.ProfitLossRatio, jStatistics[14].Value() }, - { PerformanceMetrics.SharpeRatio, jStatistics[15].Value() }, - // TODO: Add SortinoRatio - // TODO: Add StartingEquity - // TODO: Add EndingEquity - // TODO: Add DrawdownRecovery - { PerformanceMetrics.TotalFees, jStatistics[16].Value() }, - { PerformanceMetrics.TotalOrders, jStatistics[17].Value() }, - { PerformanceMetrics.TrackingError, jStatistics[18].Value() }, - { PerformanceMetrics.TreynorRatio, jStatistics[19].Value() }, - { PerformanceMetrics.WinRate, jStatistics[20].Value() }, - }; + statistics = new(); + foreach (var statistic in jStatistics.Children()) + { + var statisticName = TryConvertToLeanStatisticIndex(statistic.Name, out var index) + ? StatisticNames[index] + : statistic.Name; + statistics[statisticName] = statistic.Value.Value(); + } + } } var parameterSet = serializer.Deserialize(jObject["parameterSet"].CreateReader()); @@ -220,5 +246,11 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist return optimizationBacktest; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryConvertToLeanStatisticIndex(string statistic, out int index) + { + return int.TryParse(statistic, out index) && index >= 0 && index < StatisticNames.Length; + } } } diff --git a/Tests/Algorithm/AlgorithmPlottingTests.cs b/Tests/Algorithm/AlgorithmPlottingTests.cs index 34ed1b646277..648e05e39d1c 100644 --- a/Tests/Algorithm/AlgorithmPlottingTests.cs +++ b/Tests/Algorithm/AlgorithmPlottingTests.cs @@ -220,5 +220,16 @@ public void PlotIndicatorPlotsBaseIndicator() Assert.AreEqual("PlotTest", chart.Name); Assert.AreEqual(sma1.Current.Value / sma2.Current.Value, chart.Series[ratio.Name].GetValues().First().y); } + + private static string[] ReservedSummaryStatisticNames => Enumerable.Range(0, 101).Select(i => i.ToStringInvariant()).ToArray(); + + [TestCaseSource(nameof(ReservedSummaryStatisticNames))] + public void ThrowsOnReservedSummaryStatisticName(string statisticName) + { + Assert.Throws(() => _algorithm.SetSummaryStatistic(statisticName, 0.1m)); + Assert.Throws(() => _algorithm.SetSummaryStatistic(statisticName, 0.1)); + Assert.Throws(() => _algorithm.SetSummaryStatistic(statisticName, 1)); + Assert.Throws(() => _algorithm.SetSummaryStatistic(statisticName, "0.1")); + } } } diff --git a/Tests/Api/OptimizationBacktestJsonConverterTests.cs b/Tests/Api/OptimizationBacktestJsonConverterTests.cs index 1854d4db9f61..0f3c6c50da96 100644 --- a/Tests/Api/OptimizationBacktestJsonConverterTests.cs +++ b/Tests/Api/OptimizationBacktestJsonConverterTests.cs @@ -27,14 +27,34 @@ namespace QuantConnect.Tests.API public class OptimizationBacktestJsonConverterTests { private const string _validSerialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + - "\"startDate\":\"2023-01-01T00:00:00Z\",\"endDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleMaxEndDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleDays\":10,\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," + + "\"startDate\":\"2023-01-01T00:00:00Z\",\"endDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleMaxEndDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleDays\":10,\"statistics\":{\"0\":0.374,\"1\":0.217,\"2\":0.047,\"3\":-4.51,\"4\":2.86,\"5\":-0.664,\"6\":52.602,\"7\":17.800,\"8\":6300000.00,\"9\":0.196,\"10\":1.571,\"11\":27.0,\"12\":123.888,\"13\":77.188,\"14\":0.63,\"15\":1.707,\"16\":1390.49,\"17\":180.0,\"18\":0.233,\"19\":-0.558,\"20\":73.0,\"21\":0.1,\"22\":100000.0,\"23\":200000.0,\"24\":3.0,\"25\":3.0}," + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0,1.0,1.0,1.0],[2,2.0,2.0,2.0,2.0],[3,3.0,3.0,3.0,3.0]]}"; private const string _oldValidSerialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + - "\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," + + "\"statistics\":{\"0\":0.374,\"1\":0.217,\"2\":0.047,\"3\":-4.51,\"4\":2.86,\"5\":-0.664,\"6\":52.602,\"7\":17.800,\"8\":6300000.00,\"9\":0.196,\"10\":1.571,\"11\":27.0,\"12\":123.888,\"13\":77.188,\"14\":0.63,\"15\":1.707,\"16\":1390.49,\"17\":180.0,\"18\":0.233,\"19\":-0.558,\"20\":73.0,\"21\":0.1,\"22\":100000.0,\"23\":200000.0,\"24\":3.0,\"25\":3.0}," + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0,1.0,1.0,1.0],[2,2.0,2.0,2.0,2.0],[3,3.0,3.0,3.0,3.0]]}"; private const string _oldValid2Serialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + + "\"statistics\":{\"0\":0.374,\"1\":0.217,\"2\":0.047,\"3\":-4.51,\"4\":2.86,\"5\":-0.664,\"6\":52.602,\"7\":17.800,\"8\":6300000.00,\"9\":0.196,\"10\":1.571,\"11\":27.0,\"12\":123.888,\"13\":77.188,\"14\":0.63,\"15\":1.707,\"16\":1390.49,\"17\":180.0,\"18\":0.233,\"19\":-0.558,\"20\":73.0,\"21\":0.1,\"22\":100000.0,\"23\":200000.0,\"24\":3.0,\"25\":3.0}," + + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0],[2,2.0],[3,3.0]]}"; + private const string _oldValid3Serialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + + "\"statistics\":{\"0\":0.374,\"1\":0.217,\"2\":0.047,\"3\":-4.51,\"4\":2.86,\"5\":-0.664,\"6\":52.602,\"7\":17.800,\"8\":6300000.00,\"9\":0.196,\"10\":1.571,\"11\":27.0,\"12\":123.888,\"13\":77.188,\"14\":0.63,\"15\":1.707,\"16\":1390.49,\"17\":180.0,\"18\":0.233,\"19\":-0.558,\"20\":73.0}," + + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0],[2,2.0],[3,3.0]]}"; + + private const string _validSerializationWithCustomStats = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + + "\"startDate\":\"2023-01-01T00:00:00Z\",\"endDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleMaxEndDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleDays\":10,\"statistics\":{\"0\":0.374,\"1\":0.217,\"2\":0.047,\"3\":-4.51,\"4\":2.86,\"5\":-0.664,\"6\":52.602,\"7\":17.800,\"8\":6300000.00,\"9\":0.196,\"10\":1.571,\"11\":27.0,\"12\":123.888,\"13\":77.188,\"14\":0.63,\"15\":1.707,\"16\":1390.49,\"17\":180.0,\"18\":0.233,\"19\":-0.558,\"20\":73.0,\"21\":0.1,\"22\":100000.0,\"23\":200000.0,\"24\":3.0,\"25\":3.0,\"customstat2\":5.4321,\"customstat1\":1.2345}," + + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0,1.0,1.0,1.0],[2,2.0,2.0,2.0,2.0],[3,3.0,3.0,3.0,3.0]]}"; + + private const string _validOldStatsDeserialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + + "\"startDate\":\"2023-01-01T00:00:00Z\",\"endDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleMaxEndDate\":\"2024-01-01T00:00:00Z\",\"outOfSampleDays\":10,\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," + + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0,1.0,1.0,1.0],[2,2.0,2.0,2.0,2.0],[3,3.0,3.0,3.0,3.0]]}"; + private const string _validOldStatsDeserialization2 = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + + "\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," + + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0,1.0,1.0,1.0],[2,2.0,2.0,2.0,2.0],[3,3.0,3.0,3.0,3.0]]}"; + private const string _validOldStatsDeserialization3 = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + "\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0],[2,2.0],[3,3.0]]}"; + private const string _validOldStatsDeserializationWithLessStats = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," + + "\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63]," + + "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0],[2,2.0],[3,3.0]]}"; [Test] public void SerializationNulls() @@ -58,7 +78,7 @@ public void Serialization(int version) optimizationBacktest.Statistics = new Dictionary { - { "Total Trades", "180" }, + { "Total Orders", "180" }, { "Average Win", "2.86%" }, { "Average Loss", "-4.51%" }, { "Compounding Annual Return", "52.602%" }, @@ -68,6 +88,7 @@ public void Serialization(int version) { "End Equity", "200000" }, { "Net Profit", "123.888%" }, { "Sharpe Ratio", "1.707" }, + { "Sortino Ratio", "0.1" }, { "Probabilistic Sharpe Ratio", "77.188%" }, { "Loss Rate", "27%" }, { "Win Rate", "73%" }, @@ -81,7 +102,8 @@ public void Serialization(int version) { "Treynor Ratio", "-0.558" }, { "Total Fees", "$1390.49" }, { "Estimated Strategy Capacity", "ZRX6300000.00" }, - { "Drawdown Recovery", "3" } + { "Drawdown Recovery", "3" }, + { "Portfolio Turnover", "3%" } }; optimizationBacktest.Equity = new CandlestickSeries @@ -106,10 +128,73 @@ public void Serialization(int version) Assert.AreEqual(expected, serialized); } - [TestCase(_validSerialization)] - [TestCase(_oldValidSerialization)] - [TestCase(_oldValid2Serialization)] - public void Deserialization(string serialization) + [Test] + public void SerializationWithCustomStatistics() + { + var optimizationBacktest = new OptimizationBacktest(new ParameterSet(18, + new Dictionary + { + { "pinocho", "19" }, + { "pepe", "-1" } + }), "backtestId", "ImABacktestName"); + + optimizationBacktest.Statistics = new Dictionary + { + { "customstat2", "5.4321" }, + { "customstat1", "1.2345" }, + { "Total Orders", "180" }, + { "Average Win", "2.86%" }, + { "Average Loss", "-4.51%" }, + { "Compounding Annual Return", "52.602%" }, + { "Drawdown", "17.800%" }, + { "Expectancy", "0.196" }, + { "Start Equity", "100000" }, + { "End Equity", "200000" }, + { "Net Profit", "123.888%" }, + { "Sharpe Ratio", "1.707" }, + { "Sortino Ratio", "0.1" }, + { "Probabilistic Sharpe Ratio", "77.188%" }, + { "Loss Rate", "27%" }, + { "Win Rate", "73%" }, + { "Profit-Loss Ratio", "0.63" }, + { "Alpha", "0.374" }, + { "Beta", "-0.664" }, + { "Annual Standard Deviation", "0.217" }, + { "Annual Variance", "0.047" }, + { "Information Ratio", "1.571" }, + { "Tracking Error", "0.233" }, + { "Treynor Ratio", "-0.558" }, + { "Total Fees", "$1390.49" }, + { "Estimated Strategy Capacity", "ZRX6300000.00" }, + { "Drawdown Recovery", "3" }, + { "Portfolio Turnover", "3%" } + }; + + optimizationBacktest.Equity = new CandlestickSeries + { + Values = new List { new Candlestick(1, 1, 1, 1, 1), new Candlestick(2, 2, 2, 2, 2), new Candlestick(3, 3, 3, 3, 3) } + }; + optimizationBacktest.StartDate = new DateTime(2023, 01, 01); + optimizationBacktest.EndDate = new DateTime(2024, 01, 01); + optimizationBacktest.OutOfSampleMaxEndDate = new DateTime(2024, 01, 01); + optimizationBacktest.OutOfSampleDays = 10; + + var serialized = JsonConvert.SerializeObject(optimizationBacktest); + + Assert.AreEqual(_validSerializationWithCustomStats, serialized); + } + + [TestCase(_validSerialization, false, 26)] + [TestCase(_oldValidSerialization, false, 26)] + [TestCase(_oldValid2Serialization, false, 26)] + // This case has only 21 stats because Sortino Ratio, Start Equity, End Equity and Drawdown Recovery were not supported + [TestCase(_oldValid3Serialization, false, 21)] + [TestCase(_validOldStatsDeserialization, false, 21)] + [TestCase(_validOldStatsDeserialization2, false, 21)] + [TestCase(_validOldStatsDeserialization3, false, 21)] + [TestCase(_validOldStatsDeserializationWithLessStats, false, 15)] + [TestCase(_validSerializationWithCustomStats, true, 26)] + public void Deserialization(string serialization, bool hasCustomStats, int expectedLeanStats) { var deserialized = JsonConvert.DeserializeObject(serialization); Assert.IsNotNull(deserialized); @@ -133,6 +218,19 @@ public void Deserialization(string serialization) Assert.IsTrue(((Candlestick)deserialized.Equity.Values[i]).Close == expected); } Assert.AreEqual("77.188", deserialized.Statistics[PerformanceMetrics.ProbabilisticSharpeRatio]); + + if (!hasCustomStats) + { + Assert.AreEqual(expectedLeanStats, deserialized.Statistics.Count); + } + else + { + // There are 2 custom stats + Assert.AreEqual(expectedLeanStats + 2, deserialized.Statistics.Count); + + Assert.AreEqual("1.2345", deserialized.Statistics["customstat1"]); + Assert.AreEqual("5.4321", deserialized.Statistics["customstat2"]); + } } } }