From 35757550f5fc1f8d25ca1080502e0a2ffebb3dc2 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 20 Nov 2025 07:03:04 -0700 Subject: [PATCH 01/13] Timestamp precision in encoding options --- .../evm/report_codec_evm_abi_encode_unpacked_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go index 1f20178..42e736a 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go @@ -97,7 +97,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { {Name: "baseMarketDepth", Type: mustNewABIType("int192")}, {Name: "quoteMarketDepth", Type: mustNewABIType("int192")}, }) - runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier, marketDepthMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleDexBasedAssetPrice, sampleBaseMarketDepth, sampleQuoteMarketDepth decimal.Decimal) bool { + runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier, marketDepthMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleDexBasedAssetPrice, sampleBaseMarketDepth, sampleQuoteMarketDepth decimal.Decimal, sampleTimestampPrecision TimestampPrecision) bool { report := llo.Report{ ConfigDigest: types.ConfigDigest{0x01}, SeqNr: 0x02, @@ -198,6 +198,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { genBenchmarkPrice(), genBaseMarketDepth(), genQuoteMarketDepth(), + genTimestampPrecision(), )) properties.TestingRun(t) From 1daf7a419ee9b6d6c52462ad9a87e9f86e1b7bfe Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Fri, 21 Nov 2025 14:54:38 -0700 Subject: [PATCH 02/13] remove sample from test input --- .../evm/report_codec_evm_abi_encode_unpacked_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go index 42e736a..1f20178 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go @@ -97,7 +97,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { {Name: "baseMarketDepth", Type: mustNewABIType("int192")}, {Name: "quoteMarketDepth", Type: mustNewABIType("int192")}, }) - runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier, marketDepthMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleDexBasedAssetPrice, sampleBaseMarketDepth, sampleQuoteMarketDepth decimal.Decimal, sampleTimestampPrecision TimestampPrecision) bool { + runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier, marketDepthMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleDexBasedAssetPrice, sampleBaseMarketDepth, sampleQuoteMarketDepth decimal.Decimal) bool { report := llo.Report{ ConfigDigest: types.ConfigDigest{0x01}, SeqNr: 0x02, @@ -198,7 +198,6 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { genBenchmarkPrice(), genBaseMarketDepth(), genQuoteMarketDepth(), - genTimestampPrecision(), )) properties.TestingRun(t) From 590d969c90bbdb7c09459a21e7ae4efa06300fb2 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Wed, 3 Dec 2025 17:22:01 -0700 Subject: [PATCH 03/13] DS-1732 channel opts cache --- llo/channel_definitions.go | 44 ++++++ llo/plugin.go | 18 +++ llo/plugin_outcome.go | 61 +++++++-- llo/plugin_outcome_test.go | 127 ++++++++++++++---- llo/plugin_reports.go | 2 +- llo/reportcodecs/evm/report_codec_common.go | 57 +------- .../report_codec_evm_abi_encode_unpacked.go | 31 ++++- ...port_codec_evm_abi_encode_unpacked_expr.go | 28 +++- ...codec_evm_abi_encode_unpacked_expr_test.go | 58 ++++---- ...port_codec_evm_abi_encode_unpacked_test.go | 92 ++++++------- .../evm/report_codec_premium_legacy.go | 14 ++ llo/types.go | 60 +++++++++ 12 files changed, 413 insertions(+), 179 deletions(-) diff --git a/llo/channel_definitions.go b/llo/channel_definitions.go index 4585363..1b3a8fe 100644 --- a/llo/channel_definitions.go +++ b/llo/channel_definitions.go @@ -69,3 +69,47 @@ func subtractChannelDefinitions(minuend llotypes.ChannelDefinitions, subtrahend return difference } + +type channelDefinitionOptsCache struct { + cache map[llotypes.ChannelID]interface{} +} + +var _ ChannelDefinitionOptsCache = (*channelDefinitionOptsCache)(nil) + +// NewChannelDefinitionOptsCache creates a new ChannelDefinitionOptsCache +func NewChannelDefinitionOptsCache() ChannelDefinitionOptsCache { + return &channelDefinitionOptsCache{ + cache: make(map[llotypes.ChannelID]interface{}), + } +} + +// If the codec does not implement OptsParser, nothing is cached +func (c *channelDefinitionOptsCache) Set( + channelID llotypes.ChannelID, + channelOpts llotypes.ChannelOpts, + codec ReportCodec, +) error { + // Check if codec implements optional OptsParser interface + optsParser, ok := codec.(OptsParser) + if !ok { + // Codec doesn't implement OptsParser, nothing to cache + return nil + } + + parsedOpts, err := optsParser.ParseOpts(channelOpts) + if err != nil { + return fmt.Errorf("failed to parse opts for channelID %d: %w", channelID, err) + } + + c.cache[channelID] = parsedOpts + return nil +} + +func (c *channelDefinitionOptsCache) Get(channelID llotypes.ChannelID) (interface{}, bool) { + val, ok := c.cache[channelID] + return val, ok +} + +func (c *channelDefinitionOptsCache) Delete(channelID llotypes.ChannelID) { + delete(c.cache, channelID) +} diff --git a/llo/plugin.go b/llo/plugin.go index 2df127e..c23ca4a 100644 --- a/llo/plugin.go +++ b/llo/plugin.go @@ -165,6 +165,20 @@ type ChannelDefinitionCache interface { Definitions() llotypes.ChannelDefinitions } +// ChannelDefinitionOptsCache is a cache of channel definition opts +// It is used to avoid repeated JSON parsing of channel definition opts +type ChannelDefinitionOptsCache interface { + // Set parses and caches the channel definition opts for the given channelID + // The channelOpts should match the ReportCodec's opts type. + Set(channelID llotypes.ChannelID, channelOpts llotypes.ChannelOpts, codec ReportCodec) error + // Get retrieves cached opts for the given channelID + // Returning `interface{}` requires type assertion to the specific ReportCodec's opts type. + // This is still considered better than parsing the opts from JSON every time we need to access them. + Get(channelID llotypes.ChannelID) (interface{}, bool) + // Delete removes cached opts for the given channelID + Delete(channelID llotypes.ChannelID) +} + // A ReportingPlugin allows plugging custom logic into the OCR3 protocol. The OCR // protocol handles cryptography, networking, ensuring that a sufficient number // of nodes is in agreement about any report, transmitting the report to the @@ -278,6 +292,8 @@ func (f *PluginFactory) NewReportingPlugin(ctx context.Context, cfg ocr3types.Re ballastAlloc = make([]byte, ballastSz) }) + channelOptsCache := NewChannelDefinitionOptsCache() + return &Plugin{ f.Config, onchainConfig.PredecessorConfigDigest, @@ -285,6 +301,7 @@ func (f *PluginFactory) NewReportingPlugin(ctx context.Context, cfg ocr3types.Re f.PredecessorRetirementReportCache, f.ShouldRetireCache, f.ChannelDefinitionCache, + channelOptsCache, f.DataSource, l, cfg.N, @@ -320,6 +337,7 @@ type Plugin struct { PredecessorRetirementReportCache PredecessorRetirementReportCache ShouldRetireCache ShouldRetireCache ChannelDefinitionCache ChannelDefinitionCache + ChannelDefinitionOptsCache ChannelDefinitionOptsCache DataSource DataSource Logger logger.Logger N int diff --git a/llo/plugin_outcome.go b/llo/plugin_outcome.go index 230e10c..0ff5a9d 100644 --- a/llo/plugin_outcome.go +++ b/llo/plugin_outcome.go @@ -104,6 +104,7 @@ func (p *Plugin) outcome(outctx ocr3types.OutcomeContext, query types.Query, aos } removedChannelIDs = append(removedChannelIDs, channelID) delete(outcome.ChannelDefinitions, channelID) + p.ChannelDefinitionOptsCache.Delete(channelID) } type hashWithID struct { @@ -150,6 +151,9 @@ func (p *Plugin) outcome(outctx ocr3types.OutcomeContext, query types.Query, aos ) } outcome.ChannelDefinitions[defWithID.ChannelID] = defWithID.ChannelDefinition + + // Parse and cache opts to avoid repeated JSON parsing where opts are needed + p.parseAndCacheChannelOpts(defWithID.ChannelID, defWithID.ChannelDefinition) } ///////////////////////////////// @@ -162,7 +166,7 @@ func (p *Plugin) outcome(outctx ocr3types.OutcomeContext, query types.Query, aos if outcome.ValidAfterNanoseconds == nil { outcome.ValidAfterNanoseconds = map[llotypes.ChannelID]uint64{} for channelID, previousValidAfterNanoseconds := range previousOutcome.ValidAfterNanoseconds { - if err3 := previousOutcome.IsReportable(channelID, p.ProtocolVersion, p.DefaultMinReportIntervalNanoseconds); err3 != nil { + if err3 := previousOutcome.IsReportable(channelID, p.ProtocolVersion, p.DefaultMinReportIntervalNanoseconds, p.ReportCodecs, p.ChannelDefinitionOptsCache); err3 != nil { if p.Config.VerboseLogging { p.Logger.Debugw("Channel is not reportable", "channelID", channelID, "err", err3, "stage", "Outcome", "seqNr", outctx.SeqNr) } @@ -400,7 +404,7 @@ func (out *Outcome) GenRetirementReport(protocolVersion uint32) RetirementReport // NOTE: A channel is still reportable even if missing some or all stream // values. The report codec is expected to handle nils and act accordingly // (e.g. some values may be optional). -func (out *Outcome) IsReportable(channelID llotypes.ChannelID, protocolVersion uint32, minReportInterval uint64) *UnreportableChannelError { +func (out *Outcome) IsReportable(channelID llotypes.ChannelID, protocolVersion uint32, minReportInterval uint64, codecsMap map[llotypes.ReportFormat]ReportCodec, optsCache ChannelDefinitionOptsCache) *UnreportableChannelError { if out.LifeCycleStage == LifeCycleStageRetired { return &UnreportableChannelError{nil, "IsReportable=false; retired channel", channelID} } @@ -410,6 +414,11 @@ func (out *Outcome) IsReportable(channelID llotypes.ChannelID, protocolVersion u return &UnreportableChannelError{nil, "IsReportable=false; no channel definition with this ID", channelID} } + codec, hasCodec := codecsMap[cd.ReportFormat] + if !hasCodec { + return &UnreportableChannelError{nil, fmt.Sprintf("IsReportable=false; no codec found for report format %d", cd.ReportFormat), channelID} + } + validAfterNanos, ok := out.ValidAfterNanoseconds[channelID] if !ok { // No ValidAfterNanoseconds entry yet, this must be a new channel. @@ -436,7 +445,11 @@ func (out *Outcome) IsReportable(channelID llotypes.ChannelID, protocolVersion u // This keeps compatibility with old nodes that may not have nanosecond resolution // // Also use seconds resolution for report formats that require it to prevent overlap - if protocolVersion == 0 || IsSecondsResolution(cd.ReportFormat) { + isSecondsResolution, err := IsSecondsResolution(channelID, codec, optsCache) + if err != nil { + return &UnreportableChannelError{err, "IsReportable=false; failed to determine time resolution", channelID} + } + if protocolVersion == 0 || isSecondsResolution { validAfterSeconds := validAfterNanos / 1e9 obsTsSeconds := obsTsNanos / 1e9 if validAfterSeconds >= obsTsSeconds { @@ -447,25 +460,33 @@ func (out *Outcome) IsReportable(channelID llotypes.ChannelID, protocolVersion u return nil } -func IsSecondsResolution(reportFormat llotypes.ReportFormat) bool { - switch reportFormat { - // TODO: Might be cleaner to expose a TimeResolution() uint64 field on the - // ReportCodec so that the plugin doesn't have to have special knowledge of - // the report format details - case llotypes.ReportFormatEVMPremiumLegacy, llotypes.ReportFormatEVMABIEncodeUnpacked: - return true - default: - return false +func IsSecondsResolution(channelID llotypes.ChannelID, codec ReportCodec, optsCache ChannelDefinitionOptsCache) (bool, error) { + // Try to determine the time resolution from the channel definition opts + if optsCache != nil { + if cachedOpts, cached := optsCache.Get(channelID); cached { + if optsParser, ok := codec.(OptsParser); ok { + resolution, err := optsParser.TimeResolution(cachedOpts) + if err != nil { + // This should not happen since caching the cached opts should logically ensure that this channel's opts + // are valid for channels report codec. If this error occurs it would indicate a wrong codec/opts mismatch. + return false, fmt.Errorf("failed to parse time resolution from opts (wrong codec/opts mismatch?) :%w", err) + } + return resolution == ResolutionSeconds, nil + } + } } + + // Fall back to protocol default time resolution when codecs don't implement OptsParser + return false, nil } // List of reportable channels (according to IsReportable), sorted according // to a canonical ordering -func (out *Outcome) ReportableChannels(protocolVersion uint32, defaultMinReportInterval uint64) (reportable []llotypes.ChannelID, unreportable []*UnreportableChannelError) { +func (out *Outcome) ReportableChannels(protocolVersion uint32, defaultMinReportInterval uint64, codecsMap map[llotypes.ReportFormat]ReportCodec, optsCache ChannelDefinitionOptsCache) (reportable []llotypes.ChannelID, unreportable []*UnreportableChannelError) { for channelID := range out.ChannelDefinitions { // In theory in future, minReportInterval could be overridden on a // per-channel basis in the ChannelDefinitions - if err := out.IsReportable(channelID, protocolVersion, defaultMinReportInterval); err != nil { + if err := out.IsReportable(channelID, protocolVersion, defaultMinReportInterval, codecsMap, optsCache); err != nil { unreportable = append(unreportable, err) } else { reportable = append(reportable, channelID) @@ -576,3 +597,15 @@ func makeOutcomeTelemetry(outcome Outcome, configDigest types.ConfigDigest, seqN } return ot, nil } + +// parseAndCacheChannelOpts parses and caches channel opts to avoid repeated JSON parsing +func (p *Plugin) parseAndCacheChannelOpts(channelID llotypes.ChannelID, cd llotypes.ChannelDefinition) { + codec, exists := p.ReportCodecs[cd.ReportFormat] + if !exists { + return + } + + if err := p.ChannelDefinitionOptsCache.Set(channelID, cd.Opts, codec); err != nil { + p.Logger.Warnw("Failed to parse opts for caching", "channelID", channelID, "reportFormat", cd.ReportFormat, "err", err) + } +} diff --git a/llo/plugin_outcome_test.go b/llo/plugin_outcome_test.go index 14490c9..a7bfc19 100644 --- a/llo/plugin_outcome_test.go +++ b/llo/plugin_outcome_test.go @@ -38,6 +38,11 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { ObservationCodec: obsCodec, DonID: 10000043, ConfigDigest: types.ConfigDigest{1, 2, 3, 4}, + ReportCodecs: map[llotypes.ReportFormat]ReportCodec{ + llotypes.ReportFormatJSON: mockCodec{timeResolution: ResolutionNanoseconds}, + llotypes.ReportFormatEVMPremiumLegacy: mockCodec{timeResolution: ResolutionSeconds}, + }, + ChannelDefinitionOptsCache: NewChannelDefinitionOptsCache(), } testStartTS := time.Now() testStartNanos := uint64(testStartTS.UnixNano()) //nolint:gosec // safe cast in tests @@ -147,6 +152,35 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { assert.Equal(t, newCd, decoded.ChannelDefinitions[42]) }) + t.Run("removes a channel definition if there are enough votes", func(t *testing.T) { + t.Skip("removal votes are not implemented yet") + newCd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormat(2), + Streams: []llotypes.Stream{{StreamID: 1, Aggregator: llotypes.AggregatorMedian}, {StreamID: 2, Aggregator: llotypes.AggregatorMedian}, {StreamID: 3, Aggregator: llotypes.AggregatorMedian}}, + } + obs, err := p.ObservationCodec.Encode(Observation{ + UpdateChannelDefinitions: map[llotypes.ChannelID]llotypes.ChannelDefinition{ + 42: newCd, + }, + }) + require.NoError(t, err) + aos := []types.AttributedObservation{} + for i := uint8(0); i < 4; i++ { + aos = append(aos, + types.AttributedObservation{ + Observation: obs, + Observer: commontypes.OracleID(i), + }) + } + outcome, err := p.Outcome(ctx, ocr3types.OutcomeContext{SeqNr: 2}, types.Query{}, aos) + require.NoError(t, err) + + decoded, err := p.OutcomeCodec.Decode(outcome) + require.NoError(t, err) + + assert.Equal(t, newCd, decoded.ChannelDefinitions[42]) + }) + t.Run("does not add channels beyond MaxOutcomeChannelDefinitionsLength", func(t *testing.T) { newCd := llotypes.ChannelDefinition{ ReportFormat: llotypes.ReportFormat(2), @@ -695,7 +729,46 @@ func Test_MakeChannelHash(t *testing.T) { }) } +type mockCodec struct { + timeResolution TimeResolution +} + +var ( + _ ReportCodec = mockCodec{} + _ OptsParser = mockCodec{} +) + +func (mockCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { + return nil, nil +} + +func (mockCodec) Verify(llotypes.ChannelDefinition) error { + return nil +} + +func (c mockCodec) ParseOpts(opts []byte) (interface{}, error) { + // TODO do we need to parse opts in anyway here? + return c, nil +} + +func (c mockCodec) TimeResolution(parsedOpts interface{}) (TimeResolution, error) { + if tc, ok := parsedOpts.(mockCodec); ok { + return tc.timeResolution, nil + } + return c.timeResolution, nil +} + func Test_Outcome_Methods(t *testing.T) { + // Cache for parsed channel definition opts + optsCache := NewChannelDefinitionOptsCache() + // Use test codecs that mimic real codec behavior + codecs := map[llotypes.ReportFormat]ReportCodec{ + llotypes.ReportFormat(0): mockCodec{timeResolution: ResolutionNanoseconds}, + llotypes.ReportFormatEVMPremiumLegacy: mockCodec{timeResolution: ResolutionSeconds}, + llotypes.ReportFormatEVMABIEncodeUnpacked: mockCodec{timeResolution: ResolutionNanoseconds}, + llotypes.ReportFormatEVMABIEncodeUnpackedExpr: mockCodec{timeResolution: ResolutionNanoseconds}, + } + t.Run("protocol version 0", func(t *testing.T) { t.Run("IsReportable", func(t *testing.T) { outcome := Outcome{} @@ -703,31 +776,31 @@ func Test_Outcome_Methods(t *testing.T) { // Not reportable if retired outcome.LifeCycleStage = LifeCycleStageRetired - require.EqualError(t, outcome.IsReportable(cid, 0, 0), "ChannelID: 1; Reason: IsReportable=false; retired channel") + require.EqualError(t, outcome.IsReportable(cid, 0, 0, codecs, optsCache), "ChannelID: 1; Reason: IsReportable=false; retired channel") // No channel definition with ID outcome.LifeCycleStage = LifeCycleStageProduction outcome.ObservationTimestampNanoseconds = uint64(time.Unix(1726670490, 0).UnixNano()) //nolint:gosec // time won't be negative outcome.ChannelDefinitions = map[llotypes.ChannelID]llotypes.ChannelDefinition{} - require.EqualError(t, outcome.IsReportable(cid, 0, 0), "ChannelID: 1; Reason: IsReportable=false; no channel definition with this ID") + require.EqualError(t, outcome.IsReportable(cid, 0, 0, codecs, optsCache), "ChannelID: 1; Reason: IsReportable=false; no channel definition with this ID") // No ValidAfterNanoseconds yet outcome.ChannelDefinitions = map[llotypes.ChannelID]llotypes.ChannelDefinition{ cid: {}, } - require.EqualError(t, outcome.IsReportable(cid, 0, 0), "ChannelID: 1; Reason: IsReportable=false; no ValidAfterNanoseconds entry yet, this must be a new channel") + require.EqualError(t, outcome.IsReportable(cid, 0, 0, codecs, optsCache), "ChannelID: 1; Reason: IsReportable=false; no ValidAfterNanoseconds entry yet, this must be a new channel") // ValidAfterNanoseconds is in the future outcome.ValidAfterNanoseconds = map[llotypes.ChannelID]uint64{cid: uint64(1726670491 * time.Second)} - require.EqualError(t, outcome.IsReportable(cid, 0, 0), "ChannelID: 1; Reason: ChannelID: 1; Reason: IsReportable=false; not valid yet (observationsTimestampSeconds=1726670490, validAfterSeconds=1726670491)") + require.EqualError(t, outcome.IsReportable(cid, 0, 0, codecs, optsCache), "ChannelID: 1; Reason: ChannelID: 1; Reason: IsReportable=false; not valid yet (observationsTimestampSeconds=1726670490, validAfterSeconds=1726670491)") // ValidAfterSeconds=ObservationTimestampSeconds; IsReportable=false outcome.ValidAfterNanoseconds = map[llotypes.ChannelID]uint64{cid: uint64(1726670490 * time.Second)} - require.EqualError(t, outcome.IsReportable(cid, 0, 0), "ChannelID: 1; Reason: ChannelID: 1; Reason: IsReportable=false; not valid yet (observationsTimestampSeconds=1726670490, validAfterSeconds=1726670490)") + require.EqualError(t, outcome.IsReportable(cid, 0, 0, codecs, optsCache), "ChannelID: 1; Reason: ChannelID: 1; Reason: IsReportable=false; not valid yet (observationsTimestampSeconds=1726670490, validAfterSeconds=1726670490)") // ValidAfterSeconds= 1s, does report outcome.ValidAfterNanoseconds[cid] = obsTSNanos - uint64(1*time.Second) - assert.Nil(t, outcome.IsReportable(cid, 1, uint64(100*time.Millisecond))) + assert.Nil(t, outcome.IsReportable(cid, 1, uint64(100*time.Millisecond), codecs, optsCache)) // if cadence is exactly 1s, if time is >= 1s, does report - assert.Nil(t, outcome.IsReportable(cid, 1, uint64(1*time.Second))) + assert.Nil(t, outcome.IsReportable(cid, 1, uint64(1*time.Second), codecs, optsCache)) // if cadence is 5s, if time is < 5s, does not report because cadence hasn't elapsed - require.EqualError(t, outcome.IsReportable(cid, 1, uint64(5*time.Second)), "ChannelID: 1; Reason: IsReportable=false; not valid yet (ObservationTimestampNanoseconds=1726670490999999999, validAfterNanoseconds=1726670489999999999, minReportInterval=5000000000); 4.000000 seconds (4000000000ns) until reportable") + require.EqualError(t, outcome.IsReportable(cid, 1, uint64(5*time.Second), codecs, optsCache), "ChannelID: 1; Reason: IsReportable=false; not valid yet (ObservationTimestampNanoseconds=1726670490999999999, validAfterNanoseconds=1726670489999999999, minReportInterval=5000000000); 4.000000 seconds (4000000000ns) until reportable") }) t.Run("ReportableChannels", func(t *testing.T) { defaultMinReportInterval := uint64(1 * time.Second) @@ -837,7 +914,7 @@ func Test_Outcome_Methods(t *testing.T) { 3: uint64(1726670489 * time.Second), }, } - reportable, unreportable := outcome.ReportableChannels(1, defaultMinReportInterval) + reportable, unreportable := outcome.ReportableChannels(1, defaultMinReportInterval, codecs, optsCache) assert.Equal(t, []llotypes.ChannelID{1, 3}, reportable) require.Len(t, unreportable, 1) assert.Equal(t, "ChannelID: 2; Reason: IsReportable=false; no ValidAfterNanoseconds entry yet, this must be a new channel", unreportable[0].Error()) diff --git a/llo/plugin_reports.go b/llo/plugin_reports.go index d77ad2f..e2d8cfb 100644 --- a/llo/plugin_reports.go +++ b/llo/plugin_reports.go @@ -46,7 +46,7 @@ func (p *Plugin) reports(ctx context.Context, seqNr uint64, rawOutcome ocr3types }) } - reportableChannels, unreportableChannels := outcome.ReportableChannels(p.ProtocolVersion, p.DefaultMinReportIntervalNanoseconds) + reportableChannels, unreportableChannels := outcome.ReportableChannels(p.ProtocolVersion, p.DefaultMinReportIntervalNanoseconds, p.ReportCodecs, p.ChannelDefinitionOptsCache) if p.Config.VerboseLogging { p.Logger.Debugw("Reportable channels", "lifeCycleStage", outcome.LifeCycleStage, "reportableChannels", reportableChannels, "unreportableChannels", unreportableChannels, "stage", "Report", "seqNr", seqNr) } diff --git a/llo/reportcodecs/evm/report_codec_common.go b/llo/reportcodecs/evm/report_codec_common.go index 23b79a5..1084280 100644 --- a/llo/reportcodecs/evm/report_codec_common.go +++ b/llo/reportcodecs/evm/report_codec_common.go @@ -16,64 +16,17 @@ import ( ubig "github.com/smartcontractkit/chainlink-data-streams/llo/reportcodecs/evm/utils" ) -// TimestampPrecision represents the precision for timestamp conversion -type TimestampPrecision uint8 - -const ( - PrecisionSeconds TimestampPrecision = iota - PrecisionMilliseconds - PrecisionMicroseconds - PrecisionNanoseconds -) - -func (tp TimestampPrecision) MarshalJSON() ([]byte, error) { - var s string - switch tp { - case PrecisionSeconds: - s = "s" - case PrecisionMilliseconds: - s = "ms" - case PrecisionMicroseconds: - s = "us" - case PrecisionNanoseconds: - s = "ns" - default: - return nil, fmt.Errorf("invalid timestamp precision %d", tp) - } - return json.Marshal(s) -} - -// UnmarshalJSON unmarshals TimestampPrecision from JSON - used to unmarshal from the Opts structs. -func (tp *TimestampPrecision) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - switch s { - case "s": - *tp = PrecisionSeconds - case "ms": - *tp = PrecisionMilliseconds - case "us": - *tp = PrecisionMicroseconds - case "ns": - *tp = PrecisionNanoseconds - default: - return fmt.Errorf("invalid timestamp precision %q", s) - } - return nil -} // ConvertTimestamp converts a nanosecond timestamp to a specified precision. -func ConvertTimestamp(timestampNanos uint64, precision TimestampPrecision) uint64 { +func ConvertTimestamp(timestampNanos uint64, precision llo.TimeResolution) uint64 { switch precision { - case PrecisionSeconds: + case llo.ResolutionSeconds: return timestampNanos / 1e9 - case PrecisionMilliseconds: + case llo.ResolutionMilliseconds: return timestampNanos / 1e6 - case PrecisionMicroseconds: + case llo.ResolutionMicroseconds: return timestampNanos / 1e3 - case PrecisionNanoseconds: + case llo.ResolutionNanoseconds: return timestampNanos default: return timestampNanos diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go index 6db0077..4680d2d 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go @@ -20,6 +20,7 @@ import ( var ( _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpacked{} + _ llo.OptsParser = ReportCodecEVMABIEncodeUnpacked{} zero = big.NewInt(0) ) @@ -56,10 +57,10 @@ type ReportFormatEVMABIEncodeOpts struct { // top-level elements in this ABI array (stream 0 is always the native // token price and stream 1 is the link token price). ABI []ABIEncoder `json:"abi"` - // TimestampPrecision is the precision of the timestamps in the report. + // TimeResolution is the resolution of the timestamps in the report. // Seconds use uint32 ABI encoding, while milliseconds/microseconds/nanoseconds use uint64. // Defaults to "s" (seconds) if not specified. - TimestampPrecision TimestampPrecision `json:"timestampPrecision,omitempty"` + TimeResolution llo.TimeResolution `json:"timeResolution,omitempty"` } func (r *ReportFormatEVMABIEncodeOpts) Decode(opts []byte) error { @@ -105,8 +106,8 @@ func (r ReportCodecEVMABIEncodeUnpacked) Encode(report llo.Report, cd llotypes.C return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) } - validAfter := ConvertTimestamp(report.ValidAfterNanoseconds, opts.TimestampPrecision) - observationTimestamp := ConvertTimestamp(report.ObservationTimestampNanoseconds, opts.TimestampPrecision) + validAfter := ConvertTimestamp(report.ValidAfterNanoseconds, opts.TimeResolution) + observationTimestamp := ConvertTimestamp(report.ObservationTimestampNanoseconds, opts.TimeResolution) rf := BaseReportFields{ FeedID: opts.FeedID, @@ -117,7 +118,7 @@ func (r ReportCodecEVMABIEncodeUnpacked) Encode(report llo.Report, cd llotypes.C ExpiresAt: observationTimestamp + uint64(opts.ExpirationWindow), } - header, err := r.buildHeader(rf, opts.TimestampPrecision) + header, err := r.buildHeader(rf, opts.TimeResolution) if err != nil { return nil, fmt.Errorf("failed to build base report; %w", err) } @@ -205,7 +206,7 @@ func getBaseSchema(timestampType string) abi.Arguments { }) } -func (r ReportCodecEVMABIEncodeUnpacked) buildHeader(rf BaseReportFields, precision TimestampPrecision) ([]byte, error) { +func (r ReportCodecEVMABIEncodeUnpacked) buildHeader(rf BaseReportFields, precision llo.TimeResolution) ([]byte, error) { var merr error if rf.LinkFee == nil { merr = errors.Join(merr, errors.New("linkFee may not be nil")) @@ -223,7 +224,7 @@ func (r ReportCodecEVMABIEncodeUnpacked) buildHeader(rf BaseReportFields, precis var b []byte var err error - if precision == PrecisionSeconds { + if precision == llo.ResolutionSeconds { if rf.ValidFromTimestamp > math.MaxUint32 { return nil, fmt.Errorf("validFromTimestamp %d exceeds uint32 range", rf.ValidFromTimestamp) } @@ -257,3 +258,19 @@ func (r ReportCodecEVMABIEncodeUnpacked) buildHeader(rf BaseReportFields, precis } return b, nil } + +func (r ReportCodecEVMABIEncodeUnpacked) ParseOpts(opts []byte) (interface{}, error) { + var o ReportFormatEVMABIEncodeOpts + if err := json.Unmarshal(opts, &o); err != nil { + return nil, fmt.Errorf("failed to parse EVMABIEncodeUnpacked opts: %w", err) + } + return o, nil +} + +func (r ReportCodecEVMABIEncodeUnpacked) TimeResolution(parsedOpts interface{}) (llo.TimeResolution, error) { + opts, ok := parsedOpts.(ReportFormatEVMABIEncodeOpts) + if !ok { + return llo.ResolutionSeconds, fmt.Errorf("expected ReportFormatEVMABIEncodeOpts, got %T", parsedOpts) + } + return opts.TimeResolution, nil +} diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go index 8da3918..b72a555 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go @@ -6,6 +6,7 @@ import ( "math" "github.com/ethereum/go-ethereum/common" + "github.com/goccy/go-json" "github.com/smartcontractkit/chainlink-common/pkg/logger" llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" @@ -14,6 +15,7 @@ import ( var ( _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.OptsParser = ReportCodecEVMABIEncodeUnpackedExpr{} ) type ReportCodecEVMABIEncodeUnpackedExpr struct { @@ -49,8 +51,8 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) Encode(report llo.Report, cd llotyp return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) } - validAfter := ConvertTimestamp(report.ValidAfterNanoseconds, opts.TimestampPrecision) - observationTimestamp := ConvertTimestamp(report.ObservationTimestampNanoseconds, opts.TimestampPrecision) + validAfter := ConvertTimestamp(report.ValidAfterNanoseconds, opts.TimeResolution) + observationTimestamp := ConvertTimestamp(report.ObservationTimestampNanoseconds, opts.TimeResolution) rf := BaseReportFields{ FeedID: opts.FeedID, @@ -61,7 +63,7 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) Encode(report llo.Report, cd llotyp ExpiresAt: observationTimestamp + uint64(opts.ExpirationWindow), } - header, err := r.buildHeader(rf, opts.TimestampPrecision) + header, err := r.buildHeader(rf, opts.TimeResolution) if err != nil { return nil, fmt.Errorf("failed to build base report; %w", err) } @@ -91,7 +93,7 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) Verify(cd llotypes.ChannelDefinitio return nil } -func (r ReportCodecEVMABIEncodeUnpackedExpr) buildHeader(rf BaseReportFields, precision TimestampPrecision) ([]byte, error) { +func (r ReportCodecEVMABIEncodeUnpackedExpr) buildHeader(rf BaseReportFields, precision llo.TimeResolution) ([]byte, error) { var merr error if rf.LinkFee == nil { merr = errors.Join(merr, errors.New("linkFee may not be nil")) @@ -109,7 +111,7 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) buildHeader(rf BaseReportFields, pr var b []byte var err error - if precision == PrecisionSeconds { + if precision == llo.ResolutionSeconds { if rf.ValidFromTimestamp > math.MaxUint32 { return nil, fmt.Errorf("validFromTimestamp %d exceeds uint32 range", rf.ValidFromTimestamp) } @@ -143,3 +145,19 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) buildHeader(rf BaseReportFields, pr } return b, nil } + +func (r ReportCodecEVMABIEncodeUnpackedExpr) ParseOpts(opts []byte) (interface{}, error) { + var o ReportFormatEVMABIEncodeOpts + if err := json.Unmarshal(opts, &o); err != nil { + return nil, fmt.Errorf("failed to parse EVMABIEncodeUnpackedExpr opts: %w", err) + } + return o, nil +} + +func (r ReportCodecEVMABIEncodeUnpackedExpr) TimeResolution(parsedOpts interface{}) (llo.TimeResolution, error) { + opts, ok := parsedOpts.(ReportFormatEVMABIEncodeOpts) + if !ok { + return llo.ResolutionSeconds, fmt.Errorf("expected ReportFormatEVMABIEncodeOpts, got %T", parsedOpts) + } + return opts.TimeResolution, nil +} diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go index 68fac36..bb7ac19 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go @@ -41,8 +41,8 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - TimestampPrecision: PrecisionSeconds, - ABI: []ABIEncoder{}, + TimeResolution: llo.ResolutionSeconds, + ABI: []ABIEncoder{}, } serializedOpts, err := opts.Encode() require.NoError(t, err) @@ -110,10 +110,10 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - BaseUSDFee: sampleBaseUSDFee, - ExpirationWindow: sampleExpirationWindow, - FeedID: sampleFeedID, - TimestampPrecision: PrecisionSeconds, + BaseUSDFee: sampleBaseUSDFee, + ExpirationWindow: sampleExpirationWindow, + FeedID: sampleFeedID, + TimeResolution: llo.ResolutionSeconds, ABI: []ABIEncoder{ // benchmark price newSingleABIEncoder("int192", priceMultiplier), @@ -187,10 +187,10 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { }) t.Run("varying timestamp precision schemas", func(t *testing.T) { - runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier, marketDepthMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleDexBasedAssetPrice, sampleBaseMarketDepth, sampleQuoteMarketDepth decimal.Decimal, sampleTimestampPrecision TimestampPrecision) bool { + runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier, marketDepthMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleDexBasedAssetPrice, sampleBaseMarketDepth, sampleQuoteMarketDepth decimal.Decimal, sampleTimeResolution llo.TimeResolution) bool { // Determine timestamp type based on precision timestampType := "uint64" - if sampleTimestampPrecision == PrecisionSeconds { + if sampleTimeResolution == llo.ResolutionSeconds { timestampType = "uint32" } @@ -223,10 +223,10 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - BaseUSDFee: sampleBaseUSDFee, - ExpirationWindow: sampleExpirationWindow, - FeedID: sampleFeedID, - TimestampPrecision: sampleTimestampPrecision, + BaseUSDFee: sampleBaseUSDFee, + ExpirationWindow: sampleExpirationWindow, + FeedID: sampleFeedID, + TimeResolution: sampleTimeResolution, ABI: []ABIEncoder{ // benchmark price newSingleABIEncoder("int192", priceMultiplier), @@ -282,8 +282,8 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { } // Verify timestamps per precision type - expectedValidFrom := ConvertTimestamp(sampleValidAfterNanoseconds, sampleTimestampPrecision) + 1 - expectedObservationTimestamp := ConvertTimestamp(sampleObservationTimestampNanoseconds, sampleTimestampPrecision) + expectedValidFrom := ConvertTimestamp(sampleValidAfterNanoseconds, sampleTimeResolution) + 1 + expectedObservationTimestamp := ConvertTimestamp(sampleObservationTimestampNanoseconds, sampleTimeResolution) expectedExpiresAt := expectedObservationTimestamp + uint64(sampleExpirationWindow) if timestampType == "uint32" { checks = append(checks, @@ -317,7 +317,7 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { genDexBasedAssetPrice(), genMarketDepth(), genMarketDepth(), - genTimestampPrecision(), + genTimeResolution(), )) properties.TestingRun(t) }) @@ -435,13 +435,13 @@ func genMarketDepth() gopter.Gen { // TestReportCodecEVMABIEncodeUnpackedExpr_EncodeOpts func TestReportCodecEVMABIEncodeUnpackedExpr_EncodeOpts(t *testing.T) { - t.Run("zero value is PrecisionSeconds", func(t *testing.T) { - var defaultPrecision TimestampPrecision - assert.Equal(t, PrecisionSeconds, defaultPrecision, "zero value must be PrecisionSeconds for backward compatibility") - assert.Equal(t, TimestampPrecision(0), PrecisionSeconds, "PrecisionSeconds must be 0") + t.Run("zero value is llo.ResolutionSeconds", func(t *testing.T) { + var defaultPrecision llo.TimeResolution + assert.Equal(t, llo.ResolutionSeconds, defaultPrecision, "zero value must be llo.ResolutionSeconds for backward compatibility") + assert.Equal(t, llo.TimeResolution(0), llo.ResolutionSeconds, "llo.ResolutionSeconds must be 0") }) - t.Run("JSON opts without timestampPrecision defaults to seconds", func(t *testing.T) { + t.Run("JSON opts without timeResolution defaults to seconds", func(t *testing.T) { jsonConfig := `{ "baseUSDFee": "1.5", "expirationWindow": 3600, @@ -453,34 +453,34 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_EncodeOpts(t *testing.T) { err := opts.Decode([]byte(jsonConfig)) require.NoError(t, err) - assert.Equal(t, PrecisionSeconds, opts.TimestampPrecision) + assert.Equal(t, llo.ResolutionSeconds, opts.TimeResolution) }) - t.Run("JSON opts with timestampPrecision uses correct value", func(t *testing.T) { + t.Run("JSON opts with timeResolution uses correct value", func(t *testing.T) { testCases := []struct { name string jsonPrecision string - expectedPrecision TimestampPrecision + expectedPrecision llo.TimeResolution }{ { name: "seconds", jsonPrecision: "s", - expectedPrecision: PrecisionSeconds, + expectedPrecision: llo.ResolutionSeconds, }, { name: "milliseconds", jsonPrecision: "ms", - expectedPrecision: PrecisionMilliseconds, + expectedPrecision: llo.ResolutionMilliseconds, }, { name: "microseconds", jsonPrecision: "us", - expectedPrecision: PrecisionMicroseconds, + expectedPrecision: llo.ResolutionMicroseconds, }, { name: "nanoseconds", jsonPrecision: "ns", - expectedPrecision: PrecisionNanoseconds, + expectedPrecision: llo.ResolutionNanoseconds, }, } @@ -491,14 +491,14 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_EncodeOpts(t *testing.T) { "expirationWindow": 3600, "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", "abi": [{"type": "uint192"}], - "timestampPrecision": "%s" + "timeResolution": "%s" }`, tc.jsonPrecision) var opts ReportFormatEVMABIEncodeOpts err := opts.Decode([]byte(jsonConfig)) require.NoError(t, err) - assert.Equal(t, tc.expectedPrecision, opts.TimestampPrecision) + assert.Equal(t, tc.expectedPrecision, opts.TimeResolution) }) } }) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go index 1f20178..4afdd2c 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go @@ -51,11 +51,11 @@ func TestReportFormatEVMABIEncodeOpts_Decode_Encode_properties(t *testing.T) { properties.Property("Encodes values", prop.ForAll( runTest, gen.StrictStruct(reflect.TypeOf(&ReportFormatEVMABIEncodeOpts{}), map[string]gopter.Gen{ - "BaseUSDFee": genBaseUSDFee(), - "ExpirationWindow": genExpirationWindow(), - "FeedID": genFeedID(), - "ABI": genABI(), - "TimestampPrecision": genTimestampPrecision(), + "BaseUSDFee": genBaseUSDFee(), + "ExpirationWindow": genExpirationWindow(), + "FeedID": genFeedID(), + "ABI": genABI(), + "TimeResolution": genTimeResolution(), }))) properties.TestingRun(t) @@ -115,10 +115,10 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - BaseUSDFee: sampleBaseUSDFee, - ExpirationWindow: sampleExpirationWindow, - FeedID: sampleFeedID, - TimestampPrecision: PrecisionSeconds, + BaseUSDFee: sampleBaseUSDFee, + ExpirationWindow: sampleExpirationWindow, + FeedID: sampleFeedID, + TimeResolution: llo.ResolutionSeconds, ABI: []ABIEncoder{ // benchmark price newSingleABIEncoder("int192", priceMultiplier), @@ -230,10 +230,10 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - BaseUSDFee: sampleBaseUSDFee, - ExpirationWindow: sampleExpirationWindow, - FeedID: sampleFeedID, - TimestampPrecision: PrecisionSeconds, + BaseUSDFee: sampleBaseUSDFee, + ExpirationWindow: sampleExpirationWindow, + FeedID: sampleFeedID, + TimeResolution: llo.ResolutionSeconds, ABI: []ABIEncoder{ // market status newSingleABIEncoder("uint32", nil), @@ -334,10 +334,10 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - BaseUSDFee: sampleBaseUSDFee, - ExpirationWindow: sampleExpirationWindow, - FeedID: sampleFeedID, - TimestampPrecision: PrecisionSeconds, + BaseUSDFee: sampleBaseUSDFee, + ExpirationWindow: sampleExpirationWindow, + FeedID: sampleFeedID, + TimeResolution: llo.ResolutionSeconds, ABI: []ABIEncoder{ // benchmark price newSingleABIEncoder("int192", priceMultiplier), @@ -409,10 +409,10 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { }) t.Run("varying timestamp precision schemas", func(t *testing.T) { - runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleBenchmarkPrice decimal.Decimal, sampleTimestampPrecision TimestampPrecision) bool { + runTest := func(sampleFeedID common.Hash, sampleObservationTimestampNanoseconds, sampleValidAfterNanoseconds uint64, sampleExpirationWindow uint32, priceMultiplier *ubig.Big, sampleBaseUSDFee, sampleLinkBenchmarkPrice, sampleNativeBenchmarkPrice, sampleBenchmarkPrice decimal.Decimal, sampleTimeResolution llo.TimeResolution) bool { // Determine timestamp type based on precision timestampType := "uint64" - if sampleTimestampPrecision == PrecisionSeconds { + if sampleTimeResolution == llo.ResolutionSeconds { timestampType = "uint32" } @@ -441,10 +441,10 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - BaseUSDFee: sampleBaseUSDFee, - ExpirationWindow: sampleExpirationWindow, - FeedID: sampleFeedID, - TimestampPrecision: sampleTimestampPrecision, + BaseUSDFee: sampleBaseUSDFee, + ExpirationWindow: sampleExpirationWindow, + FeedID: sampleFeedID, + TimeResolution: sampleTimeResolution, ABI: []ABIEncoder{ newSingleABIEncoder("int192", priceMultiplier), }, @@ -486,8 +486,8 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { } // Verify timestamps per precision type - expectedValidFrom := ConvertTimestamp(sampleValidAfterNanoseconds, sampleTimestampPrecision) + 1 - expectedObservationTimestamp := ConvertTimestamp(sampleObservationTimestampNanoseconds, sampleTimestampPrecision) + expectedValidFrom := ConvertTimestamp(sampleValidAfterNanoseconds, sampleTimeResolution) + 1 + expectedObservationTimestamp := ConvertTimestamp(sampleObservationTimestampNanoseconds, sampleTimeResolution) expectedExpiresAt := expectedObservationTimestamp + uint64(sampleExpirationWindow) if timestampType == "uint32" { checks = append(checks, @@ -518,7 +518,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { genLinkBenchmarkPrice(), genNativeBenchmarkPrice(), genBenchmarkPrice(), - genTimestampPrecision(), + genTimeResolution(), )) properties.TestingRun(t) }) @@ -560,10 +560,10 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { } opts := ReportFormatEVMABIEncodeOpts{ - BaseUSDFee: sampleBaseUSDFee, - ExpirationWindow: sampleExpirationWindow, - FeedID: sampleFeedID, - TimestampPrecision: PrecisionSeconds, + BaseUSDFee: sampleBaseUSDFee, + ExpirationWindow: sampleExpirationWindow, + FeedID: sampleFeedID, + TimeResolution: llo.ResolutionSeconds, ABI: []ABIEncoder{ newSingleABIEncoder("int192", nil), newSingleABIEncoder("uint32", nil), @@ -737,8 +737,8 @@ func genTimestampThatFitsUint32Seconds() gopter.Gen { }) } -func genTimestampPrecision() gopter.Gen { - return gen.OneConstOf(PrecisionSeconds, PrecisionMilliseconds, PrecisionMicroseconds, PrecisionNanoseconds) +func genTimeResolution() gopter.Gen { + return gen.OneConstOf(llo.ResolutionSeconds, llo.ResolutionMilliseconds, llo.ResolutionMicroseconds, llo.ResolutionNanoseconds) } func genExpirationWindow() gopter.Gen { @@ -936,13 +936,13 @@ func newSingleABIEncoder(t string, m *ubig.Big) ABIEncoder { // TestReportCodecEVMABIEncodeUnpacked_EncodeOpts func TestReportCodecEVMABIEncodeUnpacked_EncodeOpts(t *testing.T) { - t.Run("zero value is PrecisionSeconds", func(t *testing.T) { - var defaultPrecision TimestampPrecision - assert.Equal(t, PrecisionSeconds, defaultPrecision, "zero value must be PrecisionSeconds for backward compatibility") - assert.Equal(t, TimestampPrecision(0), PrecisionSeconds, "PrecisionSeconds must be 0") + t.Run("zero value is llo.ResolutionSeconds", func(t *testing.T) { + var defaultPrecision llo.TimeResolution + assert.Equal(t, llo.ResolutionSeconds, defaultPrecision, "zero value must be llo.ResolutionSeconds for backward compatibility") + assert.Equal(t, llo.TimeResolution(0), llo.ResolutionSeconds, "llo.ResolutionSeconds must be 0") }) - t.Run("JSON opts without timestampPrecision defaults to seconds", func(t *testing.T) { + t.Run("JSON opts without timeResolution defaults to seconds", func(t *testing.T) { jsonConfig := `{ "baseUSDFee": "1.5", "expirationWindow": 3600, @@ -954,34 +954,34 @@ func TestReportCodecEVMABIEncodeUnpacked_EncodeOpts(t *testing.T) { err := opts.Decode([]byte(jsonConfig)) require.NoError(t, err) - assert.Equal(t, PrecisionSeconds, opts.TimestampPrecision) + assert.Equal(t, llo.ResolutionSeconds, opts.TimeResolution) }) - t.Run("JSON opts with timestampPrecision uses correct value", func(t *testing.T) { + t.Run("JSON opts with timeResolution uses correct value", func(t *testing.T) { testCases := []struct { name string jsonPrecision string - expectedPrecision TimestampPrecision + expectedPrecision llo.TimeResolution }{ { name: "seconds", jsonPrecision: "s", - expectedPrecision: PrecisionSeconds, + expectedPrecision: llo.ResolutionSeconds, }, { name: "milliseconds", jsonPrecision: "ms", - expectedPrecision: PrecisionMilliseconds, + expectedPrecision: llo.ResolutionMilliseconds, }, { name: "microseconds", jsonPrecision: "us", - expectedPrecision: PrecisionMicroseconds, + expectedPrecision: llo.ResolutionMicroseconds, }, { name: "nanoseconds", jsonPrecision: "ns", - expectedPrecision: PrecisionNanoseconds, + expectedPrecision: llo.ResolutionNanoseconds, }, } @@ -992,14 +992,14 @@ func TestReportCodecEVMABIEncodeUnpacked_EncodeOpts(t *testing.T) { "expirationWindow": 3600, "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", "abi": [{"type": "uint192"}], - "timestampPrecision": "%s" + "timeResolution": "%s" }`, tc.jsonPrecision) var opts ReportFormatEVMABIEncodeOpts err := opts.Decode([]byte(jsonConfig)) require.NoError(t, err) - assert.Equal(t, tc.expectedPrecision, opts.TimestampPrecision) + assert.Equal(t, tc.expectedPrecision, opts.TimeResolution) }) } }) diff --git a/llo/reportcodecs/evm/report_codec_premium_legacy.go b/llo/reportcodecs/evm/report_codec_premium_legacy.go index db7257c..096cff4 100644 --- a/llo/reportcodecs/evm/report_codec_premium_legacy.go +++ b/llo/reportcodecs/evm/report_codec_premium_legacy.go @@ -25,6 +25,7 @@ import ( var ( _ llo.ReportCodec = ReportCodecPremiumLegacy{} + _ llo.OptsParser = ReportCodecPremiumLegacy{} PayloadTypes = getPayloadTypes() ) @@ -270,3 +271,16 @@ func LegacyReportContext(cd ocr2types.ConfigDigest, seqNr uint64, donID uint32) ExtraHash: LLOExtraHash(donID), // ExtraHash is always zero for mercury, we use LLOExtraHash here to differentiate from the legacy plugin }, nil } + +func (r ReportCodecPremiumLegacy) ParseOpts(opts []byte) (interface{}, error) { + var o ReportFormatEVMPremiumLegacyOpts + if err := json.Unmarshal(opts, &o); err != nil { + return nil, fmt.Errorf("failed to parse EVMPremiumLegacy opts: %w", err) + } + return o, nil +} + +func (r ReportCodecPremiumLegacy) TimeResolution(parsedOpts interface{}) (llo.TimeResolution, error) { + // Premium legacy always uses seconds resolution + return llo.ResolutionSeconds, nil +} diff --git a/llo/types.go b/llo/types.go index af6c4df..0d6c0a1 100644 --- a/llo/types.go +++ b/llo/types.go @@ -1,6 +1,9 @@ package llo import ( + "fmt" + + "github.com/goccy/go-json" "github.com/smartcontractkit/libocr/offchainreporting2/types" "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" @@ -31,6 +34,16 @@ type ReportCodec interface { Verify(llotypes.ChannelDefinition) error } +type OptsParser interface { + // ParseOpts parses the ReportCodec's opts and returns an interface{} which can be type asserted + // to the specific ReportCodec's opts type. + // Use when parsing of Opts is considered too expensive to do repeatedly. + ParseOpts(opts []byte) (interface{}, error) + + // TimeResolution returns the time resolution available in the parsed opts + TimeResolution(parsedOpts interface{}) (TimeResolution, error) +} + type ChannelDefinitionWithID struct { llotypes.ChannelDefinition ChannelID llotypes.ChannelID @@ -45,3 +58,50 @@ type Transmitter interface { // - FromAccount() should return CSA public key ocr3types.ContractTransmitter[llotypes.ReportInfo] } + +// TimeResolution can be used to represent the resolution of an epoch timestamp +type TimeResolution uint8 + +const ( + ResolutionSeconds TimeResolution = iota + ResolutionMilliseconds + ResolutionMicroseconds + ResolutionNanoseconds +) + +func (tp TimeResolution) MarshalJSON() ([]byte, error) { + var s string + switch tp { + case ResolutionSeconds: + s = "s" + case ResolutionMilliseconds: + s = "ms" + case ResolutionMicroseconds: + s = "us" + case ResolutionNanoseconds: + s = "ns" + default: + return nil, fmt.Errorf("invalid time resolution %d", tp) + } + return json.Marshal(s) +} + +func (tp *TimeResolution) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + switch s { + case "s": + *tp = ResolutionSeconds + case "ms": + *tp = ResolutionMilliseconds + case "us": + *tp = ResolutionMicroseconds + case "ns": + *tp = ResolutionNanoseconds + default: + return fmt.Errorf("invalid time resolution %q", s) + } + return nil +} From ac01b02531329d7538b207fbd498fccfd09e895f Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 4 Dec 2025 21:02:45 +0000 Subject: [PATCH 04/13] Test cases for usage of ChannelDefinitionOptsCache --- llo/plugin_outcome.go | 2 +- llo/plugin_outcome_test.go | 185 +++++++++++++++++++++++++++++++------ 2 files changed, 160 insertions(+), 27 deletions(-) diff --git a/llo/plugin_outcome.go b/llo/plugin_outcome.go index 0ff5a9d..4c6de84 100644 --- a/llo/plugin_outcome.go +++ b/llo/plugin_outcome.go @@ -469,7 +469,7 @@ func IsSecondsResolution(channelID llotypes.ChannelID, codec ReportCodec, optsCa if err != nil { // This should not happen since caching the cached opts should logically ensure that this channel's opts // are valid for channels report codec. If this error occurs it would indicate a wrong codec/opts mismatch. - return false, fmt.Errorf("failed to parse time resolution from opts (wrong codec/opts mismatch?) :%w", err) + return false, fmt.Errorf("invariant violation: failed to parse time resolution from opts (wrong codec/opts mismatch?) :%w", err) } return resolution == ResolutionSeconds, nil } diff --git a/llo/plugin_outcome_test.go b/llo/plugin_outcome_test.go index a7bfc19..33272b9 100644 --- a/llo/plugin_outcome_test.go +++ b/llo/plugin_outcome_test.go @@ -39,8 +39,9 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { DonID: 10000043, ConfigDigest: types.ConfigDigest{1, 2, 3, 4}, ReportCodecs: map[llotypes.ReportFormat]ReportCodec{ - llotypes.ReportFormatJSON: mockCodec{timeResolution: ResolutionNanoseconds}, - llotypes.ReportFormatEVMPremiumLegacy: mockCodec{timeResolution: ResolutionSeconds}, + llotypes.ReportFormatEVMABIEncodeUnpacked: mockCodec{timeResolution: ResolutionNanoseconds}, + llotypes.ReportFormatEVMPremiumLegacy: mockCodec{timeResolution: ResolutionSeconds}, + llotypes.ReportFormatJSON: basicCodec{}, }, ChannelDefinitionOptsCache: NewChannelDefinitionOptsCache(), } @@ -86,10 +87,16 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { t.Run("channel definitions", func(t *testing.T) { t.Run("adds a new channel definition if there are enough votes", func(t *testing.T) { + // Use EVMPremiumLegacy which implements OptsParser (unlike JSON format) newCd := llotypes.ChannelDefinition{ - ReportFormat: llotypes.ReportFormat(2), + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, Streams: []llotypes.Stream{{StreamID: 1, Aggregator: llotypes.AggregatorMedian}, {StreamID: 2, Aggregator: llotypes.AggregatorMedian}, {StreamID: 3, Aggregator: llotypes.AggregatorMedian}}, } + + // Verify cache is empty before + _, cached := p.ChannelDefinitionOptsCache.Get(42) + assert.False(t, cached, "cache should be empty before channel is added") + obs, err := p.ObservationCodec.Encode(Observation{ UpdateChannelDefinitions: map[llotypes.ChannelID]llotypes.ChannelDefinition{ 42: newCd, @@ -111,13 +118,28 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { require.NoError(t, err) assert.Equal(t, newCd, decoded.ChannelDefinitions[42]) + + // Verify cache was populated after channel added + cachedOpts, cached := p.ChannelDefinitionOptsCache.Get(42) + assert.True(t, cached, "cache should be populated after channel is added") + assert.NotNil(t, cachedOpts, "cached opts should not be nil") }) t.Run("replaces an existing channel definition if there are enough votes", func(t *testing.T) { + // Use different formats to verify cache Set() was actually called during update + oldCd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, // seconds resolution + Streams: []llotypes.Stream{{StreamID: 2, Aggregator: llotypes.AggregatorMedian}, {StreamID: 3, Aggregator: llotypes.AggregatorMedian}, {StreamID: 4, Aggregator: llotypes.AggregatorMedian}}, + } newCd := llotypes.ChannelDefinition{ - ReportFormat: llotypes.ReportFormat(2), + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpacked, // nanoseconds resolution Streams: []llotypes.Stream{{StreamID: 1, Aggregator: llotypes.AggregatorQuote}, {StreamID: 2, Aggregator: llotypes.AggregatorMedian}, {StreamID: 3, Aggregator: llotypes.AggregatorMedian}}, } + + // Pre-populate cache with old definition + populateCache(t, p.ChannelDefinitionOptsCache, 42, oldCd, p.ReportCodecs) + oldCachedOpts, _ := p.ChannelDefinitionOptsCache.Get(42) + obs, err := p.ObservationCodec.Encode(Observation{ UpdateChannelDefinitions: map[llotypes.ChannelID]llotypes.ChannelDefinition{ 42: newCd, @@ -135,10 +157,7 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { previousOutcome, err := p.OutcomeCodec.Encode(Outcome{ ChannelDefinitions: map[llotypes.ChannelID]llotypes.ChannelDefinition{ - 42: { - ReportFormat: llotypes.ReportFormat(1), - Streams: []llotypes.Stream{{StreamID: 2, Aggregator: llotypes.AggregatorMedian}, {StreamID: 3, Aggregator: llotypes.AggregatorMedian}, {StreamID: 4, Aggregator: llotypes.AggregatorMedian}}, - }, + 42: oldCd, }, }) require.NoError(t, err) @@ -150,17 +169,29 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { require.NoError(t, err) assert.Equal(t, newCd, decoded.ChannelDefinitions[42]) + + // Verify cache was updated by checking cached value changed + newCachedOpts, cached := p.ChannelDefinitionOptsCache.Get(42) + assert.True(t, cached, "optsCache should be populated after update") + assert.NotNil(t, newCachedOpts, "cached opts should not be nil after update") + // The old/new cached opts must be different because the report formats are different + assert.NotEqual(t, oldCachedOpts, newCachedOpts, "cached opts must change when format changes (proves Set was called)") }) t.Run("removes a channel definition if there are enough votes", func(t *testing.T) { - t.Skip("removal votes are not implemented yet") - newCd := llotypes.ChannelDefinition{ - ReportFormat: llotypes.ReportFormat(2), + existingCd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, Streams: []llotypes.Stream{{StreamID: 1, Aggregator: llotypes.AggregatorMedian}, {StreamID: 2, Aggregator: llotypes.AggregatorMedian}, {StreamID: 3, Aggregator: llotypes.AggregatorMedian}}, + Opts: []byte(`{"existing":"channel"}`), } + + // Pre-populate cache with existing channel + populateCache(t, p.ChannelDefinitionOptsCache, 42, existingCd, p.ReportCodecs) + + // Vote to remove channel 42 obs, err := p.ObservationCodec.Encode(Observation{ - UpdateChannelDefinitions: map[llotypes.ChannelID]llotypes.ChannelDefinition{ - 42: newCd, + RemoveChannelIDs: map[llotypes.ChannelID]struct{}{ + 42: {}, }, }) require.NoError(t, err) @@ -172,13 +203,27 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { Observer: commontypes.OracleID(i), }) } - outcome, err := p.Outcome(ctx, ocr3types.OutcomeContext{SeqNr: 2}, types.Query{}, aos) + + previousOutcome, err := p.OutcomeCodec.Encode(Outcome{ + ChannelDefinitions: map[llotypes.ChannelID]llotypes.ChannelDefinition{ + 42: existingCd, + }, + }) + require.NoError(t, err) + + // Will process votes to remove channel 42 + outcome, err := p.Outcome(ctx, ocr3types.OutcomeContext{PreviousOutcome: previousOutcome, SeqNr: 2}, types.Query{}, aos) require.NoError(t, err) decoded, err := p.OutcomeCodec.Decode(outcome) require.NoError(t, err) - assert.Equal(t, newCd, decoded.ChannelDefinitions[42]) + // Channel should be removed from definitions + assert.NotContains(t, decoded.ChannelDefinitions, llotypes.ChannelID(42)) + + // Verify cache entry was deleted + _, cached := p.ChannelDefinitionOptsCache.Get(42) + assert.False(t, cached, "cache should not contain channel after removal") }) t.Run("does not add channels beyond MaxOutcomeChannelDefinitionsLength", func(t *testing.T) { @@ -729,8 +774,16 @@ func Test_MakeChannelHash(t *testing.T) { }) } +func populateCache(t *testing.T, cache ChannelDefinitionOptsCache, channelID llotypes.ChannelID, cd llotypes.ChannelDefinition, codecs map[llotypes.ReportFormat]ReportCodec) { + err := cache.Set(channelID, cd.Opts, codecs[cd.ReportFormat]) + require.NoError(t, err) + _, cached := cache.Get(channelID) + require.True(t, cached, "cache should be populated after Set") +} + type mockCodec struct { - timeResolution TimeResolution + timeResolution TimeResolution + timeResolutionErr error } var ( @@ -738,35 +791,43 @@ var ( _ OptsParser = mockCodec{} ) -func (mockCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { - return nil, nil -} +func (mockCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { return nil, nil } -func (mockCodec) Verify(llotypes.ChannelDefinition) error { - return nil -} +func (mockCodec) Verify(llotypes.ChannelDefinition) error { return nil } func (c mockCodec) ParseOpts(opts []byte) (interface{}, error) { - // TODO do we need to parse opts in anyway here? + // Ignoring opts bytes parsing here is acceptable for integration tests because + // that is a responsibility of the codec implementation. + // Real codec unit tests should verify actual parsing behavior return c, nil } func (c mockCodec) TimeResolution(parsedOpts interface{}) (TimeResolution, error) { + if c.timeResolutionErr != nil { + return 0, c.timeResolutionErr + } if tc, ok := parsedOpts.(mockCodec); ok { return tc.timeResolution, nil } return c.timeResolution, nil } +// basicCodec is a codec that does not implement OptsParser +type basicCodec struct{} + +var _ ReportCodec = basicCodec{} + +func (basicCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { return nil, nil } +func (basicCodec) Verify(llotypes.ChannelDefinition) error { return nil } + func Test_Outcome_Methods(t *testing.T) { - // Cache for parsed channel definition opts optsCache := NewChannelDefinitionOptsCache() - // Use test codecs that mimic real codec behavior codecs := map[llotypes.ReportFormat]ReportCodec{ llotypes.ReportFormat(0): mockCodec{timeResolution: ResolutionNanoseconds}, llotypes.ReportFormatEVMPremiumLegacy: mockCodec{timeResolution: ResolutionSeconds}, llotypes.ReportFormatEVMABIEncodeUnpacked: mockCodec{timeResolution: ResolutionNanoseconds}, llotypes.ReportFormatEVMABIEncodeUnpackedExpr: mockCodec{timeResolution: ResolutionNanoseconds}, + llotypes.ReportFormatJSON: basicCodec{}, } t.Run("protocol version 0", func(t *testing.T) { @@ -867,6 +928,42 @@ func Test_Outcome_Methods(t *testing.T) { // zero report cadence allows overlaps (but still respects seconds resolution boundary) outcome.ValidAfterNanoseconds = map[llotypes.ChannelID]uint64{cid: obsTSNanos - uint64(1*time.Second)} require.Nil(t, outcome.IsReportable(cid, 1, 0, codecs, optsCache)) + + t.Run("returns error when TimeResolution fails (codec/opts mismatch)", func(t *testing.T) { + cid := llotypes.ChannelID(3) + obsTSNanos := uint64(time.Unix(1726670490, 1000).UnixNano()) //nolint:gosec // time won't be negative + + outcome := Outcome{ + LifeCycleStage: LifeCycleStageProduction, + ObservationTimestampNanoseconds: obsTSNanos, + ChannelDefinitions: map[llotypes.ChannelID]llotypes.ChannelDefinition{ + cid: {ReportFormat: llotypes.ReportFormatEVMPremiumLegacy}, + }, + ValidAfterNanoseconds: map[llotypes.ChannelID]uint64{ + cid: obsTSNanos - uint64(2*time.Second), + }, + } + + // Create a mockCodec that will return an error when TimeResolution is called + codecWithError := mockCodec{ + timeResolution: ResolutionSeconds, + timeResolutionErr: fmt.Errorf("could not marshall structure"), + } + codecsWithError := map[llotypes.ReportFormat]ReportCodec{ + llotypes.ReportFormatEVMPremiumLegacy: codecWithError, + } + + // Populate cache with opts that will be mismatched with the codec + // This simulates a scenario where the cache returnes Opts that the Codec cannot parse + // this mimics if the outcome method improperly uses the cache or the scenario where the cache itself fails + cd := outcome.ChannelDefinitions[cid] + err := optsCache.Set(cid, cd.Opts, codecWithError) + require.NoError(t, err) + + // IsReportable should return an error about the invariant violation + err = outcome.IsReportable(cid, 1, uint64(100*time.Millisecond), codecsWithError, optsCache) + require.EqualError(t, err, "ChannelID: 3; Reason: IsReportable=false; failed to determine time resolution; Err: invariant violation: failed to parse time resolution from opts (wrong codec/opts mismatch?) :could not marshall structure") + }) }) t.Run("IsReportable with seconds resolution", func(t *testing.T) { outcome := Outcome{} @@ -883,7 +980,7 @@ func Test_Outcome_Methods(t *testing.T) { cid: obsTSNanos - uint64(500*time.Millisecond), } - // OptsCache should be populated after successful channel voting + // OptsCache should be populated after successful channel voting cd := outcome.ChannelDefinitions[cid] _ = optsCache.Set(cid, cd.Opts, codecs[cd.ReportFormat]) @@ -899,6 +996,42 @@ func Test_Outcome_Methods(t *testing.T) { // if cadence is 5s, if time is < 5s, does not report because cadence hasn't elapsed require.EqualError(t, outcome.IsReportable(cid, 1, uint64(5*time.Second), codecs, optsCache), "ChannelID: 1; Reason: IsReportable=false; not valid yet (ObservationTimestampNanoseconds=1726670490999999999, validAfterNanoseconds=1726670489999999999, minReportInterval=5000000000); 4.000000 seconds (4000000000ns) until reportable") }) + + t.Run("IsSecondsResolution returns false when codec does not implement OptsParser", func(t *testing.T) { + cid := llotypes.ChannelID(2) + + cd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatJSON, + } + + // Cannot populate cache because codec does not implement OptsParser + err := optsCache.Set(cid, cd.Opts, codecs[cd.ReportFormat]) + require.NoError(t, err) + _, cached := optsCache.Get(cid) + require.False(t, cached, "cache should not be populated after Set (Codec does not implement OptsParser)") + + // ReportFormatJSON is using a codec which does not implement OptsParser - IsSecondsResolution returns false + // indicating to use the default time resolution + isSecondsResolution, err := IsSecondsResolution(cid, codecs[llotypes.ReportFormatJSON], optsCache) + require.NoError(t, err) + assert.False(t, isSecondsResolution) + }) + + t.Run("IsSecondsResolution returns true when codec requires seconds resolution", func(t *testing.T) { + cid := llotypes.ChannelID(2) + + cd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, + } + + populateCache(t, optsCache, cid, cd, codecs) + + isSecondsResolution, err := IsSecondsResolution(cid, codecs[llotypes.ReportFormatEVMPremiumLegacy], optsCache) + require.NoError(t, err) + assert.True(t, isSecondsResolution) + + }) + t.Run("ReportableChannels", func(t *testing.T) { defaultMinReportInterval := uint64(1 * time.Second) From 8ea6fbbbf96e25b6b3862d14268c6044ae27af56 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 4 Dec 2025 14:27:33 -0700 Subject: [PATCH 05/13] split OptsParser from OptsProvider interfaces --- llo/plugin_outcome.go | 6 ++-- llo/plugin_outcome_test.go | 5 ++-- .../report_codec_evm_abi_encode_unpacked.go | 5 ++-- ...port_codec_evm_abi_encode_unpacked_expr.go | 5 ++-- .../evm/report_codec_premium_legacy.go | 7 +++-- llo/stream_calculated.go | 6 ++-- llo/types.go | 28 ++++++++++++++++--- 7 files changed, 42 insertions(+), 20 deletions(-) diff --git a/llo/plugin_outcome.go b/llo/plugin_outcome.go index 4c6de84..68393b6 100644 --- a/llo/plugin_outcome.go +++ b/llo/plugin_outcome.go @@ -464,8 +464,8 @@ func IsSecondsResolution(channelID llotypes.ChannelID, codec ReportCodec, optsCa // Try to determine the time resolution from the channel definition opts if optsCache != nil { if cachedOpts, cached := optsCache.Get(channelID); cached { - if optsParser, ok := codec.(OptsParser); ok { - resolution, err := optsParser.TimeResolution(cachedOpts) + if timeResProvider, ok := codec.(TimeResolutionProvider); ok { + resolution, err := timeResProvider.TimeResolution(cachedOpts) if err != nil { // This should not happen since caching the cached opts should logically ensure that this channel's opts // are valid for channels report codec. If this error occurs it would indicate a wrong codec/opts mismatch. @@ -476,7 +476,7 @@ func IsSecondsResolution(channelID llotypes.ChannelID, codec ReportCodec, optsCa } } - // Fall back to protocol default time resolution when codecs don't implement OptsParser + // Fall back to protocol default time resolution when codecs don't implement TimeResolutionProvider return false, nil } diff --git a/llo/plugin_outcome_test.go b/llo/plugin_outcome_test.go index 33272b9..3ae4e34 100644 --- a/llo/plugin_outcome_test.go +++ b/llo/plugin_outcome_test.go @@ -787,8 +787,9 @@ type mockCodec struct { } var ( - _ ReportCodec = mockCodec{} - _ OptsParser = mockCodec{} + _ ReportCodec = mockCodec{} + _ OptsParser = mockCodec{} + _ TimeResolutionProvider = mockCodec{} ) func (mockCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { return nil, nil } diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go index 4680d2d..5e0a62f 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go @@ -19,8 +19,9 @@ import ( ) var ( - _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpacked{} - _ llo.OptsParser = ReportCodecEVMABIEncodeUnpacked{} + _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpacked{} + _ llo.OptsParser = ReportCodecEVMABIEncodeUnpacked{} + _ llo.TimeResolutionProvider = ReportCodecEVMABIEncodeUnpacked{} zero = big.NewInt(0) ) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go index b72a555..c86eee3 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go @@ -14,8 +14,9 @@ import ( ) var ( - _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpackedExpr{} - _ llo.OptsParser = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.OptsParser = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.TimeResolutionProvider = ReportCodecEVMABIEncodeUnpackedExpr{} ) type ReportCodecEVMABIEncodeUnpackedExpr struct { diff --git a/llo/reportcodecs/evm/report_codec_premium_legacy.go b/llo/reportcodecs/evm/report_codec_premium_legacy.go index 096cff4..5094e26 100644 --- a/llo/reportcodecs/evm/report_codec_premium_legacy.go +++ b/llo/reportcodecs/evm/report_codec_premium_legacy.go @@ -24,9 +24,10 @@ import ( ) var ( - _ llo.ReportCodec = ReportCodecPremiumLegacy{} - _ llo.OptsParser = ReportCodecPremiumLegacy{} - PayloadTypes = getPayloadTypes() + _ llo.ReportCodec = ReportCodecPremiumLegacy{} + _ llo.OptsParser = ReportCodecPremiumLegacy{} + _ llo.TimeResolutionProvider = ReportCodecPremiumLegacy{} + PayloadTypes = getPayloadTypes() ) func getPayloadTypes() abi.Arguments { diff --git a/llo/stream_calculated.go b/llo/stream_calculated.go index f5223f5..6bb1c3c 100644 --- a/llo/stream_calculated.go +++ b/llo/stream_calculated.go @@ -563,10 +563,8 @@ func (p *Plugin) ProcessCalculatedStreams(outcome *Outcome) { continue } - // TODO: we can potentially cache the opts for each channel definition - // and avoid unmarshalling the options on outcome. - // for now keep it simple as this will require invalidating on - // channel definitions updates. + // TODO: Use ChannelDefinitionOptsCache to avoid repeated JSON unmarshaling. + // Type assert cached opts to evm.ReportFormatEVMABIEncodeOpts and access ABI field directly. copt := opts{} if err := json.Unmarshal(cd.Opts, &copt); err != nil { p.Logger.Errorw("failed to unmarshal channel definition options", "channelID", cid, "error", err) diff --git a/llo/types.go b/llo/types.go index 0d6c0a1..4187395 100644 --- a/llo/types.go +++ b/llo/types.go @@ -34,13 +34,33 @@ type ReportCodec interface { Verify(llotypes.ChannelDefinition) error } +// OptsParser parses raw channel opts bytes into a codec-specific structure. +// ReportCodecs may implement this interface to enable caching of parsed opts +// to avoid repeated unmarshalling of Opts bytes. For example unmarshalling +// from JSON is much more expensive than peforming a type assertion on an `interface{}` type. +// Since not all Codec Opts might have the same Options - you can create +// `OptsProvider` interfaces where needed. For Example `TimeResolutionProvider` or `ABIProvider` +// and the Codecs can implement all interfaces that match the specific Opt. type OptsParser interface { - // ParseOpts parses the ReportCodec's opts and returns an interface{} which can be type asserted - // to the specific ReportCodec's opts type. - // Use when parsing of Opts is considered too expensive to do repeatedly. + // ParseOpts parses the raw opts bytes and returns a codec-specific + // parsed opts structure as interface{}. + // Use the returned interface to type assert to the specific Opts type. + // For Example: + // opts, err := codec.ParseOpts(optsBytes) + // if err != nil { + // return nil, err + // } + // optsProvider, ok := opts.(OptsProvider) // where OptsProvider is the interface you created for the specific Opts type + // if !ok { + // return nil, fmt.Errorf("invalid opts type") + // } ParseOpts(opts []byte) (interface{}, error) +} - // TimeResolution returns the time resolution available in the parsed opts +// TimeResolutionProvider extracts time resolution information from codec opts +type TimeResolutionProvider interface { + // TimeResolution returns the time resolution from the parsed opts. + // The parsedOpts must be the value returned by OptsParser.ParseOpts. TimeResolution(parsedOpts interface{}) (TimeResolution, error) } From b66712d8e643721f988637cdba8299e542d5cf41 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 4 Dec 2025 16:58:05 -0700 Subject: [PATCH 06/13] updates ReportCodec interface to accept parsed channel definition. channelDefinitionOptsCache add locking to operations to protect cache validity --- llo/channel_definitions.go | 4 +- llo/channel_definitions_test.go | 107 +++++++++++++++++- llo/json_report_codec.go | 2 +- llo/json_report_codec_test.go | 4 +- llo/plugin.go | 2 + llo/plugin_outcome_test.go | 18 +-- llo/plugin_reports.go | 10 +- .../report_codec_evm_abi_encode_unpacked.go | 21 ++-- ...port_codec_evm_abi_encode_unpacked_expr.go | 21 ++-- ...codec_evm_abi_encode_unpacked_expr_test.go | 50 +++++++- ...port_codec_evm_abi_encode_unpacked_test.go | 63 +++++++++-- .../evm/report_codec_evm_streamlined.go | 3 +- .../evm/report_codec_evm_streamlined_test.go | 4 +- .../evm/report_codec_premium_legacy.go | 21 ++-- .../evm/report_codec_premium_legacy_test.go | 51 ++++++++- llo/types.go | 8 +- 16 files changed, 324 insertions(+), 65 deletions(-) diff --git a/llo/channel_definitions.go b/llo/channel_definitions.go index 1b3a8fe..53d6d5b 100644 --- a/llo/channel_definitions.go +++ b/llo/channel_definitions.go @@ -83,7 +83,6 @@ func NewChannelDefinitionOptsCache() ChannelDefinitionOptsCache { } } -// If the codec does not implement OptsParser, nothing is cached func (c *channelDefinitionOptsCache) Set( channelID llotypes.ChannelID, channelOpts llotypes.ChannelOpts, @@ -92,8 +91,7 @@ func (c *channelDefinitionOptsCache) Set( // Check if codec implements optional OptsParser interface optsParser, ok := codec.(OptsParser) if !ok { - // Codec doesn't implement OptsParser, nothing to cache - return nil + return fmt.Errorf("codec does not implement OptsParser interface") } parsedOpts, err := optsParser.ParseOpts(channelOpts) diff --git a/llo/channel_definitions_test.go b/llo/channel_definitions_test.go index 74b78ba..7695780 100644 --- a/llo/channel_definitions_test.go +++ b/llo/channel_definitions_test.go @@ -13,7 +13,7 @@ type mockReportCodec struct { err error } -func (m mockReportCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { +func (m mockReportCodec) Encode(Report, llotypes.ChannelDefinition, interface{}) ([]byte, error) { return nil, nil } @@ -122,3 +122,108 @@ func Test_VerifyChannelDefinitions(t *testing.T) { require.NoError(t, err) }) } + +func Test_ChannelDefinitionsOptsCache(t *testing.T) { + t.Run("Set and Get with OptsParser codec", func(t *testing.T) { + cache := NewChannelDefinitionOptsCache() + codec := mockCodec{timeResolution: 4} + channelID := llotypes.ChannelID(1) + + setErr := cache.Set(channelID, llotypes.ChannelOpts{}, codec) + require.NoError(t, setErr) + + val, exists := cache.Get(channelID) + require.True(t, exists) + require.NotNil(t, val) + }) + + t.Run("Set returns error when codec does not implement OptsParser", func(t *testing.T) { + cache := NewChannelDefinitionOptsCache() + codec := mockReportCodec{} + + err := cache.Set(llotypes.ChannelID(1), llotypes.ChannelOpts{}, codec) + require.Error(t, err) + require.Contains(t, err.Error(), "does not implement OptsParser") + + val, exists := cache.Get(llotypes.ChannelID(1)) + require.False(t, exists) + require.Nil(t, val) + }) + + t.Run("Set replaces existing value", func(t *testing.T) { + cache := NewChannelDefinitionOptsCache() + codec1 := mockCodec{timeResolution: 4} + codec2 := mockCodec{timeResolution: 3} + channelID := llotypes.ChannelID(1) + + cache.Set(channelID, llotypes.ChannelOpts{}, codec1) + cache.Set(channelID, llotypes.ChannelOpts{}, codec2) + + val, exists := cache.Get(channelID) + require.True(t, exists) + mc := val.(mockCodec) + require.Equal(t, codec2.timeResolution, mc.timeResolution) + }) + + t.Run("Multiple items in cache", func(t *testing.T) { + cache := NewChannelDefinitionOptsCache() + codec := mockCodec{timeResolution: 4} + + cache.Set(llotypes.ChannelID(1), llotypes.ChannelOpts{}, codec) + cache.Set(llotypes.ChannelID(2), llotypes.ChannelOpts{}, codec) + cache.Set(llotypes.ChannelID(3), llotypes.ChannelOpts{}, codec) + + val1, exists1 := cache.Get(llotypes.ChannelID(1)) + require.True(t, exists1) + require.Equal(t, codec.timeResolution, val1.(mockCodec).timeResolution) + + val2, exists2 := cache.Get(llotypes.ChannelID(2)) + require.True(t, exists2) + require.Equal(t, codec.timeResolution, val2.(mockCodec).timeResolution) + + val3, exists3 := cache.Get(llotypes.ChannelID(3)) + require.True(t, exists3) + require.Equal(t, codec.timeResolution, val3.(mockCodec).timeResolution) + }) + + t.Run("Delete removes item", func(t *testing.T) { + cache := NewChannelDefinitionOptsCache() + codec := mockCodec{} + channelID := llotypes.ChannelID(1) + + cache.Set(channelID, llotypes.ChannelOpts{}, codec) + cache.Delete(channelID) + + val, exists := cache.Get(channelID) + require.False(t, exists) + require.Nil(t, val) + }) + + t.Run("Delete does not affect other items", func(t *testing.T) { + cache := NewChannelDefinitionOptsCache() + codec := mockCodec{timeResolution: 4} + + cache.Set(llotypes.ChannelID(1), llotypes.ChannelOpts{}, codec) + cache.Set(llotypes.ChannelID(2), llotypes.ChannelOpts{}, codec) + + val1, exists1 := cache.Get(llotypes.ChannelID(1)) + require.True(t, exists1) + require.NotNil(t, val1) + + val2, exists2 := cache.Get(llotypes.ChannelID(2)) + require.True(t, exists2) + require.NotNil(t, val2) + + cache.Delete(llotypes.ChannelID(1)) + + // ChannelID 1 should be deleted + val, exists := cache.Get(llotypes.ChannelID(1)) + require.False(t, exists) + require.Nil(t, val) + + // ChannelID 2 should still exist + val, exists = cache.Get(llotypes.ChannelID(2)) + require.True(t, exists) + require.NotNil(t, val) + }) +} diff --git a/llo/json_report_codec.go b/llo/json_report_codec.go index efb47a3..cda85bc 100644 --- a/llo/json_report_codec.go +++ b/llo/json_report_codec.go @@ -19,7 +19,7 @@ var _ ReportCodec = JSONReportCodec{} type JSONReportCodec struct{} -func (cdc JSONReportCodec) Encode(r Report, _ llotypes.ChannelDefinition) ([]byte, error) { +func (cdc JSONReportCodec) Encode(r Report, _ llotypes.ChannelDefinition, _ interface{}) ([]byte, error) { type encode struct { ConfigDigest types.ConfigDigest SeqNr uint64 diff --git a/llo/json_report_codec_test.go b/llo/json_report_codec_test.go index 3d4ae78..98d30b7 100644 --- a/llo/json_report_codec_test.go +++ b/llo/json_report_codec_test.go @@ -94,7 +94,7 @@ func Test_JSONCodec_Properties(t *testing.T) { properties.Property("Encode/Decode", prop.ForAll( func(r Report) bool { - b, err := codec.Encode(r, cd) + b, err := codec.Encode(r, cd, nil) require.NoError(t, err) r2, err := codec.Decode(b) require.NoError(t, err) @@ -305,7 +305,7 @@ func Test_JSONCodec(t *testing.T) { cdc := JSONReportCodec{} - encoded, err := cdc.Encode(r, llo.ChannelDefinition{}) + encoded, err := cdc.Encode(r, llo.ChannelDefinition{}, nil) require.NoError(t, err) assert.Equal(t, `{"ConfigDigest":"0102030000000000000000000000000000000000000000000000000000000000","SeqNr":43,"ChannelID":46,"ValidAfterNanoseconds":44,"ObservationTimestampNanoseconds":45,"Values":[{"t":0,"v":"1"},{"t":0,"v":"2"},{"t":1,"v":"Q{Bid: 3.13, Benchmark: 4.4, Ask: 5.12}"}],"Specimen":true}`, string(encoded)) //nolint:testifylint // need to verify exact match including order for determinism diff --git a/llo/plugin.go b/llo/plugin.go index c23ca4a..3b5e71c 100644 --- a/llo/plugin.go +++ b/llo/plugin.go @@ -170,6 +170,8 @@ type ChannelDefinitionCache interface { type ChannelDefinitionOptsCache interface { // Set parses and caches the channel definition opts for the given channelID // The channelOpts should match the ReportCodec's opts type. + // The codec is responsible for having a method to parse `channelOpts` into the codec's expected opts type. + // Failure to cache opts should return an error. Set(channelID llotypes.ChannelID, channelOpts llotypes.ChannelOpts, codec ReportCodec) error // Get retrieves cached opts for the given channelID // Returning `interface{}` requires type assertion to the specific ReportCodec's opts type. diff --git a/llo/plugin_outcome_test.go b/llo/plugin_outcome_test.go index 3ae4e34..2f02e8a 100644 --- a/llo/plugin_outcome_test.go +++ b/llo/plugin_outcome_test.go @@ -41,7 +41,7 @@ func testOutcome(t *testing.T, outcomeCodec OutcomeCodec) { ReportCodecs: map[llotypes.ReportFormat]ReportCodec{ llotypes.ReportFormatEVMABIEncodeUnpacked: mockCodec{timeResolution: ResolutionNanoseconds}, llotypes.ReportFormatEVMPremiumLegacy: mockCodec{timeResolution: ResolutionSeconds}, - llotypes.ReportFormatJSON: basicCodec{}, + llotypes.ReportFormatJSON: mockReportCodec{}, }, ChannelDefinitionOptsCache: NewChannelDefinitionOptsCache(), } @@ -792,7 +792,9 @@ var ( _ TimeResolutionProvider = mockCodec{} ) -func (mockCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { return nil, nil } +func (mockCodec) Encode(Report, llotypes.ChannelDefinition, interface{}) ([]byte, error) { + return nil, nil +} func (mockCodec) Verify(llotypes.ChannelDefinition) error { return nil } @@ -813,14 +815,6 @@ func (c mockCodec) TimeResolution(parsedOpts interface{}) (TimeResolution, error return c.timeResolution, nil } -// basicCodec is a codec that does not implement OptsParser -type basicCodec struct{} - -var _ ReportCodec = basicCodec{} - -func (basicCodec) Encode(Report, llotypes.ChannelDefinition) ([]byte, error) { return nil, nil } -func (basicCodec) Verify(llotypes.ChannelDefinition) error { return nil } - func Test_Outcome_Methods(t *testing.T) { optsCache := NewChannelDefinitionOptsCache() codecs := map[llotypes.ReportFormat]ReportCodec{ @@ -828,7 +822,7 @@ func Test_Outcome_Methods(t *testing.T) { llotypes.ReportFormatEVMPremiumLegacy: mockCodec{timeResolution: ResolutionSeconds}, llotypes.ReportFormatEVMABIEncodeUnpacked: mockCodec{timeResolution: ResolutionNanoseconds}, llotypes.ReportFormatEVMABIEncodeUnpackedExpr: mockCodec{timeResolution: ResolutionNanoseconds}, - llotypes.ReportFormatJSON: basicCodec{}, + llotypes.ReportFormatJSON: mockReportCodec{}, } t.Run("protocol version 0", func(t *testing.T) { @@ -1007,7 +1001,7 @@ func Test_Outcome_Methods(t *testing.T) { // Cannot populate cache because codec does not implement OptsParser err := optsCache.Set(cid, cd.Opts, codecs[cd.ReportFormat]) - require.NoError(t, err) + require.Error(t, err) _, cached := optsCache.Get(cid) require.False(t, cached, "cache should not be populated after Set (Codec does not implement OptsParser)") diff --git a/llo/plugin_reports.go b/llo/plugin_reports.go index e2d8cfb..7673bd3 100644 --- a/llo/plugin_reports.go +++ b/llo/plugin_reports.go @@ -101,7 +101,15 @@ func (p *Plugin) encodeReport(r Report, cd llotypes.ChannelDefinition) (types.Re return nil, fmt.Errorf("codec missing for ReportFormat=%q", cd.ReportFormat) } p.captureReportTelemetry(r, cd) - return codec.Encode(r, cd) + + // Lookup cached opts if available + var cachedOpts interface{} + if p.ChannelDefinitionOptsCache != nil { + cachedOpts, _ = p.ChannelDefinitionOptsCache.Get(r.ChannelID) + // cachedOpts may be nil - that's fine, codec will parse cd.Opts + } + + return codec.Encode(r, cd, cachedOpts) } func (p *Plugin) captureReportTelemetry(r Report, cd llotypes.ChannelDefinition) { diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go index 5e0a62f..d46c785 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go @@ -83,7 +83,7 @@ type BaseReportFields struct { ExpiresAt uint64 } -func (r ReportCodecEVMABIEncodeUnpacked) Encode(report llo.Report, cd llotypes.ChannelDefinition) ([]byte, error) { +func (r ReportCodecEVMABIEncodeUnpacked) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) { if report.Specimen { return nil, errors.New("ReportCodecEVMABIEncodeUnpacked does not support encoding specimen reports") } @@ -99,12 +99,19 @@ func (r ReportCodecEVMABIEncodeUnpacked) Encode(report llo.Report, cd llotypes.C return nil, fmt.Errorf("ReportCodecEVMABIEncodeUnpacked failed to extract link price: %w", err) } - // NOTE: It seems suboptimal to have to parse the opts on every encode but - // not sure how to avoid it. Should be negligible performance hit as long - // as Opts is small. - opts := ReportFormatEVMABIEncodeOpts{} - if err = (&opts).Decode(cd.Opts); err != nil { - return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) + var opts ReportFormatEVMABIEncodeOpts + if parsedOpts != nil { + // Use cached opts + var ok bool + opts, ok = parsedOpts.(ReportFormatEVMABIEncodeOpts) + if !ok { + return nil, fmt.Errorf("expected ReportFormatEVMABIEncodeOpts, got %T", parsedOpts) + } + } else { + // Fall back to parsing JSON + if err = (&opts).Decode(cd.Opts); err != nil { + return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) + } } validAfter := ConvertTimestamp(report.ValidAfterNanoseconds, opts.TimeResolution) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go index c86eee3..736dd81 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go @@ -28,7 +28,7 @@ func NewReportCodecEVMABIEncodeUnpackedExpr(lggr logger.Logger, donID uint32) Re return ReportCodecEVMABIEncodeUnpackedExpr{logger.Sugared(lggr).Named("ReportCodecEVMABIEncodeUnpackedExpr"), donID} } -func (r ReportCodecEVMABIEncodeUnpackedExpr) Encode(report llo.Report, cd llotypes.ChannelDefinition) ([]byte, error) { +func (r ReportCodecEVMABIEncodeUnpackedExpr) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) { if report.Specimen { return nil, errors.New("ReportCodecEVMABIEncodeUnpackedExpr does not support encoding specimen reports") } @@ -44,12 +44,19 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) Encode(report llo.Report, cd llotyp return nil, fmt.Errorf("ReportCodecEVMABIEncodeUnpackedExpr failed to extract link price: %w", err) } - // NOTE: It seems suboptimal to have to parse the opts on every encode but - // not sure how to avoid it. Should be negligible performance hit as long - // as Opts is small. - opts := ReportFormatEVMABIEncodeOpts{} - if err = (&opts).Decode(cd.Opts); err != nil { - return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) + var opts ReportFormatEVMABIEncodeOpts + if parsedOpts != nil { + // Use cached opts + var ok bool + opts, ok = parsedOpts.(ReportFormatEVMABIEncodeOpts) + if !ok { + return nil, fmt.Errorf("expected ReportFormatEVMABIEncodeOpts, got %T", parsedOpts) + } + } else { + // Fall back to parsing JSON + if err = (&opts).Decode(cd.Opts); err != nil { + return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) + } } validAfter := ConvertTimestamp(report.ValidAfterNanoseconds, opts.TimeResolution) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go index bb7ac19..869650c 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go @@ -72,7 +72,7 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { } codec := ReportCodecEVMABIEncodeUnpackedExpr{} - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.Error(t, err) assert.Contains(t, err.Error(), "ABI and values length mismatch") }) @@ -149,7 +149,7 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { } codec := ReportCodecEVMABIEncodeUnpackedExpr{} - encoded, err := codec.Encode(report, cd) + encoded, err := codec.Encode(report, cd, nil) require.NoError(t, err) values, err := expectedDEXBasedAssetSchema.Unpack(encoded) @@ -161,7 +161,7 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { for i := range report.Values { report.Values[i] = nil } - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.Error(t, err) return true @@ -262,7 +262,7 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_Encode(t *testing.T) { } codec := ReportCodecEVMABIEncodeUnpackedExpr{} - encoded, err := codec.Encode(report, cd) + encoded, err := codec.Encode(report, cd, nil) require.NoError(t, err) values, err := schema.Unpack(encoded) @@ -503,3 +503,45 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_EncodeOpts(t *testing.T) { } }) } + +func TestReportCodecEVMABIEncodeUnpackedExpr_WithAndWithoutParsedOpts(t *testing.T) { + codec := ReportCodecEVMABIEncodeUnpackedExpr{} + + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [{"type": "uint192"}] + }`) + + cd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpackedExpr, + Streams: []llotypes.Stream{{StreamID: 1}, {StreamID: 2}, {StreamID: 3}}, + Opts: optsJSON, + } + + report := llo.Report{ + ValidAfterNanoseconds: 1234567890000000000, + ObservationTimestampNanoseconds: 1234567891000000000, + Values: []llo.StreamValue{ + llo.ToDecimal(decimal.NewFromFloat(1.5)), + llo.ToDecimal(decimal.NewFromFloat(2.5)), + llo.ToDecimal(decimal.NewFromFloat(100.123)), + }, + } + + // Parse opts using OptsParser (simulates cache hit) + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + // Encode with parsed opts + encodedWithCache, err := codec.Encode(report, cd, parsedOpts) + require.NoError(t, err) + + // Encode without parsed opts + encodedWithoutCache, err := codec.Encode(report, cd, nil) + require.NoError(t, err) + + // Both paths should produce identical output + assert.Equal(t, encodedWithCache, encodedWithoutCache) +} diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go index 4afdd2c..a25bd2f 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go @@ -153,7 +153,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { Opts: serializedOpts, } - encoded, err := codec.Encode(report, cd) + encoded, err := codec.Encode(report, cd, nil) require.NoError(t, err) values, err := expectedDEXBasedAssetSchema.Unpack(encoded) @@ -168,7 +168,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { for i := range report.Values { report.Values[i] = nil } - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.Error(t, err) return AllTrue([]bool{ @@ -264,7 +264,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { Opts: serializedOpts, } - encoded, err := codec.Encode(report, cd) + encoded, err := codec.Encode(report, cd, nil) require.NoError(t, err) values, err := expectedRWASchema.Unpack(encoded) @@ -279,7 +279,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { for i := range report.Values { report.Values[i] = nil } - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.Error(t, err) return AllTrue([]bool{ @@ -362,7 +362,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { Opts: serializedOpts, } - encoded, err := codec.Encode(report, cd) + encoded, err := codec.Encode(report, cd, nil) require.NoError(t, err) values, err := expectedDEXBasedAssetSchema.Unpack(encoded) @@ -377,7 +377,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { for i := range report.Values { report.Values[i] = nil } - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.Error(t, err) return AllTrue([]bool{ @@ -468,7 +468,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { Opts: serializedOpts, } - encoded, err := codec.Encode(report, cd) + encoded, err := codec.Encode(report, cd, nil) require.NoError(t, err) values, err := schema.Unpack(encoded) @@ -598,7 +598,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { Opts: serializedOpts, } - encoded, err := codec.Encode(report, cd) + encoded, err := codec.Encode(report, cd, nil) require.NoError(t, err) values, err := expectedFundingRateSchema.Unpack(encoded) @@ -613,7 +613,7 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode_properties(t *testing.T) { for i := range report.Values { report.Values[i] = nil } - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.Error(t, err) return AllTrue([]bool{ @@ -699,11 +699,11 @@ func TestReportCodecEVMABIEncodeUnpacked_Encode(t *testing.T) { } codec := ReportCodecEVMABIEncodeUnpacked{} - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.EqualError(t, err, "failed to build payload; ABI and values length mismatch; ABI: 0, Values: 3") report.Values = []llo.StreamValue{} - _, err = codec.Encode(report, cd) + _, err = codec.Encode(report, cd, nil) require.EqualError(t, err, "ReportCodecEVMABIEncodeUnpacked requires at least 2 values (NativePrice, LinkPrice, ...); got report.Values: []") }) } @@ -1004,3 +1004,44 @@ func TestReportCodecEVMABIEncodeUnpacked_EncodeOpts(t *testing.T) { } }) } + +func TestReportCodecEVMABIEncodeUnpacked_WithAndWithoutParsedOpts(t *testing.T) { + codec := ReportCodecEVMABIEncodeUnpacked{} + + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [{"type": "uint192"}] + }`) + + cd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpacked, + Streams: []llotypes.Stream{{StreamID: 1}, {StreamID: 2}, {StreamID: 3}}, + Opts: optsJSON, + } + + report := llo.Report{ + ValidAfterNanoseconds: 1234567890000000000, + ObservationTimestampNanoseconds: 1234567891000000000, + Values: []llo.StreamValue{ + llo.ToDecimal(decimal.NewFromFloat(1.5)), + llo.ToDecimal(decimal.NewFromFloat(2.5)), + llo.ToDecimal(decimal.NewFromFloat(100.123)), + }, + } + + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + // Encode with parsed opts + encodedWithCache, err := codec.Encode(report, cd, parsedOpts) + require.NoError(t, err) + + // Encode without parsed opts + encodedWithoutCache, err := codec.Encode(report, cd, nil) + require.NoError(t, err) + + // Both paths should produce identical output + assert.Equal(t, encodedWithCache, encodedWithoutCache) +} diff --git a/llo/reportcodecs/evm/report_codec_evm_streamlined.go b/llo/reportcodecs/evm/report_codec_evm_streamlined.go index b713d2f..2161e30 100644 --- a/llo/reportcodecs/evm/report_codec_evm_streamlined.go +++ b/llo/reportcodecs/evm/report_codec_evm_streamlined.go @@ -28,7 +28,8 @@ func NewReportCodecStreamlined() ReportCodecEVMStreamlined { type ReportCodecEVMStreamlined struct{} -func (rc ReportCodecEVMStreamlined) Encode(r llo.Report, cd llotypes.ChannelDefinition) (payload []byte, err error) { +func (rc ReportCodecEVMStreamlined) Encode(r llo.Report, cd llotypes.ChannelDefinition, _ interface{}) (payload []byte, err error) { + // TODO: implement OptsParser. This codec does not implement it so parsedOpts is ignored opts := ReportFormatEVMStreamlinedOpts{} if err = (&opts).Decode(cd.Opts); err != nil { return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) diff --git a/llo/reportcodecs/evm/report_codec_evm_streamlined_test.go b/llo/reportcodecs/evm/report_codec_evm_streamlined_test.go index 656af70..43d1dfe 100644 --- a/llo/reportcodecs/evm/report_codec_evm_streamlined_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_streamlined_test.go @@ -33,7 +33,7 @@ func TestReportCodecEVMStreamlined(t *testing.T) { Values: []llo.StreamValue{ llo.ToDecimal(decimal.NewFromFloat(1123455935.123)), }, - }, cd) + }, cd, nil) require.NoError(t, err) require.Len(t, payload, 32) // Report Format @@ -56,7 +56,7 @@ func TestReportCodecEVMStreamlined(t *testing.T) { Values: []llo.StreamValue{ llo.ToDecimal(decimal.NewFromFloat(1123455935.123)), }, - }, cd) + }, cd, nil) require.NoError(t, err) require.Len(t, payload, 64) assert.Equal(t, feedID, hex.EncodeToString(payload[:32])) // feed id diff --git a/llo/reportcodecs/evm/report_codec_premium_legacy.go b/llo/reportcodecs/evm/report_codec_premium_legacy.go index 5094e26..47e559c 100644 --- a/llo/reportcodecs/evm/report_codec_premium_legacy.go +++ b/llo/reportcodecs/evm/report_codec_premium_legacy.go @@ -79,7 +79,7 @@ func (r *ReportFormatEVMPremiumLegacyOpts) Decode(opts []byte) error { return decoder.Decode(r) } -func (r ReportCodecPremiumLegacy) Encode(report llo.Report, cd llotypes.ChannelDefinition) ([]byte, error) { +func (r ReportCodecPremiumLegacy) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) { if report.Specimen { return nil, errors.New("ReportCodecPremiumLegacy does not support encoding specimen reports") } @@ -88,12 +88,19 @@ func (r ReportCodecPremiumLegacy) Encode(report llo.Report, cd llotypes.ChannelD return nil, fmt.Errorf("ReportCodecPremiumLegacy cannot encode; got unusable report; %w", err) } - // NOTE: It seems suboptimal to have to parse the opts on every encode but - // not sure how to avoid it. Should be negligible performance hit as long - // as Opts is small. - opts := ReportFormatEVMPremiumLegacyOpts{} - if err = (&opts).Decode(cd.Opts); err != nil { - return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) + var opts ReportFormatEVMPremiumLegacyOpts + if parsedOpts != nil { + // Use cached opts + var ok bool + opts, ok = parsedOpts.(ReportFormatEVMPremiumLegacyOpts) + if !ok { + return nil, fmt.Errorf("expected ReportFormatEVMPremiumLegacyOpts, got %T", parsedOpts) + } + } else { + // Fall back to parsing JSON + if err = (&opts).Decode(cd.Opts); err != nil { + return nil, fmt.Errorf("failed to decode opts; got: '%s'; %w", cd.Opts, err) + } } var multiplier decimal.Decimal if opts.Multiplier == nil { diff --git a/llo/reportcodecs/evm/report_codec_premium_legacy_test.go b/llo/reportcodecs/evm/report_codec_premium_legacy_test.go index 4d3304f..0c0ecd5 100644 --- a/llo/reportcodecs/evm/report_codec_premium_legacy_test.go +++ b/llo/reportcodecs/evm/report_codec_premium_legacy_test.go @@ -32,7 +32,7 @@ func FuzzReportCodecPremiumLegacy_Decode(f *testing.F) { codec := ReportCodecPremiumLegacy{logger.Test(f), 100002} - validEncodedReport, err := codec.Encode(validReport, cd) + validEncodedReport, err := codec.Encode(validReport, cd, nil) require.NoError(f, err) f.Add(validEncodedReport) @@ -60,7 +60,7 @@ func Test_ReportCodecPremiumLegacy(t *testing.T) { cd := llotypes.ChannelDefinition{Opts: llotypes.ChannelOpts(fmt.Sprintf(`{"baseUSDFee":"10.50","expirationWindow":60,"feedId":"0x%x","multiplier":10}`, feedID))} t.Run("Encode errors if no values", func(t *testing.T) { - _, err := rc.Encode(llo.Report{}, cd) + _, err := rc.Encode(llo.Report{}, cd, nil) require.Error(t, err) assert.Contains(t, err.Error(), "ReportCodecPremiumLegacy cannot encode; got unusable report; ReportCodecPremiumLegacy requires exactly 3 values (NativePrice, LinkPrice, Quote{Bid, Mid, Ask}); got report.Values: []") @@ -70,7 +70,7 @@ func Test_ReportCodecPremiumLegacy(t *testing.T) { report := newValidPremiumLegacyReport() report.Specimen = true - _, err := rc.Encode(report, cd) + _, err := rc.Encode(report, cd, nil) require.Error(t, err) require.EqualError(t, err, "ReportCodecPremiumLegacy does not support encoding specimen reports") }) @@ -78,7 +78,7 @@ func Test_ReportCodecPremiumLegacy(t *testing.T) { t.Run("Encode constructs a report from observations", func(t *testing.T) { report := newValidPremiumLegacyReport() - encoded, err := rc.Encode(report, cd) + encoded, err := rc.Encode(report, cd, nil) require.NoError(t, err) assert.Len(t, encoded, 288) @@ -119,7 +119,7 @@ func Test_ReportCodecPremiumLegacy(t *testing.T) { Values: []llo.StreamValue{nil, nil, &llo.Quote{Bid: decimal.NewFromInt(37), Benchmark: decimal.NewFromInt(38), Ask: decimal.NewFromInt(39)}}, } - encoded, err := rc.Encode(report, cd) + encoded, err := rc.Encode(report, cd, nil) require.NoError(t, err) assert.Len(t, encoded, 288) @@ -328,3 +328,44 @@ func Test_ReportCodecPremiumLegacy_Verify(t *testing.T) { require.NoError(t, err) }) } + +func TestReportCodecPremiumLegacy_WithAndWithoutParsedOpts(t *testing.T) { + codec := ReportCodecPremiumLegacy{Logger: logger.Test(t)} + + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031" + }`) + + cd := llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, + Streams: []llotypes.Stream{{StreamID: 1}, {StreamID: 2}, {StreamID: 3}}, + Opts: optsJSON, + } + + report := llo.Report{ + ValidAfterNanoseconds: 1234567890000000000, + ObservationTimestampNanoseconds: 1234567891000000000, + Values: []llo.StreamValue{ + llo.ToDecimal(decimal.NewFromFloat(1.5)), + llo.ToDecimal(decimal.NewFromFloat(2.5)), + &llo.Quote{Bid: decimal.NewFromFloat(100.1), Benchmark: decimal.NewFromFloat(100.2), Ask: decimal.NewFromFloat(100.3)}, + }, + } + + // Parse opts using OptsParser + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + // Encode with parsed opts + encodedWithCache, err := codec.Encode(report, cd, parsedOpts) + require.NoError(t, err) + + // Encode without parsed opts + encodedWithoutCache, err := codec.Encode(report, cd, nil) + require.NoError(t, err) + + // Both paths should produce identical output + assert.Equal(t, encodedWithCache, encodedWithoutCache) +} diff --git a/llo/types.go b/llo/types.go index 4187395..5aaba9d 100644 --- a/llo/types.go +++ b/llo/types.go @@ -24,7 +24,13 @@ type ReportCodec interface { // Encode may be lossy, so no Decode function is expected // Encode should handle nil stream aggregate values without panicking (it // may return error instead) - Encode(Report, llotypes.ChannelDefinition) ([]byte, error) + // + // parsedOpts is a pre-parsed version of ChannelDefinition.Opts which avoids repeated Opts parsing. + // ChannelDefinition.Opts can be nil and is up to the codec to determine if it needs Opts and if + // the codec does not need ChannelDefinition.Opts it should pass in nil + // If parsedOpts is nil, the codec is expected to parse cd.Opts directly. + // If parsedOpts is non-nil, the codec should type-assert it to its expected opts type. + Encode(r Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) // Verify may optionally verify a channel definition to ensure it is valid // for the given report codec. If a codec does not wish to implement // validation it may simply return nil here. If any definition fails From 7a7244f2bbfc834e0b2a3047af6d97676bbddac6 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 4 Dec 2025 16:59:05 -0700 Subject: [PATCH 07/13] Add tests to codecs for parsing opts and TimeResolutionProvider methods channelDefinitionOptsCache add locking to operations to protect cache validity --- llo/channel_definitions.go | 8 ++++ ...codec_evm_abi_encode_unpacked_expr_test.go | 41 +++++++++++++++++ ...port_codec_evm_abi_encode_unpacked_test.go | 43 +++++++++++++++++- .../evm/report_codec_premium_legacy_test.go | 44 +++++++++++++++++++ 4 files changed, 135 insertions(+), 1 deletion(-) diff --git a/llo/channel_definitions.go b/llo/channel_definitions.go index 53d6d5b..8fecf1a 100644 --- a/llo/channel_definitions.go +++ b/llo/channel_definitions.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "sort" + "sync" llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" ) @@ -71,6 +72,7 @@ func subtractChannelDefinitions(minuend llotypes.ChannelDefinitions, subtrahend } type channelDefinitionOptsCache struct { + mu sync.RWMutex cache map[llotypes.ChannelID]interface{} } @@ -99,15 +101,21 @@ func (c *channelDefinitionOptsCache) Set( return fmt.Errorf("failed to parse opts for channelID %d: %w", channelID, err) } + c.mu.Lock() + defer c.mu.Unlock() c.cache[channelID] = parsedOpts return nil } func (c *channelDefinitionOptsCache) Get(channelID llotypes.ChannelID) (interface{}, bool) { + c.mu.RLock() + defer c.mu.RUnlock() val, ok := c.cache[channelID] return val, ok } func (c *channelDefinitionOptsCache) Delete(channelID llotypes.ChannelID) { + c.mu.Lock() + defer c.mu.Unlock() delete(c.cache, channelID) } diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go index 869650c..4b209cd 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go @@ -545,3 +545,44 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_WithAndWithoutParsedOpts(t *testing // Both paths should produce identical output assert.Equal(t, encodedWithCache, encodedWithoutCache) } + +func TestReportCodecEVMABIEncodeUnpackedExpr_ParseOpts(t *testing.T) { + codec := ReportCodecEVMABIEncodeUnpackedExpr{} + + t.Run("valid opts", func(t *testing.T) { + opts := []byte(`{"baseUSDFee":"2.5","expirationWindow":7200,"feedID":"0x0001020304050607080910111213141516171819202122232425262728293031","abi":[],"timeResolution":"us"}`) + result, err := codec.ParseOpts(opts) + require.NoError(t, err) + require.NotNil(t, result) + + parsed, ok := result.(ReportFormatEVMABIEncodeOpts) + require.True(t, ok) + require.Equal(t, "2.5", parsed.BaseUSDFee.String()) + require.Equal(t, uint32(7200), parsed.ExpirationWindow) + require.Equal(t, llo.ResolutionMicroseconds, parsed.TimeResolution) + }) + + t.Run("invalid JSON", func(t *testing.T) { + _, err := codec.ParseOpts([]byte(`{invalid`)) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to parse EVMABIEncodeUnpackedExpr opts") + }) +} + +func TestReportCodecEVMABIEncodeUnpackedExpr_TimeResolution(t *testing.T) { + codec := ReportCodecEVMABIEncodeUnpackedExpr{} + + t.Run("valid parsed opts", func(t *testing.T) { + opts := ReportFormatEVMABIEncodeOpts{TimeResolution: llo.ResolutionNanoseconds} + res, err := codec.TimeResolution(opts) + require.NoError(t, err) + require.Equal(t, llo.ResolutionNanoseconds, res) + }) + + t.Run("invalid type", func(t *testing.T) { + type wrongType struct{} + _, err := codec.TimeResolution(wrongType{}) + require.Error(t, err) + require.Contains(t, err.Error(), "expected ReportFormatEVMABIEncodeOpts") + }) +} diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go index a25bd2f..c2dfa02 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_test.go @@ -1038,10 +1038,51 @@ func TestReportCodecEVMABIEncodeUnpacked_WithAndWithoutParsedOpts(t *testing.T) encodedWithCache, err := codec.Encode(report, cd, parsedOpts) require.NoError(t, err) - // Encode without parsed opts + // Encode without parsed opts encodedWithoutCache, err := codec.Encode(report, cd, nil) require.NoError(t, err) // Both paths should produce identical output assert.Equal(t, encodedWithCache, encodedWithoutCache) } + +func TestReportCodecEVMABIEncodeUnpacked_ParseOpts(t *testing.T) { + codec := ReportCodecEVMABIEncodeUnpacked{} + + t.Run("valid opts", func(t *testing.T) { + opts := []byte(`{"baseUSDFee":"1.5","expirationWindow":3600,"feedID":"0x0001020304050607080910111213141516171819202122232425262728293031","abi":[],"timeResolution":"ms"}`) + result, err := codec.ParseOpts(opts) + require.NoError(t, err) + require.NotNil(t, result) + + parsed, ok := result.(ReportFormatEVMABIEncodeOpts) + require.True(t, ok) + require.Equal(t, "1.5", parsed.BaseUSDFee.String()) + require.Equal(t, uint32(3600), parsed.ExpirationWindow) + require.Equal(t, llo.ResolutionMilliseconds, parsed.TimeResolution) + }) + + t.Run("invalid JSON", func(t *testing.T) { + _, err := codec.ParseOpts([]byte(`{invalid`)) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to parse EVMABIEncodeUnpacked opts") + }) +} + +func TestReportCodecEVMABIEncodeUnpacked_TimeResolution(t *testing.T) { + codec := ReportCodecEVMABIEncodeUnpacked{} + + t.Run("valid parsed opts", func(t *testing.T) { + opts := ReportFormatEVMABIEncodeOpts{TimeResolution: llo.ResolutionMilliseconds} + res, err := codec.TimeResolution(opts) + require.NoError(t, err) + require.Equal(t, llo.ResolutionMilliseconds, res) + }) + + t.Run("invalid type", func(t *testing.T) { + type wrongType struct{} + _, err := codec.TimeResolution(wrongType{}) + require.Error(t, err) + require.Contains(t, err.Error(), "expected ReportFormatEVMABIEncodeOpts") + }) +} diff --git a/llo/reportcodecs/evm/report_codec_premium_legacy_test.go b/llo/reportcodecs/evm/report_codec_premium_legacy_test.go index 0c0ecd5..32a9088 100644 --- a/llo/reportcodecs/evm/report_codec_premium_legacy_test.go +++ b/llo/reportcodecs/evm/report_codec_premium_legacy_test.go @@ -369,3 +369,47 @@ func TestReportCodecPremiumLegacy_WithAndWithoutParsedOpts(t *testing.T) { // Both paths should produce identical output assert.Equal(t, encodedWithCache, encodedWithoutCache) } + +func TestReportCodecPremiumLegacy_ParseOpts(t *testing.T) { + codec := ReportCodecPremiumLegacy{} + + t.Run("valid opts", func(t *testing.T) { + opts := []byte(`{"baseUSDFee":"0.5","expirationWindow":1800,"feedID":"0x0001020304050607080910111213141516171819202122232425262728293031","multiplier":"1000000000000000000"}`) + result, err := codec.ParseOpts(opts) + require.NoError(t, err) + require.NotNil(t, result) + + parsed, ok := result.(ReportFormatEVMPremiumLegacyOpts) + require.True(t, ok) + require.Equal(t, "0.5", parsed.BaseUSDFee.String()) + require.Equal(t, uint32(1800), parsed.ExpirationWindow) + }) + + t.Run("empty JSON object", func(t *testing.T) { + result, err := codec.ParseOpts([]byte(`{}`)) + require.NoError(t, err) + require.NotNil(t, result) + }) + + t.Run("invalid JSON", func(t *testing.T) { + _, err := codec.ParseOpts([]byte(`{invalid`)) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to parse EVMPremiumLegacy opts") + }) +} + +func TestReportCodecPremiumLegacy_TimeResolution(t *testing.T) { + codec := ReportCodecPremiumLegacy{} + + t.Run("always returns seconds resolution", func(t *testing.T) { + // Premium legacy always uses seconds, regardless of input + type anyType struct{} + res, err := codec.TimeResolution(anyType{}) + require.NoError(t, err) + require.Equal(t, llo.ResolutionSeconds, res) + + res, err = codec.TimeResolution(nil) + require.NoError(t, err) + require.Equal(t, llo.ResolutionSeconds, res) + }) +} From 5c42c5885f285fd994f5c6615f1e03739427d8cc Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 4 Dec 2025 17:06:03 -0700 Subject: [PATCH 08/13] comment --- llo/channel_definitions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llo/channel_definitions.go b/llo/channel_definitions.go index 8fecf1a..d49edd8 100644 --- a/llo/channel_definitions.go +++ b/llo/channel_definitions.go @@ -78,7 +78,6 @@ type channelDefinitionOptsCache struct { var _ ChannelDefinitionOptsCache = (*channelDefinitionOptsCache)(nil) -// NewChannelDefinitionOptsCache creates a new ChannelDefinitionOptsCache func NewChannelDefinitionOptsCache() ChannelDefinitionOptsCache { return &channelDefinitionOptsCache{ cache: make(map[llotypes.ChannelID]interface{}), @@ -90,7 +89,8 @@ func (c *channelDefinitionOptsCache) Set( channelOpts llotypes.ChannelOpts, codec ReportCodec, ) error { - // Check if codec implements optional OptsParser interface + // codec may or may not implement OptsParser interface - that is the codec's choice. + // if codec does not then we cannot cache the opts. Codec may do this if they do not have opts. optsParser, ok := codec.(OptsParser) if !ok { return fmt.Errorf("codec does not implement OptsParser interface") From f64bdb2168943f0d3eb3138ad21f02faf5e4120d Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 4 Dec 2025 20:14:48 -0700 Subject: [PATCH 09/13] switch signature --- llo/channel_definitions.go | 6 +++--- llo/plugin.go | 11 ++++++---- llo/plugin_reports.go | 4 ++-- .../report_codec_evm_abi_encode_unpacked.go | 6 +++--- ...port_codec_evm_abi_encode_unpacked_expr.go | 6 +++--- .../evm/report_codec_evm_streamlined.go | 2 +- .../evm/report_codec_premium_legacy.go | 6 +++--- llo/types.go | 20 ++++++++++--------- 8 files changed, 33 insertions(+), 28 deletions(-) diff --git a/llo/channel_definitions.go b/llo/channel_definitions.go index d49edd8..810cf06 100644 --- a/llo/channel_definitions.go +++ b/llo/channel_definitions.go @@ -73,14 +73,14 @@ func subtractChannelDefinitions(minuend llotypes.ChannelDefinitions, subtrahend type channelDefinitionOptsCache struct { mu sync.RWMutex - cache map[llotypes.ChannelID]interface{} + cache map[llotypes.ChannelID]any } var _ ChannelDefinitionOptsCache = (*channelDefinitionOptsCache)(nil) func NewChannelDefinitionOptsCache() ChannelDefinitionOptsCache { return &channelDefinitionOptsCache{ - cache: make(map[llotypes.ChannelID]interface{}), + cache: make(map[llotypes.ChannelID]any), } } @@ -107,7 +107,7 @@ func (c *channelDefinitionOptsCache) Set( return nil } -func (c *channelDefinitionOptsCache) Get(channelID llotypes.ChannelID) (interface{}, bool) { +func (c *channelDefinitionOptsCache) Get(channelID llotypes.ChannelID) (any, bool) { c.mu.RLock() defer c.mu.RUnlock() val, ok := c.cache[channelID] diff --git a/llo/plugin.go b/llo/plugin.go index 3b5e71c..e59ffc7 100644 --- a/llo/plugin.go +++ b/llo/plugin.go @@ -165,8 +165,11 @@ type ChannelDefinitionCache interface { Definitions() llotypes.ChannelDefinitions } -// ChannelDefinitionOptsCache is a cache of channel definition opts -// It is used to avoid repeated JSON parsing of channel definition opts +// ChannelDefinitionOptsCache caches parsed channel definition opts to avoid +// repeated JSON unmarshalling in hot paths. Stores parsed opts as `any` which requires type +// assertion but is orders of magnitude faster than JSON parsing. +// Plugin controls cache lifecycle (set on channel add/update, delete on channel remove) +// OCR phases such as Observation and Reports use the cache to avoid repeated JSON unmarshalling. type ChannelDefinitionOptsCache interface { // Set parses and caches the channel definition opts for the given channelID // The channelOpts should match the ReportCodec's opts type. @@ -174,9 +177,9 @@ type ChannelDefinitionOptsCache interface { // Failure to cache opts should return an error. Set(channelID llotypes.ChannelID, channelOpts llotypes.ChannelOpts, codec ReportCodec) error // Get retrieves cached opts for the given channelID - // Returning `interface{}` requires type assertion to the specific ReportCodec's opts type. + // Returning `any` requires type assertion to the specific ReportCodec's opts type. // This is still considered better than parsing the opts from JSON every time we need to access them. - Get(channelID llotypes.ChannelID) (interface{}, bool) + Get(channelID llotypes.ChannelID) (any, bool) // Delete removes cached opts for the given channelID Delete(channelID llotypes.ChannelID) } diff --git a/llo/plugin_reports.go b/llo/plugin_reports.go index 7673bd3..916354e 100644 --- a/llo/plugin_reports.go +++ b/llo/plugin_reports.go @@ -103,10 +103,10 @@ func (p *Plugin) encodeReport(r Report, cd llotypes.ChannelDefinition) (types.Re p.captureReportTelemetry(r, cd) // Lookup cached opts if available - var cachedOpts interface{} + var cachedOpts any if p.ChannelDefinitionOptsCache != nil { cachedOpts, _ = p.ChannelDefinitionOptsCache.Get(r.ChannelID) - // cachedOpts may be nil - that's fine, codec will parse cd.Opts + // cachedOpts may be nil in the case the Codec doesn't have any Opts to parse. } return codec.Encode(r, cd, cachedOpts) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go index d46c785..753afd4 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked.go @@ -83,7 +83,7 @@ type BaseReportFields struct { ExpiresAt uint64 } -func (r ReportCodecEVMABIEncodeUnpacked) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) { +func (r ReportCodecEVMABIEncodeUnpacked) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts any) ([]byte, error) { if report.Specimen { return nil, errors.New("ReportCodecEVMABIEncodeUnpacked does not support encoding specimen reports") } @@ -267,7 +267,7 @@ func (r ReportCodecEVMABIEncodeUnpacked) buildHeader(rf BaseReportFields, precis return b, nil } -func (r ReportCodecEVMABIEncodeUnpacked) ParseOpts(opts []byte) (interface{}, error) { +func (r ReportCodecEVMABIEncodeUnpacked) ParseOpts(opts []byte) (any, error) { var o ReportFormatEVMABIEncodeOpts if err := json.Unmarshal(opts, &o); err != nil { return nil, fmt.Errorf("failed to parse EVMABIEncodeUnpacked opts: %w", err) @@ -275,7 +275,7 @@ func (r ReportCodecEVMABIEncodeUnpacked) ParseOpts(opts []byte) (interface{}, er return o, nil } -func (r ReportCodecEVMABIEncodeUnpacked) TimeResolution(parsedOpts interface{}) (llo.TimeResolution, error) { +func (r ReportCodecEVMABIEncodeUnpacked) TimeResolution(parsedOpts any) (llo.TimeResolution, error) { opts, ok := parsedOpts.(ReportFormatEVMABIEncodeOpts) if !ok { return llo.ResolutionSeconds, fmt.Errorf("expected ReportFormatEVMABIEncodeOpts, got %T", parsedOpts) diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go index 736dd81..ad7992a 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go @@ -28,7 +28,7 @@ func NewReportCodecEVMABIEncodeUnpackedExpr(lggr logger.Logger, donID uint32) Re return ReportCodecEVMABIEncodeUnpackedExpr{logger.Sugared(lggr).Named("ReportCodecEVMABIEncodeUnpackedExpr"), donID} } -func (r ReportCodecEVMABIEncodeUnpackedExpr) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) { +func (r ReportCodecEVMABIEncodeUnpackedExpr) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts any) ([]byte, error) { if report.Specimen { return nil, errors.New("ReportCodecEVMABIEncodeUnpackedExpr does not support encoding specimen reports") } @@ -154,7 +154,7 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) buildHeader(rf BaseReportFields, pr return b, nil } -func (r ReportCodecEVMABIEncodeUnpackedExpr) ParseOpts(opts []byte) (interface{}, error) { +func (r ReportCodecEVMABIEncodeUnpackedExpr) ParseOpts(opts []byte) (any, error) { var o ReportFormatEVMABIEncodeOpts if err := json.Unmarshal(opts, &o); err != nil { return nil, fmt.Errorf("failed to parse EVMABIEncodeUnpackedExpr opts: %w", err) @@ -162,7 +162,7 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) ParseOpts(opts []byte) (interface{} return o, nil } -func (r ReportCodecEVMABIEncodeUnpackedExpr) TimeResolution(parsedOpts interface{}) (llo.TimeResolution, error) { +func (r ReportCodecEVMABIEncodeUnpackedExpr) TimeResolution(parsedOpts any) (llo.TimeResolution, error) { opts, ok := parsedOpts.(ReportFormatEVMABIEncodeOpts) if !ok { return llo.ResolutionSeconds, fmt.Errorf("expected ReportFormatEVMABIEncodeOpts, got %T", parsedOpts) diff --git a/llo/reportcodecs/evm/report_codec_evm_streamlined.go b/llo/reportcodecs/evm/report_codec_evm_streamlined.go index 2161e30..829fd78 100644 --- a/llo/reportcodecs/evm/report_codec_evm_streamlined.go +++ b/llo/reportcodecs/evm/report_codec_evm_streamlined.go @@ -28,7 +28,7 @@ func NewReportCodecStreamlined() ReportCodecEVMStreamlined { type ReportCodecEVMStreamlined struct{} -func (rc ReportCodecEVMStreamlined) Encode(r llo.Report, cd llotypes.ChannelDefinition, _ interface{}) (payload []byte, err error) { +func (rc ReportCodecEVMStreamlined) Encode(r llo.Report, cd llotypes.ChannelDefinition, _ any) (payload []byte, err error) { // TODO: implement OptsParser. This codec does not implement it so parsedOpts is ignored opts := ReportFormatEVMStreamlinedOpts{} if err = (&opts).Decode(cd.Opts); err != nil { diff --git a/llo/reportcodecs/evm/report_codec_premium_legacy.go b/llo/reportcodecs/evm/report_codec_premium_legacy.go index 47e559c..32d8e98 100644 --- a/llo/reportcodecs/evm/report_codec_premium_legacy.go +++ b/llo/reportcodecs/evm/report_codec_premium_legacy.go @@ -79,7 +79,7 @@ func (r *ReportFormatEVMPremiumLegacyOpts) Decode(opts []byte) error { return decoder.Decode(r) } -func (r ReportCodecPremiumLegacy) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) { +func (r ReportCodecPremiumLegacy) Encode(report llo.Report, cd llotypes.ChannelDefinition, parsedOpts any) ([]byte, error) { if report.Specimen { return nil, errors.New("ReportCodecPremiumLegacy does not support encoding specimen reports") } @@ -280,7 +280,7 @@ func LegacyReportContext(cd ocr2types.ConfigDigest, seqNr uint64, donID uint32) }, nil } -func (r ReportCodecPremiumLegacy) ParseOpts(opts []byte) (interface{}, error) { +func (r ReportCodecPremiumLegacy) ParseOpts(opts []byte) (any, error) { var o ReportFormatEVMPremiumLegacyOpts if err := json.Unmarshal(opts, &o); err != nil { return nil, fmt.Errorf("failed to parse EVMPremiumLegacy opts: %w", err) @@ -288,7 +288,7 @@ func (r ReportCodecPremiumLegacy) ParseOpts(opts []byte) (interface{}, error) { return o, nil } -func (r ReportCodecPremiumLegacy) TimeResolution(parsedOpts interface{}) (llo.TimeResolution, error) { +func (r ReportCodecPremiumLegacy) TimeResolution(parsedOpts any) (llo.TimeResolution, error) { // Premium legacy always uses seconds resolution return llo.ResolutionSeconds, nil } diff --git a/llo/types.go b/llo/types.go index 5aaba9d..ad4b4dc 100644 --- a/llo/types.go +++ b/llo/types.go @@ -25,12 +25,13 @@ type ReportCodec interface { // Encode should handle nil stream aggregate values without panicking (it // may return error instead) // - // parsedOpts is a pre-parsed version of ChannelDefinition.Opts which avoids repeated Opts parsing. - // ChannelDefinition.Opts can be nil and is up to the codec to determine if it needs Opts and if - // the codec does not need ChannelDefinition.Opts it should pass in nil + // parsedOpts is a pre-parsed instantiation of ChannelDefinition.Opts which is created from + // the codecs Opts struct. cd.Opts can be nil and is up to the codec to determine + // if it needs Opts. If the codec does not have Opts it should pass in nil. + // For codecs with opts: // If parsedOpts is nil, the codec is expected to parse cd.Opts directly. // If parsedOpts is non-nil, the codec should type-assert it to its expected opts type. - Encode(r Report, cd llotypes.ChannelDefinition, parsedOpts interface{}) ([]byte, error) + Encode(r Report, cd llotypes.ChannelDefinition, parsedOpts any) ([]byte, error) // Verify may optionally verify a channel definition to ensure it is valid // for the given report codec. If a codec does not wish to implement // validation it may simply return nil here. If any definition fails @@ -43,31 +44,32 @@ type ReportCodec interface { // OptsParser parses raw channel opts bytes into a codec-specific structure. // ReportCodecs may implement this interface to enable caching of parsed opts // to avoid repeated unmarshalling of Opts bytes. For example unmarshalling -// from JSON is much more expensive than peforming a type assertion on an `interface{}` type. +// from JSON is much more expensive than peforming a type assertion on an `any` type. // Since not all Codec Opts might have the same Options - you can create // `OptsProvider` interfaces where needed. For Example `TimeResolutionProvider` or `ABIProvider` // and the Codecs can implement all interfaces that match the specific Opt. type OptsParser interface { // ParseOpts parses the raw opts bytes and returns a codec-specific - // parsed opts structure as interface{}. + // parsed opts structure as `any`. // Use the returned interface to type assert to the specific Opts type. // For Example: // opts, err := codec.ParseOpts(optsBytes) // if err != nil { // return nil, err // } - // optsProvider, ok := opts.(OptsProvider) // where OptsProvider is the interface you created for the specific Opts type + // + // optsProvider, ok := opts.(OptsProvider) // if !ok { // return nil, fmt.Errorf("invalid opts type") // } - ParseOpts(opts []byte) (interface{}, error) + ParseOpts(opts []byte) (any, error) } // TimeResolutionProvider extracts time resolution information from codec opts type TimeResolutionProvider interface { // TimeResolution returns the time resolution from the parsed opts. // The parsedOpts must be the value returned by OptsParser.ParseOpts. - TimeResolution(parsedOpts interface{}) (TimeResolution, error) + TimeResolution(parsedOpts any) (TimeResolution, error) } type ChannelDefinitionWithID struct { From 6100b3173a90d1e00372d9a067edb09e40e2000a Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Thu, 4 Dec 2025 20:16:07 -0700 Subject: [PATCH 10/13] switch signature --- llo/channel_definitions_test.go | 2 +- llo/json_report_codec.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llo/channel_definitions_test.go b/llo/channel_definitions_test.go index 7695780..2ba41fd 100644 --- a/llo/channel_definitions_test.go +++ b/llo/channel_definitions_test.go @@ -13,7 +13,7 @@ type mockReportCodec struct { err error } -func (m mockReportCodec) Encode(Report, llotypes.ChannelDefinition, interface{}) ([]byte, error) { +func (m mockReportCodec) Encode(Report, llotypes.ChannelDefinition, any) ([]byte, error) { return nil, nil } diff --git a/llo/json_report_codec.go b/llo/json_report_codec.go index cda85bc..72325a9 100644 --- a/llo/json_report_codec.go +++ b/llo/json_report_codec.go @@ -19,7 +19,7 @@ var _ ReportCodec = JSONReportCodec{} type JSONReportCodec struct{} -func (cdc JSONReportCodec) Encode(r Report, _ llotypes.ChannelDefinition, _ interface{}) ([]byte, error) { +func (cdc JSONReportCodec) Encode(r Report, _ llotypes.ChannelDefinition, _ any) ([]byte, error) { type encode struct { ConfigDigest types.ConfigDigest SeqNr uint64 From 5195ae699c8481143f3c2f04d178004f2a161bf5 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Mon, 8 Dec 2025 15:54:16 -0700 Subject: [PATCH 11/13] Implement ChannelOptsCache usage in calculated streams and add a benchmark test --- llo/channel_definitions.go | 15 +- llo/channel_definitions_bench_test.go | 90 +++++++++ llo/plugin.go | 10 +- llo/reportcodecs/evm/report_codec_common.go | 1 - ...port_codec_evm_abi_encode_unpacked_expr.go | 28 ++- ...codec_evm_abi_encode_unpacked_expr_test.go | 172 +++++++++++++++++- llo/stream_calculated.go | 75 ++++---- llo/stream_calculated_test.go | 141 ++++++++++++-- llo/types.go | 22 ++- 9 files changed, 486 insertions(+), 68 deletions(-) create mode 100644 llo/channel_definitions_bench_test.go diff --git a/llo/channel_definitions.go b/llo/channel_definitions.go index 810cf06..fb7af95 100644 --- a/llo/channel_definitions.go +++ b/llo/channel_definitions.go @@ -71,8 +71,13 @@ func subtractChannelDefinitions(minuend llotypes.ChannelDefinitions, subtrahend return difference } +// channelDefinitionOptsCache stores pre-parsed channel opts to avoid repeated JSON parsing. +// See: BenchmarkChannelOptsCache_* tests in channel_definitions_bench_test.go for performance details. type channelDefinitionOptsCache struct { - mu sync.RWMutex + // OCR driver should ensure that each phase is only called by one thread at a time. + // The mu here is added as a safeguard for future proofing. + // Benchmarks showed that without a mutex we were saving about 10ns per call to Get . + mu sync.Mutex cache map[llotypes.ChannelID]any } @@ -89,8 +94,8 @@ func (c *channelDefinitionOptsCache) Set( channelOpts llotypes.ChannelOpts, codec ReportCodec, ) error { - // codec may or may not implement OptsParser interface - that is the codec's choice. - // if codec does not then we cannot cache the opts. Codec may do this if they do not have opts. + // codec may or may not implement OptsParser interface - that is the codec's choice. + // if codec does not then we cannot cache the opts. Codec may do this if they do not have opts. optsParser, ok := codec.(OptsParser) if !ok { return fmt.Errorf("codec does not implement OptsParser interface") @@ -108,8 +113,8 @@ func (c *channelDefinitionOptsCache) Set( } func (c *channelDefinitionOptsCache) Get(channelID llotypes.ChannelID) (any, bool) { - c.mu.RLock() - defer c.mu.RUnlock() + c.mu.Lock() + defer c.mu.Unlock() val, ok := c.cache[channelID] return val, ok } diff --git a/llo/channel_definitions_bench_test.go b/llo/channel_definitions_bench_test.go new file mode 100644 index 0000000..1ea2743 --- /dev/null +++ b/llo/channel_definitions_bench_test.go @@ -0,0 +1,90 @@ +package llo + +import ( + "fmt" + "testing" + + "github.com/goccy/go-json" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" +) + +// ============================================================================= +// Benchmark: Direct JSON Parsing vs Cache Lookup +// ============================================================================= +// +// These benchmarks demonstrate why we cache parsed channel opts instead of +// parsing JSON on every access. Run with: +// +// go test . -bench=BenchmarkChannelOptsCache -benchmem -run=NONE +// +// Expected results (approximate): +// +// DirectParse (JSON): ~600-800 ns/op | 640 B/op | 6 allocs +// CacheGet: ~15-25 ns/op | 0 B/op | 0 allocs +// Speedup: ~40-50x faster, zero allocations +// +// ============================================================================= + +// Realistic opts JSON for existing codecs that use JSON opts format +var benchmarkOptsJSON = []byte(`{"feedID":"0x0001020304050607080910111213141516171819202122232425262728293031","baseUSDFee":"1.5","expirationWindow":3600,"timeResolution":"ns","abi":[{"type":"int192","expression":"Sum(s1,s2)","expressionStreamId":100},{"type":"int192"},{"type":"uint256"}]}`) + +// benchParsedOpts mirrors existing codec structures +type benchParsedOpts struct { + FeedID string `json:"feedID"` + BaseUSDFee string `json:"baseUSDFee"` + ExpirationWindow uint32 `json:"expirationWindow"` + TimeResolution string `json:"timeResolution,omitempty"` + ABI []abiEntry `json:"abi"` +} + +type abiEntry struct { + Type string `json:"type"` + Expression string `json:"expression,omitempty"` + ExpressionStreamID uint32 `json:"expressionStreamId,omitempty"` +} + +// benchMockCodec implements OptsParser for benchmarks +type benchMockCodec struct{} + +func (benchMockCodec) Encode(Report, llotypes.ChannelDefinition, any) ([]byte, error) { + return nil, nil +} +func (benchMockCodec) Verify(llotypes.ChannelDefinition) error { return nil } +func (benchMockCodec) ParseOpts(opts []byte) (any, error) { + var parsed benchParsedOpts + if err := json.Unmarshal(opts, &parsed); err != nil { + return nil, fmt.Errorf("failed to parse opts: %w", err) + } + return parsed, nil +} + +// BenchmarkChannelOptsCache_DirectParse measures the cost of parsing opts directly. +// Note: actual parsing cost depends on codec implementation (JSON, protobuf, etc.) +// This benchmark uses JSON as representative of current codecs. +func BenchmarkChannelOptsCache_DirectParse(b *testing.B) { + for i := 0; i < b.N; i++ { + var opts benchParsedOpts + _ = json.Unmarshal(benchmarkOptsJSON, &opts) + } +} + +// BenchmarkChannelOptsCache_CacheGet measures the cost of a cache lookup + type assertion. +// This is the required usage pattern for the cache. +func BenchmarkChannelOptsCache_CacheGet(b *testing.B) { + cache := NewChannelDefinitionOptsCache() + codec := benchMockCodec{} + + if err := cache.Set(1, benchmarkOptsJSON, codec); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + // Usage pattern is 1) Get and 2) type assertion + for i := 0; i < b.N; i++ { + val, ok := cache.Get(1) + if ok { + _ = val.(benchParsedOpts) + } + } +} diff --git a/llo/plugin.go b/llo/plugin.go index e59ffc7..39724ca 100644 --- a/llo/plugin.go +++ b/llo/plugin.go @@ -165,11 +165,11 @@ type ChannelDefinitionCache interface { Definitions() llotypes.ChannelDefinitions } -// ChannelDefinitionOptsCache caches parsed channel definition opts to avoid -// repeated JSON unmarshalling in hot paths. Stores parsed opts as `any` which requires type -// assertion but is orders of magnitude faster than JSON parsing. -// Plugin controls cache lifecycle (set on channel add/update, delete on channel remove) -// OCR phases such as Observation and Reports use the cache to avoid repeated JSON unmarshalling. +// ChannelDefinitionOptsCache stores parsed channel definition opts to avoid +// repeated JSON unmarshalling in hot paths like Observation and Reports. +// Despite the name, this acts as a store rather than a cache: there is no TTL +// and entries are managed explicitly (set on channel add/update, deleted on remove). +// Stored as `any` requiring type assertion, but orders of magnitude faster than JSON parsing. type ChannelDefinitionOptsCache interface { // Set parses and caches the channel definition opts for the given channelID // The channelOpts should match the ReportCodec's opts type. diff --git a/llo/reportcodecs/evm/report_codec_common.go b/llo/reportcodecs/evm/report_codec_common.go index 1084280..00ef3e8 100644 --- a/llo/reportcodecs/evm/report_codec_common.go +++ b/llo/reportcodecs/evm/report_codec_common.go @@ -16,7 +16,6 @@ import ( ubig "github.com/smartcontractkit/chainlink-data-streams/llo/reportcodecs/evm/utils" ) - // ConvertTimestamp converts a nanosecond timestamp to a specified precision. func ConvertTimestamp(timestampNanos uint64, precision llo.TimeResolution) uint64 { switch precision { diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go index ad7992a..e720cd8 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr.go @@ -14,9 +14,10 @@ import ( ) var ( - _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpackedExpr{} - _ llo.OptsParser = ReportCodecEVMABIEncodeUnpackedExpr{} - _ llo.TimeResolutionProvider = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.ReportCodec = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.OptsParser = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.TimeResolutionProvider = ReportCodecEVMABIEncodeUnpackedExpr{} + _ llo.CalculatedStreamABIProvider = ReportCodecEVMABIEncodeUnpackedExpr{} ) type ReportCodecEVMABIEncodeUnpackedExpr struct { @@ -169,3 +170,24 @@ func (r ReportCodecEVMABIEncodeUnpackedExpr) TimeResolution(parsedOpts any) (llo } return opts.TimeResolution, nil } + +func (r ReportCodecEVMABIEncodeUnpackedExpr) CalculatedStreamABI(parsedOpts any) ([]llo.CalculatedStreamABI, error) { + opts, ok := parsedOpts.(ReportFormatEVMABIEncodeOpts) + if !ok { + return nil, fmt.Errorf("expected ReportFormatEVMABIEncodeOpts, got %T", parsedOpts) + } + var result []llo.CalculatedStreamABI + for _, enc := range opts.ABI { + if len(enc.encoders) > 0 { + e := enc.encoders[0] + if e.Expression != "" || e.ExpressionStreamID != 0 { + result = append(result, llo.CalculatedStreamABI{ + Type: e.Type, + Expression: e.Expression, + ExpressionStreamID: e.ExpressionStreamID, + }) + } + } + } + return result, nil +} diff --git a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go index 4b209cd..acd17fd 100644 --- a/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go +++ b/llo/reportcodecs/evm/report_codec_evm_abi_encode_unpacked_expr_test.go @@ -534,15 +534,12 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_WithAndWithoutParsedOpts(t *testing parsedOpts, err := codec.ParseOpts(optsJSON) require.NoError(t, err) - // Encode with parsed opts encodedWithCache, err := codec.Encode(report, cd, parsedOpts) require.NoError(t, err) - // Encode without parsed opts encodedWithoutCache, err := codec.Encode(report, cd, nil) require.NoError(t, err) - // Both paths should produce identical output assert.Equal(t, encodedWithCache, encodedWithoutCache) } @@ -586,3 +583,172 @@ func TestReportCodecEVMABIEncodeUnpackedExpr_TimeResolution(t *testing.T) { require.Contains(t, err.Error(), "expected ReportFormatEVMABIEncodeOpts") }) } + +func TestReportCodecEVMABIEncodeUnpackedExpr_CalculatedStreamABI(t *testing.T) { + codec := ReportCodecEVMABIEncodeUnpackedExpr{} + + t.Run("parses expression fields from valid JSON", func(t *testing.T) { + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [ + {"type": "int256", "expression": "Sum(s1, s2)", "expressionStreamId": 100} + ] + }`) + + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + result, err := codec.CalculatedStreamABI(parsedOpts) + require.NoError(t, err) + require.Len(t, result, 1) + + assert.Equal(t, "int256", result[0].Type) + assert.Equal(t, "Sum(s1, s2)", result[0].Expression) + assert.Equal(t, llotypes.StreamID(100), result[0].ExpressionStreamID) + }) + + t.Run("parses multiple expression entries", func(t *testing.T) { + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [ + {"type": "int192", "expression": "Mul(s1, s2)", "expressionStreamId": 10}, + {"type": "uint256", "expression": "Div(s3, s4)", "expressionStreamId": 20}, + {"type": "int256", "expression": "Sum(s1_benchmark, s2_benchmark)", "expressionStreamId": 30} + ] + }`) + + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + result, err := codec.CalculatedStreamABI(parsedOpts) + require.NoError(t, err) + require.Len(t, result, 3) + + assert.Equal(t, "int192", result[0].Type) + assert.Equal(t, "Mul(s1, s2)", result[0].Expression) + assert.Equal(t, llotypes.StreamID(10), result[0].ExpressionStreamID) + + assert.Equal(t, "uint256", result[1].Type) + assert.Equal(t, "Div(s3, s4)", result[1].Expression) + assert.Equal(t, llotypes.StreamID(20), result[1].ExpressionStreamID) + + assert.Equal(t, "int256", result[2].Type) + assert.Equal(t, "Sum(s1_benchmark, s2_benchmark)", result[2].Expression) + assert.Equal(t, llotypes.StreamID(30), result[2].ExpressionStreamID) + }) + + t.Run("returns empty for ABI without expressions", func(t *testing.T) { + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [ + {"type": "int192"}, + {"type": "uint256"} + ] + }`) + + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + result, err := codec.CalculatedStreamABI(parsedOpts) + require.NoError(t, err) + require.Len(t, result, 0) + }) + + t.Run("filters out non-expression entries in mixed ABI", func(t *testing.T) { + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [ + {"type": "int192"}, + {"type": "int256", "expression": "Sum(s1, s2)", "expressionStreamId": 100}, + {"type": "uint256"}, + {"type": "int256", "expression": "Mul(s3, s4)", "expressionStreamId": 200} + ] + }`) + + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + result, err := codec.CalculatedStreamABI(parsedOpts) + require.NoError(t, err) + require.Len(t, result, 2) + + assert.Equal(t, llotypes.StreamID(100), result[0].ExpressionStreamID) + assert.Equal(t, llotypes.StreamID(200), result[1].ExpressionStreamID) + }) + + t.Run("includes entry with only expressionStreamId (no expression string)", func(t *testing.T) { + // Edge case: expressionStreamId is set but expression is empty + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [ + {"type": "int256", "expressionStreamId": 50} + ] + }`) + + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + result, err := codec.CalculatedStreamABI(parsedOpts) + require.NoError(t, err) + require.Len(t, result, 1) + + assert.Equal(t, "int256", result[0].Type) + assert.Equal(t, "", result[0].Expression) + assert.Equal(t, llotypes.StreamID(50), result[0].ExpressionStreamID) + }) + + t.Run("returns empty for empty ABI array", func(t *testing.T) { + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "expirationWindow": 3600, + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": [] + }`) + + parsedOpts, err := codec.ParseOpts(optsJSON) + require.NoError(t, err) + + result, err := codec.CalculatedStreamABI(parsedOpts) + require.NoError(t, err) + require.Len(t, result, 0) + }) + + t.Run("errors on invalid type", func(t *testing.T) { + type wrongType struct{} + _, err := codec.CalculatedStreamABI(wrongType{}) + require.Error(t, err) + require.Contains(t, err.Error(), "expected ReportFormatEVMABIEncodeOpts") + }) + + t.Run("errors on nil", func(t *testing.T) { + _, err := codec.CalculatedStreamABI(nil) + require.Error(t, err) + require.Contains(t, err.Error(), "expected ReportFormatEVMABIEncodeOpts") + }) + + t.Run("ParseOpts fails on invalid JSON", func(t *testing.T) { + _, err := codec.ParseOpts([]byte(`{not valid json`)) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to parse EVMABIEncodeUnpackedExpr opts") + }) + + t.Run("ParseOpts fails on malformed abi field", func(t *testing.T) { + optsJSON := []byte(`{ + "baseUSDFee": "1.5", + "feedID": "0x0001020304050607080910111213141516171819202122232425262728293031", + "abi": "not an array" + }`) + _, err := codec.ParseOpts(optsJSON) + require.Error(t, err) + }) +} diff --git a/llo/stream_calculated.go b/llo/stream_calculated.go index 6bb1c3c..bd1406e 100644 --- a/llo/stream_calculated.go +++ b/llo/stream_calculated.go @@ -9,8 +9,6 @@ import ( "sync" "time" - "github.com/goccy/go-json" - "github.com/expr-lang/expr" "github.com/expr-lang/expr/ast" "github.com/expr-lang/expr/parser" @@ -563,26 +561,17 @@ func (p *Plugin) ProcessCalculatedStreams(outcome *Outcome) { continue } - // TODO: Use ChannelDefinitionOptsCache to avoid repeated JSON unmarshaling. - // Type assert cached opts to evm.ReportFormatEVMABIEncodeOpts and access ABI field directly. - copt := opts{} - if err := json.Unmarshal(cd.Opts, &copt); err != nil { - p.Logger.Errorw("failed to unmarshal channel definition options", "channelID", cid, "error", err) - env.release() - continue - } - - if len(copt.ABI) == 0 { - p.Logger.Errorw("no expressions found in channel definition", "channelID", cid) + abiEntries, abiErr := p.getCalculatedStreamABI(cid, cd) + if abiErr != nil { + p.Logger.Errorw("failed to get calculated stream ABI", "channelID", cid, "error", abiErr) env.release() continue - } // channel definitions are inherited from the previous outcome, // so we only update the channel definition streams if we haven't done it before - if cd.Streams[len(cd.Streams)-1].StreamID != copt.ABI[len(copt.ABI)-1].ExpressionStreamID { - for _, abi := range copt.ABI { + if cd.Streams[len(cd.Streams)-1].StreamID != abiEntries[len(abiEntries)-1].ExpressionStreamID { + for _, abi := range abiEntries { cd.Streams = append(cd.Streams, llotypes.Stream{ StreamID: abi.ExpressionStreamID, Aggregator: llotypes.AggregatorCalculated, @@ -591,15 +580,45 @@ func (p *Plugin) ProcessCalculatedStreams(outcome *Outcome) { outcome.ChannelDefinitions[cid] = cd } - if err := p.evalExpression(&copt, cid, env, outcome); err != nil { + if err := p.evalCalculatedExpression(abiEntries, cid, env, outcome); err != nil { p.Logger.Errorw("failed to process expression", "channelID", cid, "error", err) } env.release() } } -func (p *Plugin) evalExpression(o *opts, cid llotypes.ChannelID, env environment, outcome *Outcome) error { - for _, abi := range o.ABI { +// getCalculatedStreamABI retrieves calculated stream ABI entries from cached opts. +// Returns an error if the codec doesn't support CalculatedStreamABIProvider or opts aren't cached. +func (p *Plugin) getCalculatedStreamABI(cid llotypes.ChannelID, cd llotypes.ChannelDefinition) ([]CalculatedStreamABI, error) { + codec, ok := p.ReportCodecs[cd.ReportFormat] + if !ok { + return nil, fmt.Errorf("codec not found for report format: %s", cd.ReportFormat) + } + + provider, ok := codec.(CalculatedStreamABIProvider) + if !ok { + return nil, fmt.Errorf("codec does not implement CalculatedStreamABIProvider") + } + + cached, exists := p.ChannelDefinitionOptsCache.Get(cid) + if !exists { + return nil, fmt.Errorf("opts not found in channel definition opts cache. opts may have failed to parse earlier if they were invalid") + } + + abiEntries, err := provider.CalculatedStreamABI(cached) + if err != nil { + return nil, fmt.Errorf("failed to get calculated stream ABI: %w", err) + } + + if len(abiEntries) == 0 { + return nil, fmt.Errorf("no expressions found in channel definition") + } + + return abiEntries, nil +} + +func (p *Plugin) evalCalculatedExpression(abiEntries []CalculatedStreamABI, cid llotypes.ChannelID, env environment, outcome *Outcome) error { + for _, abi := range abiEntries { if abi.ExpressionStreamID == 0 { return fmt.Errorf("expression stream ID is 0, channelID: %d, expression: %s", cid, abi.Expression) @@ -707,20 +726,14 @@ func (p *Plugin) ProcessCalculatedStreamsDryRun(expression string) error { } // Process the calculated streams - o := &opts{ - ABI: []struct { - Type string `json:"type"` - Expression string `json:"expression"` - ExpressionStreamID llotypes.StreamID `json:"expressionStreamID"` - }{ - { - Type: "int256", - Expression: expression, - ExpressionStreamID: 999, - }, + abiEntries := []CalculatedStreamABI{ + { + Type: "int256", + Expression: expression, + ExpressionStreamID: 999, }, } - err = p.evalExpression(o, 1, env, &outcome) + err = p.evalCalculatedExpression(abiEntries, 1, env, &outcome) if err != nil { return fmt.Errorf("failed to process expression: %w", err) } diff --git a/llo/stream_calculated_test.go b/llo/stream_calculated_test.go index b784359..b25b085 100644 --- a/llo/stream_calculated_test.go +++ b/llo/stream_calculated_test.go @@ -7,6 +7,7 @@ import ( "strconv" "testing" + "github.com/goccy/go-json" "github.com/shopspring/decimal" "github.com/smartcontractkit/chainlink-common/pkg/logger" llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" @@ -14,6 +15,36 @@ import ( "github.com/stretchr/testify/require" ) +// mockCalculatedStreamCodec implements ReportCodec, OptsParser, and CalculatedStreamABIProvider for tests. +// ParseOpts parses JSON to populate the cache; CalculatedStreamABI returns the cached value. +// Note: ParseOpts correctness is tested in the real codec's tests - this is just test setup. +type mockCalculatedStreamCodec struct{} + +func (m mockCalculatedStreamCodec) Encode(r Report, cd llotypes.ChannelDefinition, parsedOpts any) ([]byte, error) { + return nil, nil +} + +func (m mockCalculatedStreamCodec) Verify(cd llotypes.ChannelDefinition) error { + return nil +} + +func (m mockCalculatedStreamCodec) ParseOpts(opts []byte) (any, error) { + var parsed struct { + ABI []CalculatedStreamABI `json:"abi"` + } + if err := json.Unmarshal(opts, &parsed); err != nil { + return nil, err + } + return parsed.ABI, nil +} + +func (m mockCalculatedStreamCodec) CalculatedStreamABI(parsedOpts any) ([]CalculatedStreamABI, error) { + if abi, ok := parsedOpts.([]CalculatedStreamABI); ok { + return abi, nil + } + return nil, nil +} + func TestToDecimal(t *testing.T) { tests := []struct { name string @@ -1710,27 +1741,28 @@ func TestProcessStreamCalculated(t *testing.T) { }, }, }, - { - name: "invalid JSON options", - outcome: Outcome{ - ChannelDefinitions: llotypes.ChannelDefinitions{ - 1: { - ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpackedExpr, - Streams: []llotypes.Stream{ - {StreamID: 1, Aggregator: llotypes.AggregatorMedian}, - }, - Opts: []byte(`invalid json`), - }, - }, - }, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { lggr, err := logger.New() require.NoError(t, err) - p := &Plugin{Logger: lggr} + + codec := mockCalculatedStreamCodec{} + cache := NewChannelDefinitionOptsCache() + + // Populate cache - ParseOpts parses the JSON and caches the result + for cid, cd := range tt.outcome.ChannelDefinitions { + cache.Set(cid, cd.Opts, codec) + } + + p := &Plugin{ + Logger: lggr, + ReportCodecs: map[llotypes.ReportFormat]ReportCodec{ + llotypes.ReportFormatEVMABIEncodeUnpackedExpr: codec, + }, + ChannelDefinitionOptsCache: cache, + } p.ProcessCalculatedStreams(&tt.outcome) for streamID, expectedValue := range tt.expectedValues { @@ -1745,6 +1777,71 @@ func TestProcessStreamCalculated(t *testing.T) { } } +// TestProcessCalculatedStreams_CacheMissAndHit explicitly tests the cache miss and hit scenarios. +func TestProcessCalculatedStreams_CacheMissAndHit(t *testing.T) { + lggr, err := logger.New() + require.NoError(t, err) + + tests := []struct { + name string + cached bool + }{ + + { + name: "cached", + cached: true, + }, + { + name: "not cached (miss)", + cached: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + codec := mockCalculatedStreamCodec{} + cache := NewChannelDefinitionOptsCache() + + p := &Plugin{ + Logger: lggr, + ReportCodecs: map[llotypes.ReportFormat]ReportCodec{ + llotypes.ReportFormatEVMABIEncodeUnpackedExpr: codec, + }, + ChannelDefinitionOptsCache: cache, + } + outcome := Outcome{ + ChannelDefinitions: llotypes.ChannelDefinitions{ + 1: { + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpackedExpr, + Streams: []llotypes.Stream{{StreamID: 1, Aggregator: llotypes.AggregatorMedian}}, + Opts: []byte(`{"abi":[{"type":"int256","expression":"s1","expressionStreamID":2}]}`), + }, + }, + StreamAggregates: StreamAggregates{ + 1: {llotypes.AggregatorMedian: ToDecimal(decimal.NewFromInt(100))}, + }, + } + + if tt.cached { + populateCache(t, cache, 1, outcome.ChannelDefinitions[1], p.ReportCodecs) + } else { + _, cached := cache.Get(1) + assert.False(t, cached, "cache should be empty for cache miss test") + } + + p.ProcessCalculatedStreams(&outcome) + + // Stream 2 is the calculated stream (expressionStreamID: 2) + _, hasCalculatedStream := outcome.StreamAggregates[2] + if tt.cached { + assert.True(t, hasCalculatedStream, "should have calculated stream when cache hit") + } else { + assert.False(t, hasCalculatedStream, "should not have calculated stream when cache miss") + } + }) + } +} + func BenchmarkProcessCalculatedStreams(b *testing.B) { aggr := StreamAggregates{ 1: {llotypes.AggregatorMedian: ToDecimal(decimal.NewFromInt(2))}, @@ -1767,7 +1864,19 @@ func BenchmarkProcessCalculatedStreams(b *testing.B) { StreamAggregates: aggr, } - p := &Plugin{Logger: logger.Nop()} + codec := mockCalculatedStreamCodec{} + cache := NewChannelDefinitionOptsCache() + for cid, cd := range outcome.ChannelDefinitions { + cache.Set(cid, cd.Opts, codec) + } + + p := &Plugin{ + Logger: logger.Nop(), + ReportCodecs: map[llotypes.ReportFormat]ReportCodec{ + llotypes.ReportFormatEVMABIEncodeUnpackedExpr: codec, + }, + ChannelDefinitionOptsCache: cache, + } for i := 0; i < b.N; i++ { p.ProcessCalculatedStreams(&outcome) diff --git a/llo/types.go b/llo/types.go index ad4b4dc..b3a429c 100644 --- a/llo/types.go +++ b/llo/types.go @@ -26,7 +26,7 @@ type ReportCodec interface { // may return error instead) // // parsedOpts is a pre-parsed instantiation of ChannelDefinition.Opts which is created from - // the codecs Opts struct. cd.Opts can be nil and is up to the codec to determine + // the codecs Opts struct. cd.Opts can be nil and is up to the codec to determine // if it needs Opts. If the codec does not have Opts it should pass in nil. // For codecs with opts: // If parsedOpts is nil, the codec is expected to parse cd.Opts directly. @@ -50,15 +50,15 @@ type ReportCodec interface { // and the Codecs can implement all interfaces that match the specific Opt. type OptsParser interface { // ParseOpts parses the raw opts bytes and returns a codec-specific - // parsed opts structure as `any`. + // parsed opts structure as `any`. // Use the returned interface to type assert to the specific Opts type. // For Example: // opts, err := codec.ParseOpts(optsBytes) // if err != nil { // return nil, err // } - // - // optsProvider, ok := opts.(OptsProvider) + // + // optsProvider, ok := opts.(OptsProvider) // if !ok { // return nil, fmt.Errorf("invalid opts type") // } @@ -72,6 +72,20 @@ type TimeResolutionProvider interface { TimeResolution(parsedOpts any) (TimeResolution, error) } +// CalculatedStreamABI represents expression config needed for calculated streams +type CalculatedStreamABI struct { + Type string + Expression string + ExpressionStreamID llotypes.StreamID +} + +// CalculatedStreamABIProvider extracts expression ABI config from parsed codec opts +type CalculatedStreamABIProvider interface { + // CalculatedStreamABI returns the expression ABI entries from the parsed opts. + // The parsedOpts must be the value returned by OptsParser.ParseOpts. + CalculatedStreamABI(parsedOpts any) ([]CalculatedStreamABI, error) +} + type ChannelDefinitionWithID struct { llotypes.ChannelDefinition ChannelID llotypes.ChannelID From 290ec8e14a02e77e3ccd98c18dda42944ed1d841 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Mon, 8 Dec 2025 21:00:07 -0700 Subject: [PATCH 12/13] simplify --- llo/channel_definitions_bench_test.go | 66 ++++++++++++--------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/llo/channel_definitions_bench_test.go b/llo/channel_definitions_bench_test.go index 1ea2743..420205e 100644 --- a/llo/channel_definitions_bench_test.go +++ b/llo/channel_definitions_bench_test.go @@ -9,27 +9,22 @@ import ( llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" ) -// ============================================================================= -// Benchmark: Direct JSON Parsing vs Cache Lookup -// ============================================================================= -// -// These benchmarks demonstrate why we cache parsed channel opts instead of -// parsing JSON on every access. Run with: +// Benchmark the cost of parsing JSON vs using the cache. // +// Run with: // go test . -bench=BenchmarkChannelOptsCache -benchmem -run=NONE // // Expected results (approximate): // -// DirectParse (JSON): ~600-800 ns/op | 640 B/op | 6 allocs -// CacheGet: ~15-25 ns/op | 0 B/op | 0 allocs -// Speedup: ~40-50x faster, zero allocations +// DirectParse: ~600-800 ns/op | 640 B/op | 6 allocs +// CacheGet: ~15-25 ns/op | 0 B/op | 0 allocs +// Speedup: ~40-50x faster, zero allocations // // ============================================================================= -// Realistic opts JSON for existing codecs that use JSON opts format +// Example opts for existing codecs that use JSON opts format var benchmarkOptsJSON = []byte(`{"feedID":"0x0001020304050607080910111213141516171819202122232425262728293031","baseUSDFee":"1.5","expirationWindow":3600,"timeResolution":"ns","abi":[{"type":"int192","expression":"Sum(s1,s2)","expressionStreamId":100},{"type":"int192"},{"type":"uint256"}]}`) -// benchParsedOpts mirrors existing codec structures type benchParsedOpts struct { FeedID string `json:"feedID"` BaseUSDFee string `json:"baseUSDFee"` @@ -44,7 +39,6 @@ type abiEntry struct { ExpressionStreamID uint32 `json:"expressionStreamId,omitempty"` } -// benchMockCodec implements OptsParser for benchmarks type benchMockCodec struct{} func (benchMockCodec) Encode(Report, llotypes.ChannelDefinition, any) ([]byte, error) { @@ -59,32 +53,32 @@ func (benchMockCodec) ParseOpts(opts []byte) (any, error) { return parsed, nil } -// BenchmarkChannelOptsCache_DirectParse measures the cost of parsing opts directly. -// Note: actual parsing cost depends on codec implementation (JSON, protobuf, etc.) -// This benchmark uses JSON as representative of current codecs. -func BenchmarkChannelOptsCache_DirectParse(b *testing.B) { - for i := 0; i < b.N; i++ { - var opts benchParsedOpts - _ = json.Unmarshal(benchmarkOptsJSON, &opts) - } -} +func BenchmarkChannelOptsCache(b *testing.B) { + b.Run("DirectParse", func(b *testing.B) { + // Measures the cost of parsing JSON directly. + // This is the usage pattern without caching. + for i := 0; i < b.N; i++ { + var opts benchParsedOpts + _ = json.Unmarshal(benchmarkOptsJSON, &opts) + } + }) -// BenchmarkChannelOptsCache_CacheGet measures the cost of a cache lookup + type assertion. -// This is the required usage pattern for the cache. -func BenchmarkChannelOptsCache_CacheGet(b *testing.B) { - cache := NewChannelDefinitionOptsCache() - codec := benchMockCodec{} + b.Run("CacheGet", func(b *testing.B) { + // Measures the cost of cache lookup + type assertion. + // This is the required usage pattern for the cache. + cache := NewChannelDefinitionOptsCache() + codec := benchMockCodec{} - if err := cache.Set(1, benchmarkOptsJSON, codec); err != nil { - b.Fatal(err) - } + if err := cache.Set(1, benchmarkOptsJSON, codec); err != nil { + b.Fatal(err) + } - b.ResetTimer() - // Usage pattern is 1) Get and 2) type assertion - for i := 0; i < b.N; i++ { - val, ok := cache.Get(1) - if ok { - _ = val.(benchParsedOpts) + b.ResetTimer() + for i := 0; i < b.N; i++ { + val, ok := cache.Get(1) + if ok { + _ = val.(benchParsedOpts) + } } - } + }) } From 38f0d27204bf7a0931304cf3f1a3e2bbca0a40e7 Mon Sep 17 00:00:00 2001 From: Alex Kuznicki Date: Tue, 9 Dec 2025 11:22:33 -0700 Subject: [PATCH 13/13] remove redundant test --- llo/plugin_outcome_test.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/llo/plugin_outcome_test.go b/llo/plugin_outcome_test.go index 91cf806..2f02e8a 100644 --- a/llo/plugin_outcome_test.go +++ b/llo/plugin_outcome_test.go @@ -1027,34 +1027,6 @@ func Test_Outcome_Methods(t *testing.T) { }) - t.Run("IsReportable with ReportFormatEVMABIEncodeUnpackedExpr prevents same-second timestamps", func(t *testing.T) { - outcome := Outcome{} - cid := llotypes.ChannelID(1) - - // Test case from production bug: both timestamps truncate to same second - // observationTimestampNanoseconds: 1765173183742021148 → 1765173183 sec - // validAfterNanoseconds: 1765173183282997914 → 1765173183 sec - outcome.LifeCycleStage = LifeCycleStageProduction - outcome.ObservationTimestampNanoseconds = 1765173183742021148 - outcome.ChannelDefinitions = map[llotypes.ChannelID]llotypes.ChannelDefinition{ - cid: {ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpackedExpr}, - } - outcome.ValidAfterNanoseconds = map[llotypes.ChannelID]uint64{ - cid: 1765173183282997914, - } - - // Should be unreportable because timestamps are in same second - require.EqualError(t, outcome.IsReportable(cid, 1, uint64(0)), - "ChannelID: 1; Reason: ChannelID: 1; Reason: IsReportable=false; not valid yet (observationsTimestampSeconds=1765173183, validAfterSeconds=1765173183)") - - // When ValidAfter is at least 1 second before observation, should be reportable - outcome.ValidAfterNanoseconds[cid] = 1765173182999999999 // 1ns before previous second - assert.Nil(t, outcome.IsReportable(cid, 1, uint64(0))) - - // Test IsSecondsResolution returns true for calculated streams - assert.True(t, IsSecondsResolution(llotypes.ReportFormatEVMABIEncodeUnpackedExpr), - "IsSecondsResolution should return true for ReportFormatEVMABIEncodeUnpackedExpr") - }) t.Run("ReportableChannels", func(t *testing.T) { defaultMinReportInterval := uint64(1 * time.Second)