Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package legacycvomonitortests
import (
"context"
"fmt"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -639,10 +640,10 @@ func testUpgradeOperatorProgressingStateTransitions(events monitorapi.Intervals,
// CO have not shown up in CVO Progressing message
return fmt.Sprintf("%s completing its update so fast that CVO did not recogize any waiting", co)
}
from, to := fromAndTo(intervals)
if d := to.Sub(from); d < 3*time.Minute {
summary := summarizeWaitingIntervals(intervals)
if summary.total < 3*time.Minute {
// CO showed up in CVO Progressing message but the total duration is less than three minutes
return fmt.Sprintf("%s completing its update within less than three minutes: %s", co, d.String())
return fmt.Sprintf("%s completing its update within less than three minutes: %s", co, summary)
}
switch co {
case "baremetal":
Expand Down Expand Up @@ -684,8 +685,8 @@ func testUpgradeOperatorProgressingStateTransitions(events monitorapi.Intervals,
if exception != "" {
output = fmt.Sprintf("%s which is expected up to %s", output, exception)
} else {
from, to := fromAndTo(COWaiting[operatorName])
output = fmt.Sprintf("%s and CVO waited for it over 3 minutes from %s to %s", output, from.Format(time.RFC3339), to.Format(time.RFC3339))
summary := summarizeWaitingIntervals(COWaiting[operatorName])
output = fmt.Sprintf("%s and CVO waited for it at least three minutes: %s", output, summary)
}
mcTestCase.FailureOutput = &junitapi.FailureOutput{
Output: output,
Expand Down Expand Up @@ -911,17 +912,76 @@ func getUpgradeLevelFromClusterVersionHistory(history []configv1.UpdateHistory)
return patchUpgradeLevel, nil
}

func fromAndTo(intervals monitorapi.Intervals) (time.Time, time.Time) {
var from, to time.Time
for _, interval := range intervals {
if from.IsZero() || interval.From.Before(from) {
from = interval.From
type waitingIntervalSummary struct {
from time.Time
to time.Time
total time.Duration
longest time.Duration
episodes monitorapi.Intervals
}

func (s waitingIntervalSummary) String() string {
if len(s.episodes) == 0 {
return "no valid waiting intervals"
}

ranges := make([]string, 0, len(s.episodes))
for _, episode := range s.episodes {
ranges = append(ranges, fmt.Sprintf("%s..%s", episode.From.Format(time.RFC3339), episode.To.Format(time.RFC3339)))
}

return fmt.Sprintf(
"%s total across %d episode(s), longest %s, observation span %s from %s to %s; ranges: %s",
s.total,
len(s.episodes),
s.longest,
s.to.Sub(s.from),
s.from.Format(time.RFC3339),
s.to.Format(time.RFC3339),
strings.Join(ranges, ", "),
)
}

// summarizeWaitingIntervals excludes gaps during which CVO did not report waiting on the operator.
func summarizeWaitingIntervals(intervals monitorapi.Intervals) waitingIntervalSummary {
sorted := append(monitorapi.Intervals(nil), intervals...)
sort.SliceStable(sorted, func(i, j int) bool {
if sorted[i].From.Equal(sorted[j].From) {
return sorted[i].To.Before(sorted[j].To)
}
return sorted[i].From.Before(sorted[j].From)
})

var summary waitingIntervalSummary
for _, interval := range sorted {
if interval.From.IsZero() || interval.To.Before(interval.From) {
continue
}

if len(summary.episodes) == 0 || interval.From.After(summary.episodes[len(summary.episodes)-1].To) {
summary.episodes = append(summary.episodes, interval)
continue
}

last := &summary.episodes[len(summary.episodes)-1]
if interval.To.After(last.To) {
last.To = interval.To
}
if to.IsZero() || interval.To.After(to) {
to = interval.To
}

for _, episode := range summary.episodes {
duration := episode.To.Sub(episode.From)
summary.total += duration
if duration > summary.longest {
summary.longest = duration
}
}
return from, to
if len(summary.episodes) > 0 {
summary.from = summary.episodes[0].From
summary.to = summary.episodes[len(summary.episodes)-1].To
}

return summary
}

func updateCOWaiting(interval monitorapi.Interval, waiting map[string]monitorapi.Intervals) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -259,14 +259,14 @@ func Test_updateCOWaiting(t *testing.T) {
d: 2 * time.Minute,
message: "working towards ${VERSION}: 106 of 841 done (12% complete), waiting on etcd, kube-apiserver",
waiting: map[string]monitorapi.Intervals{"etcd": {interval("some", now, 3*time.Minute)}},
expect: map[string]time.Duration{"etcd": next + 2*time.Minute, "kube-apiserver": 2 * time.Minute},
expect: map[string]time.Duration{"etcd": 5 * time.Minute, "kube-apiserver": 2 * time.Minute},
},
{
name: "incremental all",
d: 2 * time.Minute,
message: "working towards ${VERSION}: 106 of 841 done (12% complete), waiting on etcd, kube-apiserver",
waiting: map[string]monitorapi.Intervals{"etcd": {interval("some", now, 3*time.Minute)}, "kube-apiserver": {interval("some", now, 6*time.Minute)}},
expect: map[string]time.Duration{"etcd": next + 2*time.Minute, "kube-apiserver": next + 2*time.Minute},
expect: map[string]time.Duration{"etcd": 5 * time.Minute, "kube-apiserver": 8 * time.Minute},
},
{
name: "unknown message",
Expand All @@ -288,14 +288,105 @@ func Test_updateCOWaiting(t *testing.T) {
updateCOWaiting(i, tt.waiting)
actual := map[string]time.Duration{}
for co, intervals := range tt.waiting {
from, to := fromAndTo(intervals)
actual[co] = to.Sub(from)
actual[co] = summarizeWaitingIntervals(intervals).total
}
assert.Equal(t, tt.expect, actual)
})
}
}

func Test_summarizeWaitingIntervals(t *testing.T) {
base := time.Date(2026, time.July, 25, 11, 48, 50, 0, time.UTC)
interval := func(from, to time.Duration) monitorapi.Interval {
return monitorapi.Interval{
From: base.Add(from),
To: base.Add(to),
}
}

tests := []struct {
name string
intervals monitorapi.Intervals
expectedTotal time.Duration
expectedLongest time.Duration
expectedSpan time.Duration
expectedEpisodes int
expectedString string
}{
{
name: "separated point does not fill gap without CVO-reported waiting",
intervals: monitorapi.Intervals{
interval(0, 15*time.Second),
interval(45*time.Minute+14*time.Second, 45*time.Minute+14*time.Second),
},
expectedTotal: 15 * time.Second,
expectedLongest: 15 * time.Second,
expectedSpan: 45*time.Minute + 14*time.Second,
expectedEpisodes: 2,
},
{
name: "overlapping and adjacent intervals form one union",
intervals: monitorapi.Intervals{
interval(2*time.Minute, 4*time.Minute),
interval(0, 3*time.Minute),
interval(4*time.Minute, 5*time.Minute),
},
expectedTotal: 5 * time.Minute,
expectedLongest: 5 * time.Minute,
expectedSpan: 5 * time.Minute,
expectedEpisodes: 1,
},
{
name: "multiple real episodes sum without double counting",
intervals: monitorapi.Intervals{
interval(10*time.Minute, 12*time.Minute),
interval(0, 2*time.Minute),
interval(time.Minute, 3*time.Minute),
},
expectedTotal: 5 * time.Minute,
expectedLongest: 3 * time.Minute,
expectedSpan: 12 * time.Minute,
expectedEpisodes: 2,
},
{
name: "invalid interval is ignored",
intervals: monitorapi.Intervals{
interval(2*time.Minute, time.Minute),
interval(0, 30*time.Second),
},
expectedTotal: 30 * time.Second,
expectedLongest: 30 * time.Second,
expectedSpan: 30 * time.Second,
expectedEpisodes: 1,
},
{
name: "empty input",
expectedString: "no valid waiting intervals",
},
{
name: "all invalid intervals",
intervals: monitorapi.Intervals{
interval(2*time.Minute, time.Minute),
interval(4*time.Minute, 3*time.Minute),
},
expectedString: "no valid waiting intervals",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
summary := summarizeWaitingIntervals(tt.intervals)
assert.Equal(t, tt.expectedTotal, summary.total)
assert.Equal(t, tt.expectedLongest, summary.longest)
assert.Equal(t, tt.expectedSpan, summary.to.Sub(summary.from))
assert.Len(t, summary.episodes, tt.expectedEpisodes)
if tt.expectedString != "" {
assert.Equal(t, tt.expectedString, summary.String())
}
})
}
}

func Test_patchUpgradeWithConfigClient(t *testing.T) {
tests := []struct {
name string
Expand Down