Alert upon an unknown future upgrade (#4474)

Author: StephenButtolph

node/node.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"math"
 	"net"
 	"net/netip"
 	"os"
@@ -25,6 +26,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 
 	"github.com/ava-labs/avalanchego/api/admin"
 	"github.com/ava-labs/avalanchego/api/health"
@@ -105,6 +107,7 @@ const (
 	responsesNamespace       = constants.PlatformName + metric.NamespaceSeparator + "responses"
 	rpcchainvmNamespace      = constants.PlatformName + metric.NamespaceSeparator + "rpcchainvm"
 	systemResourcesNamespace = constants.PlatformName + metric.NamespaceSeparator + "system_resources"
+	upgradeNamespace         = constants.PlatformName + metric.NamespaceSeparator + "upgrade"
 )
 
 var (
@@ -113,8 +116,12 @@ var (
 
 	indexerDBPrefix = []byte{0x00}
 
-	errInvalidTLSKey = errors.New("invalid TLS key")
-	errShuttingDown  = errors.New("server shutting down")
+	errInvalidTLSKey        = errors.New("invalid TLS key")
+	errShuttingDown         = errors.New("server shutting down")
+	errNoValidators         = errors.New("no validators in the current validator set")
+	errUpgradeNeeded        = errors.New("unknown network upgrade detected")
+	errUpgradeWithinTheDay  = errors.New("unknown network upgrade detected - update as soon as possible")
+	errUpgradeWithinTheHour = errors.New("imminent network upgrade detected - update immediately")
 )
 
 // New returns an instance of Node
@@ -1498,6 +1505,126 @@ func (n *Node) initHealthAPI() error {
 		return fmt.Errorf("couldn't register bls health check: %w", err)
 	}
 
+	upgradeReg, err := metrics.MakeAndRegister(
+		n.MetricsGatherer,
+		upgradeNamespace,
+	)
+	if err != nil {
+		return fmt.Errorf("couldn't create upgrade metrics register: %w", err)
+	}
+
+	timeUntilUpgradeMetric := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "time_until",
+		Help: "Time until an upcoming network upgrade (ns). +Inf means the upgrade is unscheduled.",
+	})
+	infinity := math.Inf(1)
+	timeUntilUpgradeMetric.Set(infinity)
+	if err := upgradeReg.Register(timeUntilUpgradeMetric); err != nil {
+		return fmt.Errorf("couldn't register time until upgrade metric: %w", err)
+	}
+
+	// TODO: This healthcheck calls both n.vdrs.GetMap and n.Net.PeerInfo which
+	// are expensive calls. This could be rewritten as an event based monitor to
+	// avoid expensive iteration.
+	var (
+		localUpgradeTime     = n.Config.UpgradeConfig.GraniteTime
+		localUpgradeTimeUnix = uint64(localUpgradeTime.Unix())
+		lastLogTime          time.Time
+	)
+	futureUpgradeCheck := health.CheckerFunc(func(context.Context) (interface{}, error) {
+		var (
+			currentValidators = n.vdrs.GetMap(constants.PrimaryNetworkID)
+			totalWeight       uint64
+		)
+		for _, vdr := range currentValidators {
+			totalWeight += vdr.Weight
+		}
+		if totalWeight == 0 {
+			return nil, errNoValidators
+		}
+
+		var (
+			peers               = n.Net.PeerInfo(maps.Keys(currentValidators))
+			upgradeTimes        = make(map[uint64]uint64) // upgrade time -> stake weight
+			modeUpgradeTimeUnix uint64
+			modeUpgradeWeight   uint64
+		)
+		for _, peer := range peers {
+			vdr := currentValidators[peer.ID]
+			upgradeWeight := upgradeTimes[peer.UpgradeTime]
+			upgradeWeight += vdr.Weight
+			upgradeTimes[peer.UpgradeTime] = upgradeWeight
+
+			if upgradeWeight > modeUpgradeWeight {
+				modeUpgradeTimeUnix = peer.UpgradeTime
+				modeUpgradeWeight = upgradeWeight
+			}
+		}
+
+		modeUpgradeWeightPortion := float64(modeUpgradeWeight) / float64(totalWeight)
+		result := map[string]interface{}{
+			"localUpgradeTime":            localUpgradeTime,
+			"modeUpgradeTime":             time.Unix(int64(modeUpgradeTimeUnix), 0).UTC(),
+			"modeUpgradeWeightPercentage": 100 * modeUpgradeWeightPortion,
+			"numUpgradeTimes":             len(upgradeTimes),
+		}
+		if localUpgradeTimeUnix >= modeUpgradeTimeUnix || modeUpgradeWeightPortion < .5 {
+			timeUntilUpgradeMetric.Set(infinity)
+			return result, nil
+		}
+
+		const (
+			day  = 24 * time.Hour
+			week = 7 * day
+		)
+		modeUpgradeTime := time.Unix(int64(modeUpgradeTimeUnix), 0)
+		timeUntilUpgrade := time.Until(modeUpgradeTime)
+		timeUntilUpgradeMetric.Set(float64(timeUntilUpgrade))
+		result["timeUntilUpgrade"] = timeUntilUpgrade.String()
+
+		var (
+			logFrequency time.Duration
+			log          func(msg string, fields ...zap.Field)
+			err          error
+		)
+		switch {
+		case timeUntilUpgrade > week:
+			logFrequency = 12 * time.Hour
+			log = n.Log.Info
+		case timeUntilUpgrade > 3*day:
+			logFrequency = 12 * time.Hour
+			log = n.Log.Warn
+		case timeUntilUpgrade > day:
+			logFrequency = time.Hour
+			log = n.Log.Warn
+			err = errUpgradeNeeded
+		case timeUntilUpgrade > time.Hour:
+			logFrequency = time.Hour
+			log = n.Log.Error
+			err = errUpgradeWithinTheDay
+		default:
+			logFrequency = 0 // log at the rate of the health check
+			log = n.Log.Error
+			err = errUpgradeWithinTheHour
+		}
+
+		if time.Since(lastLogTime) >= logFrequency {
+			log("unknown upgrade detected - this node should be updated to a compatible version",
+				zap.String("latestReleaseURL", "https://github.com/ava-labs/avalanchego/releases/latest"),
+				zap.Time("upgradeTime", modeUpgradeTime),
+				zap.Duration("timeUntilUpgrade", timeUntilUpgrade),
+				zap.Error(err),
+			)
+			lastLogTime = time.Now()
+		}
+		return result, err
+	})
+
+	err = n.health.RegisterHealthCheck("futureupgrade", futureUpgradeCheck, health.ApplicationTag)
+	if err != nil {
+		return fmt.Errorf("couldn't register future upgrade health check: %w", err)
+	}
+
 	handler, err := health.NewGetAndPostHandler(n.Log, n.health)
 	if err != nil {
 		return err