From de6bcc796fa85425c69674a6ca1137bfb5ed1e30 Mon Sep 17 00:00:00 2001 From: "F. Eugene Aumson" Date: Tue, 20 Aug 2024 00:17:23 +0000 Subject: [PATCH 1/2] add health check endpoint tested by putting a 30 second sleep into the signature-aggregator e2e test and then while it was sleeping did: ``` $ curl localhost:8082/health {"status":"up","details":{"signature-aggregator":{"status":"up","timestamp":"2024-08-20T01:23:14.351599017Z"}}} ``` --- go.mod | 96 +++++++++---------- signature-aggregator/config/config.go | 6 +- signature-aggregator/config/keys.go | 1 + signature-aggregator/config/viper.go | 1 + .../healthcheck/healthcheck.go | 43 +++++++++ signature-aggregator/main/main.go | 37 +++++-- tests/utils/utils.go | 1 + 7 files changed, 128 insertions(+), 57 deletions(-) create mode 100644 signature-aggregator/healthcheck/healthcheck.go diff --git a/go.mod b/go.mod index be037ce3..6a537bb9 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,9 @@ require ( ) require ( + github.com/DataDog/zstd v1.5.2 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect @@ -39,7 +42,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect github.com/aws/smithy-go v1.20.3 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/btcsuite/btcd/btcutil v1.1.3 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect @@ -50,27 +58,41 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-cmd/cmd v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect github.com/google/renameio/v2 v2.0.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/rpc v1.2.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/hashicorp/go-bexpr v0.1.10 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.2.4 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/jackpal/gateway v1.0.6 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect @@ -80,89 +102,67 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/spf13/afero v1.9.5 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.4.2 // indirect - github.com/urfave/cli/v2 v2.25.7 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - go.opentelemetry.io/otel/metric v1.22.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/tools v0.24.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - rsc.io/tmplfunc v0.0.3 // indirect -) - -require ( - github.com/DataDog/zstd v1.5.2 // indirect - github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/VictoriaMetrics/fastcache v1.12.1 // indirect - github.com/alexliesenfeld/health v0.8.0 - github.com/beorn7/perks v1.0.1 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect - github.com/btcsuite/btcd/btcutil v1.1.3 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/btree v1.1.2 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/mux v1.8.0 // indirect - github.com/gorilla/rpc v1.2.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect - github.com/holiman/bloomfilter/v2 v2.0.3 // indirect - github.com/holiman/uint256 v1.2.4 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pires/go-proxyproto v0.6.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/rs/cors v1.7.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/status-im/keycard-go v0.2.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect + github.com/urfave/cli/v2 v2.25.7 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opentelemetry.io/otel v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect go.opentelemetry.io/otel/sdk v1.22.0 // indirect go.opentelemetry.io/otel/trace v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect - go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.26.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.28.0 // indirect - golang.org/x/sync v0.8.0 golang.org/x/sys v0.23.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.24.0 // indirect gonum.org/v1/gonum v0.11.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) + +require ( + github.com/alexliesenfeld/health v0.8.0 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect + go.uber.org/atomic v1.11.0 + golang.org/x/sync v0.8.0 ) diff --git a/signature-aggregator/config/config.go b/signature-aggregator/config/config.go index 19b769d2..1ed1c105 100644 --- a/signature-aggregator/config/config.go +++ b/signature-aggregator/config/config.go @@ -12,8 +12,9 @@ import ( ) const ( - defaultAPIPort = uint16(8080) - defaultMetricsPort = uint16(8081) + defaultAPIPort = uint16(8080) + defaultMetricsPort = uint16(8081) + defaultHealthCheckPort = uint16(8082) DefaultSignatureCacheSize = uint64(1024 * 1024) ) @@ -33,6 +34,7 @@ type Config struct { InfoAPI *basecfg.APIConfig `mapstructure:"info-api" json:"info-api"` APIPort uint16 `mapstructure:"api-port" json:"api-port"` MetricsPort uint16 `mapstructure:"metrics-port" json:"metrics-port"` + HealthCheckPort uint16 `mapstructure:"health-check-port" json:"health-check-port"` SignatureCacheSize uint64 `mapstructure:"signature-cache-size" json:"signature-cache-size"` } diff --git a/signature-aggregator/config/keys.go b/signature-aggregator/config/keys.go index 97829f77..1707b744 100644 --- a/signature-aggregator/config/keys.go +++ b/signature-aggregator/config/keys.go @@ -15,5 +15,6 @@ const ( InfoAPIKey = "info-api" APIPortKey = "api-port" MetricsPortKey = "metrics-port" + HealthCheckPortKey = "health-check-port" SignatureCacheSizeKey = "signature-cache-size" ) diff --git a/signature-aggregator/config/viper.go b/signature-aggregator/config/viper.go index dc33f918..600f96d0 100644 --- a/signature-aggregator/config/viper.go +++ b/signature-aggregator/config/viper.go @@ -57,6 +57,7 @@ func SetDefaultConfigValues(v *viper.Viper) { SignatureCacheSizeKey, DefaultSignatureCacheSize, ) + v.SetDefault(HealthCheckPortKey, defaultHealthCheckPort) } // BuildConfig constructs the signature aggregator config using Viper. diff --git a/signature-aggregator/healthcheck/healthcheck.go b/signature-aggregator/healthcheck/healthcheck.go new file mode 100644 index 00000000..d388788b --- /dev/null +++ b/signature-aggregator/healthcheck/healthcheck.go @@ -0,0 +1,43 @@ +package healthcheck + +import ( + "context" + "fmt" + "net/http" + + "github.com/alexliesenfeld/health" + "github.com/ava-labs/avalanchego/utils/logging" + "go.uber.org/zap" +) + +func Run( + logger logging.Logger, + port uint16, + checkFunc func(context.Context) error, +) { + healthChecker := health.NewChecker( + health.WithCheck(health.Check{ + Name: "signature-aggregator", + Check: checkFunc, + }), + ) + + http.Handle("/health", health.NewHandler(healthChecker)) + + go func() { + logger.Info( + "Listening for health check requests", + zap.Uint16("port", port), + ) + err := http.ListenAndServe( + fmt.Sprintf(":%d", port), + nil, + ) + if err != nil { + logger.Fatal( + "failed to listen and serve HTTP", + zap.Error(err), + ) + } + }() +} diff --git a/signature-aggregator/main/main.go b/signature-aggregator/main/main.go index 88053bb2..cf7b53f4 100644 --- a/signature-aggregator/main/main.go +++ b/signature-aggregator/main/main.go @@ -4,11 +4,13 @@ package main import ( + "context" "errors" "fmt" "log" "net/http" "os" + "sync" "github.com/ava-labs/avalanchego/message" "github.com/ava-labs/avalanchego/utils/constants" @@ -17,6 +19,7 @@ import ( "github.com/ava-labs/awm-relayer/signature-aggregator/aggregator" "github.com/ava-labs/awm-relayer/signature-aggregator/api" "github.com/ava-labs/awm-relayer/signature-aggregator/config" + "github.com/ava-labs/awm-relayer/signature-aggregator/healthcheck" "github.com/ava-labs/awm-relayer/signature-aggregator/metrics" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -74,6 +77,19 @@ func main() { logger.Info("Initializing signature-aggregator") + var ready bool = false + healthcheck.Run( + logger, + cfg.HealthCheckPort, + func(context.Context) error { + if ready { + return nil + } else { + return errors.New("not yet ready") + } + }, + ) + // Initialize the global app request network logger.Info("Initializing app request network") // The app request network generates P2P networking logs that are verbose at the info level. @@ -128,11 +144,18 @@ func main() { signatureAggregator, ) - err = http.ListenAndServe(fmt.Sprintf(":%d", cfg.APIPort), nil) - if errors.Is(err, http.ErrServerClosed) { - logger.Info("server closed") - } else if err != nil { - logger.Error("server error", zap.Error(err)) - log.Fatal(err) - } + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = http.ListenAndServe(fmt.Sprintf(":%d", cfg.APIPort), nil) + if errors.Is(err, http.ErrServerClosed) { + logger.Info("server closed") + } else if err != nil { + logger.Error("server error", zap.Error(err)) + log.Fatal(err) + } + }() + ready = true + wg.Wait() } diff --git a/tests/utils/utils.go b/tests/utils/utils.go index 48500c14..5ddf8224 100644 --- a/tests/utils/utils.go +++ b/tests/utils/utils.go @@ -301,6 +301,7 @@ func CreateDefaultSignatureAggregatorConfig( }, APIPort: 8080, MetricsPort: 8081, + HealthCheckPort: 8082, SignatureCacheSize: (1024 * 1024), } } From 7edf1bd8737b3aa892bf8d77a347a0a5f38a668f Mon Sep 17 00:00:00 2001 From: "F. Eugene Aumson" Date: Tue, 20 Aug 2024 15:10:20 +0000 Subject: [PATCH 2/2] use existing port, not a new one --- signature-aggregator/config/config.go | 6 +-- signature-aggregator/config/keys.go | 1 - signature-aggregator/config/viper.go | 1 - .../healthcheck/healthcheck.go | 28 +------------ signature-aggregator/main/main.go | 39 +++++-------------- tests/utils/utils.go | 1 - 6 files changed, 13 insertions(+), 63 deletions(-) diff --git a/signature-aggregator/config/config.go b/signature-aggregator/config/config.go index 1ed1c105..19b769d2 100644 --- a/signature-aggregator/config/config.go +++ b/signature-aggregator/config/config.go @@ -12,9 +12,8 @@ import ( ) const ( - defaultAPIPort = uint16(8080) - defaultMetricsPort = uint16(8081) - defaultHealthCheckPort = uint16(8082) + defaultAPIPort = uint16(8080) + defaultMetricsPort = uint16(8081) DefaultSignatureCacheSize = uint64(1024 * 1024) ) @@ -34,7 +33,6 @@ type Config struct { InfoAPI *basecfg.APIConfig `mapstructure:"info-api" json:"info-api"` APIPort uint16 `mapstructure:"api-port" json:"api-port"` MetricsPort uint16 `mapstructure:"metrics-port" json:"metrics-port"` - HealthCheckPort uint16 `mapstructure:"health-check-port" json:"health-check-port"` SignatureCacheSize uint64 `mapstructure:"signature-cache-size" json:"signature-cache-size"` } diff --git a/signature-aggregator/config/keys.go b/signature-aggregator/config/keys.go index 1707b744..97829f77 100644 --- a/signature-aggregator/config/keys.go +++ b/signature-aggregator/config/keys.go @@ -15,6 +15,5 @@ const ( InfoAPIKey = "info-api" APIPortKey = "api-port" MetricsPortKey = "metrics-port" - HealthCheckPortKey = "health-check-port" SignatureCacheSizeKey = "signature-cache-size" ) diff --git a/signature-aggregator/config/viper.go b/signature-aggregator/config/viper.go index 600f96d0..dc33f918 100644 --- a/signature-aggregator/config/viper.go +++ b/signature-aggregator/config/viper.go @@ -57,7 +57,6 @@ func SetDefaultConfigValues(v *viper.Viper) { SignatureCacheSizeKey, DefaultSignatureCacheSize, ) - v.SetDefault(HealthCheckPortKey, defaultHealthCheckPort) } // BuildConfig constructs the signature aggregator config using Viper. diff --git a/signature-aggregator/healthcheck/healthcheck.go b/signature-aggregator/healthcheck/healthcheck.go index d388788b..9c378cee 100644 --- a/signature-aggregator/healthcheck/healthcheck.go +++ b/signature-aggregator/healthcheck/healthcheck.go @@ -2,42 +2,18 @@ package healthcheck import ( "context" - "fmt" "net/http" "github.com/alexliesenfeld/health" - "github.com/ava-labs/avalanchego/utils/logging" - "go.uber.org/zap" ) -func Run( - logger logging.Logger, - port uint16, - checkFunc func(context.Context) error, -) { +func HandleHealthCheckRequest() { healthChecker := health.NewChecker( health.WithCheck(health.Check{ Name: "signature-aggregator", - Check: checkFunc, + Check: func(context.Context) error { return nil }, }), ) http.Handle("/health", health.NewHandler(healthChecker)) - - go func() { - logger.Info( - "Listening for health check requests", - zap.Uint16("port", port), - ) - err := http.ListenAndServe( - fmt.Sprintf(":%d", port), - nil, - ) - if err != nil { - logger.Fatal( - "failed to listen and serve HTTP", - zap.Error(err), - ) - } - }() } diff --git a/signature-aggregator/main/main.go b/signature-aggregator/main/main.go index cf7b53f4..9d20842a 100644 --- a/signature-aggregator/main/main.go +++ b/signature-aggregator/main/main.go @@ -4,13 +4,11 @@ package main import ( - "context" "errors" "fmt" "log" "net/http" "os" - "sync" "github.com/ava-labs/avalanchego/message" "github.com/ava-labs/avalanchego/utils/constants" @@ -77,19 +75,6 @@ func main() { logger.Info("Initializing signature-aggregator") - var ready bool = false - healthcheck.Run( - logger, - cfg.HealthCheckPort, - func(context.Context) error { - if ready { - return nil - } else { - return errors.New("not yet ready") - } - }, - ) - // Initialize the global app request network logger.Info("Initializing app request network") // The app request network generates P2P networking logs that are verbose at the info level. @@ -143,19 +128,13 @@ func main() { metricsInstance, signatureAggregator, ) - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - err = http.ListenAndServe(fmt.Sprintf(":%d", cfg.APIPort), nil) - if errors.Is(err, http.ErrServerClosed) { - logger.Info("server closed") - } else if err != nil { - logger.Error("server error", zap.Error(err)) - log.Fatal(err) - } - }() - ready = true - wg.Wait() + healthcheck.HandleHealthCheckRequest() + + err = http.ListenAndServe(fmt.Sprintf(":%d", cfg.APIPort), nil) + if errors.Is(err, http.ErrServerClosed) { + logger.Info("server closed") + } else if err != nil { + logger.Error("server error", zap.Error(err)) + log.Fatal(err) + } } diff --git a/tests/utils/utils.go b/tests/utils/utils.go index 90855ca6..df6345a2 100644 --- a/tests/utils/utils.go +++ b/tests/utils/utils.go @@ -295,7 +295,6 @@ func CreateDefaultSignatureAggregatorConfig( }, APIPort: 8080, MetricsPort: 8081, - HealthCheckPort: 8082, SignatureCacheSize: (1024 * 1024), } }