From e9d5125858a06f1127c10436b61d0a835b51225f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Tue, 19 Mar 2024 18:29:52 +0100 Subject: [PATCH] feature: allow configuration for Go x/trace.FlightRecorder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- config/config.go | 103 +++++++++++--------- filters/builtin/builtin.go | 1 + filters/filters.go | 1 + proxy/proxy.go | 192 ++++++++++++++++++++++--------------- skipper.go | 88 ++++++++++++----- 5 files changed, 240 insertions(+), 145 deletions(-) diff --git a/config/config.go b/config/config.go index 12b27336ab..db79601487 100644 --- a/config/config.go +++ b/config/config.go @@ -69,50 +69,55 @@ type Config struct { CompressEncodings *listFlag `yaml:"compress-encodings"` // logging, metrics, profiling, tracing: - EnablePrometheusMetrics bool `yaml:"enable-prometheus-metrics"` - OpenTracing string `yaml:"opentracing"` - OpenTracingInitialSpan string `yaml:"opentracing-initial-span"` - OpenTracingExcludedProxyTags string `yaml:"opentracing-excluded-proxy-tags"` - OpenTracingDisableFilterSpans bool `yaml:"opentracing-disable-filter-spans"` - OpentracingLogFilterLifecycleEvents bool `yaml:"opentracing-log-filter-lifecycle-events"` - OpentracingLogStreamEvents bool `yaml:"opentracing-log-stream-events"` - OpentracingBackendNameTag bool `yaml:"opentracing-backend-name-tag"` - MetricsListener string `yaml:"metrics-listener"` - MetricsPrefix string `yaml:"metrics-prefix"` - EnableProfile bool `yaml:"enable-profile"` - BlockProfileRate int `yaml:"block-profile-rate"` - MutexProfileFraction int `yaml:"mutex-profile-fraction"` - MemProfileRate int `yaml:"memory-profile-rate"` - DebugGcMetrics bool `yaml:"debug-gc-metrics"` - RuntimeMetrics bool `yaml:"runtime-metrics"` - ServeRouteMetrics bool `yaml:"serve-route-metrics"` - ServeRouteCounter bool `yaml:"serve-route-counter"` - ServeHostMetrics bool `yaml:"serve-host-metrics"` - ServeHostCounter bool `yaml:"serve-host-counter"` - ServeMethodMetric bool `yaml:"serve-method-metric"` - ServeStatusCodeMetric bool `yaml:"serve-status-code-metric"` - BackendHostMetrics bool `yaml:"backend-host-metrics"` - AllFiltersMetrics bool `yaml:"all-filters-metrics"` - CombinedResponseMetrics bool `yaml:"combined-response-metrics"` - RouteResponseMetrics bool `yaml:"route-response-metrics"` - RouteBackendErrorCounters bool `yaml:"route-backend-error-counters"` - RouteStreamErrorCounters bool `yaml:"route-stream-error-counters"` - RouteBackendMetrics bool `yaml:"route-backend-metrics"` - RouteCreationMetrics bool `yaml:"route-creation-metrics"` - MetricsUseExpDecaySample bool `yaml:"metrics-exp-decay-sample"` - HistogramMetricBucketsString string `yaml:"histogram-metric-buckets"` - HistogramMetricBuckets []float64 `yaml:"-"` - DisableMetricsCompat bool `yaml:"disable-metrics-compat"` - ApplicationLog string `yaml:"application-log"` - ApplicationLogLevel log.Level `yaml:"-"` - ApplicationLogLevelString string `yaml:"application-log-level"` - ApplicationLogPrefix string `yaml:"application-log-prefix"` - ApplicationLogJSONEnabled bool `yaml:"application-log-json-enabled"` - AccessLog string `yaml:"access-log"` - AccessLogDisabled bool `yaml:"access-log-disabled"` - AccessLogJSONEnabled bool `yaml:"access-log-json-enabled"` - AccessLogStripQuery bool `yaml:"access-log-strip-query"` - SuppressRouteUpdateLogs bool `yaml:"suppress-route-update-logs"` + EnablePrometheusMetrics bool `yaml:"enable-prometheus-metrics"` + OpenTracing string `yaml:"opentracing"` + OpenTracingInitialSpan string `yaml:"opentracing-initial-span"` + OpenTracingExcludedProxyTags string `yaml:"opentracing-excluded-proxy-tags"` + OpenTracingDisableFilterSpans bool `yaml:"opentracing-disable-filter-spans"` + OpentracingLogFilterLifecycleEvents bool `yaml:"opentracing-log-filter-lifecycle-events"` + OpentracingLogStreamEvents bool `yaml:"opentracing-log-stream-events"` + OpentracingBackendNameTag bool `yaml:"opentracing-backend-name-tag"` + MetricsListener string `yaml:"metrics-listener"` + MetricsPrefix string `yaml:"metrics-prefix"` + EnableProfile bool `yaml:"enable-profile"` + BlockProfileRate int `yaml:"block-profile-rate"` + MutexProfileFraction int `yaml:"mutex-profile-fraction"` + MemProfileRate int `yaml:"memory-profile-rate"` + EnableFlightRecorder bool `yaml:"enable-flight-recorder"` + FlightRecorderSize int `yaml:"flight-recorder-size"` + FlightRecorderPeriod time.Duration `yaml:"flight-recorder-period"` + FlightRecorderProxyTookTooLong time.Duration `yaml:"flight-recorder-proxy-took-too-long"` + FlightRecorderTargetURL string `yaml:"flight-recorder-target-url"` + DebugGcMetrics bool `yaml:"debug-gc-metrics"` + RuntimeMetrics bool `yaml:"runtime-metrics"` + ServeRouteMetrics bool `yaml:"serve-route-metrics"` + ServeRouteCounter bool `yaml:"serve-route-counter"` + ServeHostMetrics bool `yaml:"serve-host-metrics"` + ServeHostCounter bool `yaml:"serve-host-counter"` + ServeMethodMetric bool `yaml:"serve-method-metric"` + ServeStatusCodeMetric bool `yaml:"serve-status-code-metric"` + BackendHostMetrics bool `yaml:"backend-host-metrics"` + AllFiltersMetrics bool `yaml:"all-filters-metrics"` + CombinedResponseMetrics bool `yaml:"combined-response-metrics"` + RouteResponseMetrics bool `yaml:"route-response-metrics"` + RouteBackendErrorCounters bool `yaml:"route-backend-error-counters"` + RouteStreamErrorCounters bool `yaml:"route-stream-error-counters"` + RouteBackendMetrics bool `yaml:"route-backend-metrics"` + RouteCreationMetrics bool `yaml:"route-creation-metrics"` + MetricsUseExpDecaySample bool `yaml:"metrics-exp-decay-sample"` + HistogramMetricBucketsString string `yaml:"histogram-metric-buckets"` + HistogramMetricBuckets []float64 `yaml:"-"` + DisableMetricsCompat bool `yaml:"disable-metrics-compat"` + ApplicationLog string `yaml:"application-log"` + ApplicationLogLevel log.Level `yaml:"-"` + ApplicationLogLevelString string `yaml:"application-log-level"` + ApplicationLogPrefix string `yaml:"application-log-prefix"` + ApplicationLogJSONEnabled bool `yaml:"application-log-json-enabled"` + AccessLog string `yaml:"access-log"` + AccessLogDisabled bool `yaml:"access-log-disabled"` + AccessLogJSONEnabled bool `yaml:"access-log-json-enabled"` + AccessLogStripQuery bool `yaml:"access-log-strip-query"` + SuppressRouteUpdateLogs bool `yaml:"suppress-route-update-logs"` // route sources: EtcdUrls string `yaml:"etcd-urls"` @@ -378,6 +383,11 @@ func NewConfig() *Config { flag.IntVar(&cfg.BlockProfileRate, "block-profile-rate", 0, "block profile sample rate, see runtime.SetBlockProfileRate") flag.IntVar(&cfg.MutexProfileFraction, "mutex-profile-fraction", 0, "mutex profile fraction rate, see runtime.SetMutexProfileFraction") flag.IntVar(&cfg.MemProfileRate, "memory-profile-rate", 0, "memory profile rate, see runtime.SetMemProfileRate, keeps default 512 kB") + flag.BoolVar(&cfg.EnableFlightRecorder, "enable-flight-recorder", false, "enable flightrecorder Go tracer") + flag.IntVar(&cfg.FlightRecorderSize, "flight-recorder-size", 0, "max flight-recorder trace data size") + flag.DurationVar(&cfg.FlightRecorderPeriod, "flight-recorder-period", 0, "sets the approximate time duration that the flight recorder's circular buffer represents.") + flag.DurationVar(&cfg.FlightRecorderProxyTookTooLong, "flight-recorder-proxy-took-too-long", 0, "sets the threshold, if proxy took longer than that the flight recorder will write out a trace.") + flag.StringVar(&cfg.FlightRecorderTargetURL, "flight-recorder-target-url", "", "sets the flight recorder target URL that is used to write out the trace to.") flag.BoolVar(&cfg.DebugGcMetrics, "debug-gc-metrics", false, "enables reporting of the Go garbage collector statistics exported in debug.GCStats") flag.BoolVar(&cfg.RuntimeMetrics, "runtime-metrics", true, "enables reporting of the Go runtime statistics exported in runtime and specifically runtime.MemStats") flag.BoolVar(&cfg.ServeRouteMetrics, "serve-route-metrics", false, "enables reporting total serve time metrics for each route") @@ -745,6 +755,11 @@ func (c *Config) ToOptions() skipper.Options { EnableProfile: c.EnableProfile, BlockProfileRate: c.BlockProfileRate, MutexProfileFraction: c.MutexProfileFraction, + EnableFlightRecorder: c.EnableFlightRecorder, + FlightRecorderSize: c.FlightRecorderSize, + FlightRecorderPeriod: c.FlightRecorderPeriod, + FlightRecorderProxyTookTooLong: c.FlightRecorderProxyTookTooLong, + FlightRecorderTargetURL: c.FlightRecorderTargetURL, EnableDebugGcMetrics: c.DebugGcMetrics, EnableRuntimeMetrics: c.RuntimeMetrics, EnableServeRouteMetrics: c.ServeRouteMetrics, diff --git a/filters/builtin/builtin.go b/filters/builtin/builtin.go index d5c9e34f24..d55d3f730c 100644 --- a/filters/builtin/builtin.go +++ b/filters/builtin/builtin.go @@ -189,6 +189,7 @@ func Filters() []filters.Spec { diag.NewNormalResponseLatency(), diag.NewHistogramRequestLatency(), diag.NewHistogramResponseLatency(), + diag.NewTrace(), tee.NewTee(), tee.NewTeeDeprecated(), tee.NewTeeNoFollow(), diff --git a/filters/filters.go b/filters/filters.go index a43c4b19e4..5a2646255f 100644 --- a/filters/filters.go +++ b/filters/filters.go @@ -264,6 +264,7 @@ const ( NormalResponseLatencyName = "normalResponseLatency" HistogramRequestLatencyName = "histogramRequestLatency" HistogramResponseLatencyName = "histogramResponseLatency" + TraceName = "trace" LogBodyName = "logBody" LogHeaderName = "logHeader" TeeName = "tee" diff --git a/proxy/proxy.go b/proxy/proxy.go index 17337ad7ef..81778dced8 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -18,7 +18,6 @@ import ( "runtime" "strconv" "strings" - "sync" "time" "unicode/utf8" @@ -318,6 +317,16 @@ type Params struct { // PassiveHealthCheck defines the parameters for the healthy endpoints checker. PassiveHealthCheck *PassiveHealthCheck + + // FlightRecorder is a started instance of https://pkg.go.dev/golang.org/x/exp/trace#FlightRecorder + FlightRecorder *trace.FlightRecorder + + // FlightRecorderTargetURL is the target to write the trace + // to. Supported targets are http URL and file URL. + FlightRecorderTargetURL string + + // FlightRecorderProxyTookTooLong defines the threshold when to write out a trace + FlightRecorderProxyTookTooLong time.Duration } type ( @@ -387,34 +396,34 @@ type PriorityRoute interface { // Proxy instances implement Skipper proxying functionality. For // initializing, see the WithParams the constructor and Params. type Proxy struct { - experimentalUpgrade bool - experimentalUpgradeAudit bool - accessLogDisabled bool - maxLoops int - defaultHTTPStatus int - routing *routing.Routing - registry *routing.EndpointRegistry - fadein *fadeIn - heathlyEndpoints *healthyEndpoints - roundTripper http.RoundTripper - priorityRoutes []PriorityRoute - flags Flags - metrics metrics.Metrics - quit chan struct{} - flushInterval time.Duration - breakers *circuit.Registry - limiters *ratelimit.Registry - log logging.Logger - tracing *proxyTracing - upgradeAuditLogOut io.Writer - upgradeAuditLogErr io.Writer - auditLogHook chan struct{} - clientTLS *tls.Config - hostname string - onPanicSometimes rate.Sometimes - flightRecorder *trace.FlightRecorder - traceOnce sync.Once - tooLong time.Duration + experimentalUpgrade bool + experimentalUpgradeAudit bool + accessLogDisabled bool + maxLoops int + defaultHTTPStatus int + routing *routing.Routing + registry *routing.EndpointRegistry + fadein *fadeIn + heathlyEndpoints *healthyEndpoints + roundTripper http.RoundTripper + priorityRoutes []PriorityRoute + flags Flags + metrics metrics.Metrics + quit chan struct{} + flushInterval time.Duration + breakers *circuit.Registry + limiters *ratelimit.Registry + log logging.Logger + tracing *proxyTracing + upgradeAuditLogOut io.Writer + upgradeAuditLogErr io.Writer + auditLogHook chan struct{} + clientTLS *tls.Config + hostname string + onPanicSometimes rate.Sometimes + flightRecorder *trace.FlightRecorder + flightRecorderURL *url.URL + flightRecorderProxyTookTooLong time.Duration } // proxyError is used to wrap errors during proxying and to indicate @@ -801,13 +810,15 @@ func WithParams(p Params) *Proxy { endpointRegistry: p.EndpointRegistry, } } - // TODO(sszuecs): expose an option to start it - fr := trace.NewFlightRecorder() - //fr.SetPeriod(d) - //fr.SetSize(bytes int) - err := fr.Start() - if err != nil { - println("Failed to start FlightRecorder:", err.Error()) + + var frURL *url.URL + if p.FlightRecorder != nil { + var err error + frURL, err = url.Parse(p.FlightRecorderTargetURL) + if err != nil { + p.FlightRecorder.Stop() + p.FlightRecorder = nil + } } return &Proxy{ @@ -817,53 +828,82 @@ func WithParams(p Params) *Proxy { rnd: rand.New(loadbalancer.NewLockedSource()), endpointRegistry: p.EndpointRegistry, }, - heathlyEndpoints: healthyEndpointsChooser, - roundTripper: p.CustomHttpRoundTripperWrap(tr), - priorityRoutes: p.PriorityRoutes, - flags: p.Flags, - metrics: m, - quit: quit, - flushInterval: p.FlushInterval, - experimentalUpgrade: p.ExperimentalUpgrade, - experimentalUpgradeAudit: p.ExperimentalUpgradeAudit, - maxLoops: p.MaxLoopbacks, - breakers: p.CircuitBreakers, - limiters: p.RateLimiters, - log: &logging.DefaultLog{}, - defaultHTTPStatus: defaultHTTPStatus, - tracing: newProxyTracing(p.OpenTracing), - accessLogDisabled: p.AccessLogDisabled, - upgradeAuditLogOut: os.Stdout, - upgradeAuditLogErr: os.Stderr, - clientTLS: tr.TLSClientConfig, - hostname: hostname, - onPanicSometimes: rate.Sometimes{First: 3, Interval: 1 * time.Minute}, - flightRecorder: fr, - traceOnce: sync.Once{}, - tooLong: 250 * time.Millisecond, + heathlyEndpoints: healthyEndpointsChooser, + roundTripper: p.CustomHttpRoundTripperWrap(tr), + priorityRoutes: p.PriorityRoutes, + flags: p.Flags, + metrics: m, + quit: quit, + flushInterval: p.FlushInterval, + experimentalUpgrade: p.ExperimentalUpgrade, + experimentalUpgradeAudit: p.ExperimentalUpgradeAudit, + maxLoops: p.MaxLoopbacks, + breakers: p.CircuitBreakers, + limiters: p.RateLimiters, + log: &logging.DefaultLog{}, + defaultHTTPStatus: defaultHTTPStatus, + tracing: newProxyTracing(p.OpenTracing), + accessLogDisabled: p.AccessLogDisabled, + upgradeAuditLogOut: os.Stdout, + upgradeAuditLogErr: os.Stderr, + clientTLS: tr.TLSClientConfig, + hostname: hostname, + onPanicSometimes: rate.Sometimes{First: 3, Interval: 1 * time.Minute}, + flightRecorder: p.FlightRecorder, + flightRecorderURL: frURL, + flightRecorderProxyTookTooLong: p.FlightRecorderProxyTookTooLong, } } func (p *Proxy) writeTraceIfTooSlow(ctx *context) { - p.log.Infof("write trace if too slow: %s > %s", time.Since(ctx.startServe), p.tooLong) - if time.Since(ctx.startServe) > p.tooLong { - p.log.Info("too slow") - // Do it only once for simplicitly, but you can take more than one. - p.traceOnce.Do(func() { - p.log.Info("write trace because we were too slow") - // Grab the snapshot. - var b bytes.Buffer - _, err := p.flightRecorder.WriteTo(&b) - if err != nil { - p.log.Errorf("Failed to write flightrecorder data: %v", err) + if p.flightRecorder == nil || p.flightRecorderURL == nil { + return + } + + d := p.flightRecorderProxyTookTooLong + if e, ok := ctx.StateBag()[filters.TraceName]; ok { + d = e.(time.Duration) + } + if d < 1*time.Microsecond { + return + } + + p.log.Infof("write trace if too slow: %s > %s", time.Since(ctx.startServe), d) + if time.Since(ctx.startServe) > d { + var b bytes.Buffer + _, err := p.flightRecorder.WriteTo(&b) + if err != nil { + p.log.Errorf("Failed to write flightrecorder data: %v", err) + return + } + + switch p.flightRecorderURL.Scheme { + case "file": + if err := os.WriteFile(p.flightRecorderURL.Path, b.Bytes(), 0o644); err != nil { + p.log.Errorf("Failed to write file trace.out: %v", err) return + } else { + p.log.Infof("FlightRecorder wrote %d bytes to trace file %q", b.Len(), p.flightRecorderURL.Path) } - // Write it to a file. - if err := os.WriteFile("trace.out", b.Bytes(), 0o755); err != nil { - p.log.Errorf("Failed to write trace.out: %v", err) - return + case "http", "https": + req, err := http.NewRequest("PUT", p.flightRecorderURL.String(), &b) + if err != nil { + p.log.Errorf("Failed to create request to %q to send a trace: %v", p.flightRecorderURL.String(), err) } - }) + + rsp, err := http.DefaultClient.Do(req) + if err != nil { + p.log.Errorf("Failed to write trace to %q: %v", p.flightRecorderURL.String(), err) + } + switch rsp.StatusCode { + case 200, 201, 204: + p.log.Infof("Successful send of a trace to %q", p.flightRecorderURL.String()) + default: + p.log.Errorf("Failed to get successful response from %s: (%d) %s", p.flightRecorderURL.String(), rsp.StatusCode, rsp.Status) + } + default: + p.log.Errorf("Failed to write trace, unknown FlightRecorderURL %q", p.flightRecorderURL.Scheme) + } } } diff --git a/skipper.go b/skipper.go index 445a30f9ef..92e32ccfa1 100644 --- a/skipper.go +++ b/skipper.go @@ -22,6 +22,7 @@ import ( ot "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "golang.org/x/exp/trace" "github.com/zalando/skipper/circuit" "github.com/zalando/skipper/dataclients/kubernetes" @@ -458,6 +459,22 @@ type Options struct { // MemProfileRate calls runtime.SetMemProfileRate(MemProfileRate) if non zero value, deactivate with <0 MemProfileRate int + // EnableFlightRecorder enables trace.FlightRecorder https://pkg.go.dev/golang.org/x/exp/trace#FlightRecorder to support Go tool trace. + EnableFlightRecorder bool + + // FlightRecorderSizeBytes set size of the FlightRecorder https://pkg.go.dev/golang.org/x/exp/trace#FlightRecorder.SetSize + FlightRecorderSize int + + // FlightRecorderPeriod set period of the FlightRecorder https://pkg.go.dev/golang.org/x/exp/trace#FlightRecorder.SetPeriod + FlightRecorderPeriod time.Duration + // FlightRecorderProxyTookTooLong sets the global + // time.Duration that is the threshold of writing the trace + FlightRecorderProxyTookTooLong time.Duration + + // FlightRecorderTargetURL is the target to write the trace + // to. Supported targets are http URL and file URL. + FlightRecorderTargetURL string + // Flag that enables reporting of the Go garbage collector statistics exported in debug.GCStats EnableDebugGcMetrics bool @@ -1990,33 +2007,54 @@ func run(o Options, sig chan os.Signal, idleConnsCH chan struct{}) error { routing := routing.New(ro) defer routing.Close() + var fr *trace.FlightRecorder + if o.EnableFlightRecorder { + fr = trace.NewFlightRecorder() + if o.FlightRecorderPeriod != 0 { + fr.SetPeriod(o.FlightRecorderPeriod) + } + if o.FlightRecorderSize != 0 { + fr.SetSize(o.FlightRecorderSize) + } + err := fr.Start() + if err != nil { + log.Errorf("Failed to start FlightRecorder: %v", err) + fr.Stop() + fr = nil + } + } + log.Infof("FlightRecorder enabled %v: %v", o.EnableFlightRecorder, fr) + proxyFlags := proxy.Flags(o.ProxyOptions) | o.ProxyFlags proxyParams := proxy.Params{ - Routing: routing, - EndpointRegistry: endpointRegistry, - EnablePassiveHealthCheck: passiveHealthCheckEnabled, - PassiveHealthCheck: passiveHealthCheck, - Flags: proxyFlags, - PriorityRoutes: o.PriorityRoutes, - IdleConnectionsPerHost: o.IdleConnectionsPerHost, - CloseIdleConnsPeriod: o.CloseIdleConnsPeriod, - FlushInterval: o.BackendFlushInterval, - ExperimentalUpgrade: o.ExperimentalUpgrade, - ExperimentalUpgradeAudit: o.ExperimentalUpgradeAudit, - MaxLoopbacks: o.MaxLoopbacks, - DefaultHTTPStatus: o.DefaultHTTPStatus, - Timeout: o.TimeoutBackend, - ResponseHeaderTimeout: o.ResponseHeaderTimeoutBackend, - ExpectContinueTimeout: o.ExpectContinueTimeoutBackend, - KeepAlive: o.KeepAliveBackend, - DualStack: o.DualStackBackend, - TLSHandshakeTimeout: o.TLSHandshakeTimeoutBackend, - MaxIdleConns: o.MaxIdleConnsBackend, - DisableHTTPKeepalives: o.DisableHTTPKeepalives, - AccessLogDisabled: o.AccessLogDisabled, - ClientTLS: o.ClientTLS, - CustomHttpRoundTripperWrap: o.CustomHttpRoundTripperWrap, - RateLimiters: ratelimitRegistry, + Routing: routing, + EndpointRegistry: endpointRegistry, + EnablePassiveHealthCheck: passiveHealthCheckEnabled, + PassiveHealthCheck: passiveHealthCheck, + Flags: proxyFlags, + PriorityRoutes: o.PriorityRoutes, + IdleConnectionsPerHost: o.IdleConnectionsPerHost, + CloseIdleConnsPeriod: o.CloseIdleConnsPeriod, + FlushInterval: o.BackendFlushInterval, + ExperimentalUpgrade: o.ExperimentalUpgrade, + ExperimentalUpgradeAudit: o.ExperimentalUpgradeAudit, + MaxLoopbacks: o.MaxLoopbacks, + DefaultHTTPStatus: o.DefaultHTTPStatus, + Timeout: o.TimeoutBackend, + ResponseHeaderTimeout: o.ResponseHeaderTimeoutBackend, + ExpectContinueTimeout: o.ExpectContinueTimeoutBackend, + KeepAlive: o.KeepAliveBackend, + DualStack: o.DualStackBackend, + TLSHandshakeTimeout: o.TLSHandshakeTimeoutBackend, + MaxIdleConns: o.MaxIdleConnsBackend, + DisableHTTPKeepalives: o.DisableHTTPKeepalives, + AccessLogDisabled: o.AccessLogDisabled, + ClientTLS: o.ClientTLS, + CustomHttpRoundTripperWrap: o.CustomHttpRoundTripperWrap, + RateLimiters: ratelimitRegistry, + FlightRecorder: fr, + FlightRecorderProxyTookTooLong: o.FlightRecorderProxyTookTooLong, + FlightRecorderTargetURL: o.FlightRecorderTargetURL, } if o.EnableBreakers || len(o.BreakerSettings) > 0 {