From dfaa2d221a8121da2016f9ff33fbc35acd97df59 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Sat, 30 Nov 2024 11:36:31 +0530 Subject: [PATCH 1/9] wip: tls support Signed-off-by: Mayank Shah --- api/everest.go | 8 +++++++- cmd/config/config.go | 8 +++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/api/everest.go b/api/everest.go index 7724a748c..ddc48fa48 100644 --- a/api/everest.go +++ b/api/everest.go @@ -25,6 +25,7 @@ import ( "fmt" "io/fs" "net/http" + "path" "slices" "github.com/casbin/casbin/v2" @@ -276,7 +277,12 @@ func newSkipperFunc() (echomiddleware.Skipper, error) { // Start starts everest server. func (e *EverestServer) Start() error { - return e.echo.Start(fmt.Sprintf("0.0.0.0:%d", e.config.HTTPPort)) + if e.config.TLSCertsPath != "" { + tlsKeyPath := path.Join(e.config.TLSCertsPath, "tls.key") + tlsCertPath := path.Join(e.config.TLSCertsPath, "tls.crt") + return e.echo.StartTLS(fmt.Sprintf("0.0.0.0:%d", e.config.ListenPort), tlsCertPath, tlsKeyPath) + } + return e.echo.Start(fmt.Sprintf("0.0.0.0:%d", e.config.ListenPort)) } // Shutdown gracefully stops the Everest server. diff --git a/cmd/config/config.go b/cmd/config/config.go index 120d902fb..ae0fbdafe 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -39,9 +39,9 @@ var ( // EverestConfig stores the configuration for the application. type EverestConfig struct { - DSN string `default:"postgres://admin:pwd@127.0.0.1:5432/postgres?sslmode=disable" envconfig:"DSN"` - HTTPPort int `default:"8080" envconfig:"HTTP_PORT"` - Verbose bool `default:"false" envconfig:"VERBOSE"` + DSN string `default:"postgres://admin:pwd@127.0.0.1:5432/postgres?sslmode=disable" envconfig:"DSN"` + ListenPort int `default:"8080" envconfig:"PORT"` + Verbose bool `default:"false" envconfig:"VERBOSE"` // TelemetryURL Everest telemetry endpoint. TelemetryURL string `envconfig:"TELEMETRY_URL"` // TelemetryInterval Everest telemetry sending frequency. @@ -54,6 +54,8 @@ type EverestConfig struct { CreateSessionRateLimit int `default:"1" envconfig:"CREATE_SESSION_RATE_LIMIT"` // VersionServiceURL contains the URL of the version service. VersionServiceURL string `default:"https://check.percona.com" envconfig:"VERSION_SERVICE_URL"` + // TLSCertsPath contains the path to the directory with the TLS certificates. + TLSCertsPath string `envconfig:"TLS_CERTS_PATH"` } // ParseConfig parses env vars and fills EverestConfig. From 75d3aca85018955f1b863be43536f441b46cf0d2 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Thu, 9 Jan 2025 13:45:30 +0530 Subject: [PATCH 2/9] update comment Signed-off-by: Mayank Shah --- cmd/config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/config/config.go b/cmd/config/config.go index ae0fbdafe..4cd0a87b2 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -55,6 +55,7 @@ type EverestConfig struct { // VersionServiceURL contains the URL of the version service. VersionServiceURL string `default:"https://check.percona.com" envconfig:"VERSION_SERVICE_URL"` // TLSCertsPath contains the path to the directory with the TLS certificates. + // Setting this will enable HTTPS on ListenPort. TLSCertsPath string `envconfig:"TLS_CERTS_PATH"` } From 8dcec89631da42cb015053ea55c26dfa440c5aa2 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Fri, 10 Jan 2025 01:00:00 +0530 Subject: [PATCH 3/9] add ability to refresh tls certs Signed-off-by: Mayank Shah --- cmd/main.go | 2 +- internal/server/everest.go | 33 +++++++++++++--- pkg/certwatcher/watcher.go | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 pkg/certwatcher/watcher.go diff --git a/cmd/main.go b/cmd/main.go index 402a2dbe7..8897fd720 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -64,7 +64,7 @@ func main() { } go func() { - err := server.Start() + err := server.Start(tCtx) if err != nil && !errors.Is(err, http.ErrServerClosed) { l.Fatal(err) } diff --git a/internal/server/everest.go b/internal/server/everest.go index 0d8ecaed6..4651ad54f 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -18,6 +18,7 @@ package server import ( "context" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -43,6 +44,7 @@ import ( k8shandler "github.com/percona/everest/internal/server/handlers/k8s" rbachandler "github.com/percona/everest/internal/server/handlers/rbac" valhandler "github.com/percona/everest/internal/server/handlers/validation" + "github.com/percona/everest/pkg/certwatcher" "github.com/percona/everest/pkg/common" "github.com/percona/everest/pkg/kubernetes" "github.com/percona/everest/pkg/oidc" @@ -295,13 +297,34 @@ func newSkipperFunc() (echomiddleware.Skipper, error) { } // Start starts everest server. -func (e *EverestServer) Start() error { +func (e *EverestServer) Start(ctx context.Context) error { + addr := fmt.Sprintf("0.0.0.0:%d", e.config.ListenPort) if e.config.TLSCertsPath != "" { - tlsKeyPath := path.Join(e.config.TLSCertsPath, "tls.key") - tlsCertPath := path.Join(e.config.TLSCertsPath, "tls.crt") - return e.echo.StartTLS(fmt.Sprintf("0.0.0.0:%d", e.config.ListenPort), tlsCertPath, tlsKeyPath) + return e.startHTTPS(ctx, addr) } - return e.echo.Start(fmt.Sprintf("0.0.0.0:%d", e.config.ListenPort)) + return e.echo.Start(addr) +} + +func (e *EverestServer) startHTTPS(ctx context.Context, addr string) error { + tlsKeyPath := path.Join(e.config.TLSCertsPath, "tls.key") + tlsCertPath := path.Join(e.config.TLSCertsPath, "tls.crt") + + watcher, err := certwatcher.New(e.l, tlsCertPath, tlsKeyPath) + if err != nil { + return fmt.Errorf("failed to create cert watcher: %w", err) + } + if err := watcher.Start(ctx); err != nil { + return fmt.Errorf("failed to start cert watcher: %w", err) + } + + e.echo.TLSServer = &http.Server{ + Addr: addr, + TLSConfig: &tls.Config{ + // server periodically calls GetCertificate and reloads the certificate. + GetCertificate: watcher.GetCertificate, + }, + } + return e.echo.StartServer(e.echo.TLSServer) } // Shutdown gracefully stops the Everest server. diff --git a/pkg/certwatcher/watcher.go b/pkg/certwatcher/watcher.go new file mode 100644 index 000000000..9fc2ed4e6 --- /dev/null +++ b/pkg/certwatcher/watcher.go @@ -0,0 +1,79 @@ +package certwatcher + +import ( + "context" + "crypto/tls" + "sync" + + "github.com/fsnotify/fsnotify" + "go.uber.org/zap" +) + +type certWatcher struct { + cert *tls.Certificate + certFile, keyFile string + log *zap.SugaredLogger + + mutex sync.RWMutex +} + +// GetCertificate returns the certificate. +func (w *certWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + w.mutex.RLock() + defer w.mutex.RUnlock() + return w.cert, nil +} + +// New returns a new cert watcher. +func New(log *zap.SugaredLogger, certFile, keyFile string) (*certWatcher, error) { + w := &certWatcher{ + certFile: certFile, + keyFile: keyFile, + log: log, + } + if err := w.loadCertificate(); err != nil { + return nil, err + } + return w, nil +} + +func (w *certWatcher) loadCertificate() error { + w.mutex.Lock() + defer w.mutex.Unlock() + + cert, err := tls.LoadX509KeyPair(w.certFile, w.keyFile) + if err != nil { + return err + } + w.cert = &cert + return nil +} + +// Start the certificate files watcher until the context is closed. +func (w *certWatcher) Start(ctx context.Context) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + watcher.Add(w.certFile) + watcher.Add(w.keyFile) + + w.log.Infow("Watching certificate files for changes", "certFile", w.certFile, "keyFile", w.keyFile) + + go func() { + for { + select { + case <-ctx.Done(): + if err := watcher.Close(); err != nil { + w.log.Errorw("Failed to close watcher", "error", err) + } + case <-watcher.Events: + w.log.Info("Certificate updated") + if err := w.loadCertificate(); err != nil { + w.log.Errorw("Failed to reload certificate", "error", err) + } + } + } + }() + return nil +} From d6cafb5ea28cd162de44e569bbac38a5f9580574 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Fri, 10 Jan 2025 10:31:49 +0530 Subject: [PATCH 4/9] update go mod Signed-off-by: Mayank Shah --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 81caa6fe4..46676f3bf 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff/v4 v4.3.0 github.com/fatih/color v1.18.0 + github.com/fsnotify/fsnotify v1.7.0 github.com/getkin/kin-openapi v0.128.0 github.com/go-logr/zapr v1.3.0 github.com/golang-jwt/jwt/v5 v5.2.1 @@ -103,7 +104,6 @@ require ( github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flosch/pongo2/v6 v6.0.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-errors/errors v1.5.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect From 02a166197d14ae0e5e4abecbb67f026125edba55 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Mon, 13 Jan 2025 15:33:45 +0530 Subject: [PATCH 5/9] add back HTTPPort, deprecate it Signed-off-by: Mayank Shah --- cmd/config/config.go | 8 +++++--- internal/server/everest.go | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/config/config.go b/cmd/config/config.go index 4cd0a87b2..448e441e0 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -39,9 +39,11 @@ var ( // EverestConfig stores the configuration for the application. type EverestConfig struct { - DSN string `default:"postgres://admin:pwd@127.0.0.1:5432/postgres?sslmode=disable" envconfig:"DSN"` - ListenPort int `default:"8080" envconfig:"PORT"` - Verbose bool `default:"false" envconfig:"VERBOSE"` + DSN string `default:"postgres://admin:pwd@127.0.0.1:5432/postgres?sslmode=disable" envconfig:"DSN"` + // DEPRECATED: Use ListenPort instead. + HTTPPort int `envconfig:"HTTP_PORT"` + ListenPort int `default:"8080" envconfig:"PORT"` + Verbose bool `default:"false" envconfig:"VERBOSE"` // TelemetryURL Everest telemetry endpoint. TelemetryURL string `envconfig:"TELEMETRY_URL"` // TelemetryInterval Everest telemetry sending frequency. diff --git a/internal/server/everest.go b/internal/server/everest.go index 4651ad54f..728f9f312 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -70,6 +70,11 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar return nil, errors.Join(err, errors.New("failed creating Kubernetes client")) } + if c.HTTPPort != 0 && c.ListenPort == 0 { + l.Warn("HTTP_PORT is deprecated, use PORT instead") + c.ListenPort = c.HTTPPort + } + echoServer := echo.New() echoServer.Use(echomiddleware.RateLimiter(echomiddleware.NewRateLimiterMemoryStore(rate.Limit(c.APIRequestsRateLimit)))) middleware, store := sessionRateLimiter(c.CreateSessionRateLimit) From 978332566c9eb02d2898535e7ca91840eb20abe6 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Mon, 13 Jan 2025 15:40:33 +0530 Subject: [PATCH 6/9] update condition Signed-off-by: Mayank Shah --- internal/server/everest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/everest.go b/internal/server/everest.go index 728f9f312..dfe56c363 100644 --- a/internal/server/everest.go +++ b/internal/server/everest.go @@ -70,7 +70,7 @@ func NewEverestServer(ctx context.Context, c *config.EverestConfig, l *zap.Sugar return nil, errors.Join(err, errors.New("failed creating Kubernetes client")) } - if c.HTTPPort != 0 && c.ListenPort == 0 { + if c.HTTPPort != 0 { l.Warn("HTTP_PORT is deprecated, use PORT instead") c.ListenPort = c.HTTPPort } From 327a819c500f33481b8ed12140a5c6c29c4d03ce Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Fri, 24 Jan 2025 11:27:30 +0530 Subject: [PATCH 7/9] add licence header Signed-off-by: Mayank Shah --- pkg/certwatcher/watcher.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/certwatcher/watcher.go b/pkg/certwatcher/watcher.go index 9fc2ed4e6..c99bb520d 100644 --- a/pkg/certwatcher/watcher.go +++ b/pkg/certwatcher/watcher.go @@ -1,3 +1,18 @@ +// everest +// Copyright (C) 2023 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package certwatcher import ( From 5607d4054e7f314f300cf52fa7ed5699cc3e66c3 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Fri, 24 Jan 2025 11:40:58 +0530 Subject: [PATCH 8/9] return copy of certs Signed-off-by: Mayank Shah --- pkg/certwatcher/watcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/certwatcher/watcher.go b/pkg/certwatcher/watcher.go index c99bb520d..17f9f1981 100644 --- a/pkg/certwatcher/watcher.go +++ b/pkg/certwatcher/watcher.go @@ -36,7 +36,9 @@ type certWatcher struct { func (w *certWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { w.mutex.RLock() defer w.mutex.RUnlock() - return w.cert, nil + + certCopy := *w.cert + return &certCopy, nil } // New returns a new cert watcher. From 506d8ebb975324ce649d0611753b29f1deb82901 Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Fri, 24 Jan 2025 11:46:53 +0530 Subject: [PATCH 9/9] add filepath cleaning Signed-off-by: Mayank Shah --- cmd/config/config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/config/config.go b/cmd/config/config.go index 448e441e0..3fd8cf37d 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -18,6 +18,7 @@ package config import ( "crypto/aes" + "path/filepath" "github.com/kelseyhightower/envconfig" ) @@ -76,5 +77,9 @@ func ParseConfig() (*EverestConfig, error) { c.TelemetryInterval = TelemetryInterval } + if c.TLSCertsPath != "" { + c.TLSCertsPath = filepath.Clean(c.TLSCertsPath) + } + return c, nil }