Merge pull request #665 from pyrra-dev/release-0.5-signalhandling

`release-0.5`: handle graceful termination

kubernetes.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"syscall"
+	"time"
 
 	"github.com/bufbuild/connect-go"
 	"github.com/go-kit/log"
@@ -81,7 +83,12 @@ func cmdKubernetes(logger log.Logger, metricsAddr string, configMapMode, generic
 	}
 	// +kubebuilder:scaffold:builder
 
-	var gr run.Group
+	var (
+		gr  run.Group
+		ctx = context.Background()
+	)
+	gr.Add(run.SignalHandler(ctx, os.Interrupt, syscall.SIGTERM))
+
 	{
 		router := http.NewServeMux()
 		router.Handle(objectivesv1alpha1connect.NewObjectiveBackendServiceHandler(&KubernetesObjectiveServer{
@@ -96,18 +103,19 @@ func cmdKubernetes(logger log.Logger, metricsAddr string, configMapMode, generic
 		gr.Add(func() error {
 			return server.ListenAndServe()
 		}, func(err error) {
-			_ = server.Shutdown(context.Background())
+			shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			defer cancel()
+			_ = server.Shutdown(shutdownCtx)
 		})
 	}
-	{
-		gr.Add(func() error {
-			setupLog.Info("starting manager")
-			return mgr.Start(ctrl.SetupSignalHandler())
-		}, func(err error) {})
-	}
 
 	if err := gr.Run(); err != nil {
+		if _, ok := err.(run.SignalError); ok {
+			setupLog.Info("terminated", "reason", err)
+			return 0
+		}
 		setupLog.Error(err, "failed to run groups")
+		return 2
 	}
 	return 0
 }

main.go

@@ -15,6 +15,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/alecthomas/kong"
@@ -24,6 +25,7 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/api"
 	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/client_golang/prometheus"
@@ -259,7 +261,33 @@ func cmdAPI(logger log.Logger, reg *prometheus.Registry, promClient api.Client,
 		})
 	}
 
-	if err := http.ListenAndServe(":9099", h2c.NewHandler(r, &http2.Server{})); err != nil {
+	var (
+		gr  run.Group
+		ctx = context.Background()
+	)
+	gr.Add(run.SignalHandler(ctx, os.Interrupt, syscall.SIGTERM))
+
+	{
+		httpServer := &http.Server{
+			Addr:    ":9099",
+			Handler: h2c.NewHandler(r, &http2.Server{}),
+		}
+		gr.Add(
+			func() error {
+				return httpServer.ListenAndServe()
+			},
+			func(error) {
+				shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+				defer cancel()
+				_ = httpServer.Shutdown(shutdownCtx)
+			},
+		)
+	}
+	if err := gr.Run(); err != nil {
+		if _, ok := err.(run.SignalError); ok {
+			level.Info(logger).Log("msg", "terminated HTTP server", "reason", err)
+			return 0
+		}
 		level.Error(logger).Log("msg", "failed to run HTTP server", "err", err)
 		return 2
 	}