feat: use shutdown impl from duty

cmd/offline.go

@@ -3,11 +3,11 @@ package cmd
 import (
 	"os"
 
-	"github.com/flanksource/canary-checker/pkg/runner"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty"
 	"github.com/flanksource/duty/api"
 	"github.com/flanksource/duty/postgrest"
+	"github.com/flanksource/duty/shutdown"
 	"github.com/spf13/cobra"
 )
 
@@ -28,12 +28,12 @@ var GoOffline = &cobra.Command{
 
 		api.DefaultConfig.ConnectionString = "embedded://" + databaseDir
 		_, closer, err := duty.Start("embedded-temp")
-		runner.AddShutdownHook(closer)
+		shutdown.AddHook(closer)
 		if err != nil {
 			logger.Fatalf("Failed to run in embedded mode: %+v", err)
 		}
 
 		// Intentionally exit with code 0 for Docker
-		runner.ShutdownAndExit(0, "Finished downloading dependencies")
+		shutdown.ShutdownAndExit(0, "Finished downloading dependencies")
 	},
 }

cmd/operator.go

@@ -1,7 +1,8 @@
 package cmd
 
 import (
-	"os"
+	"errors"
+	"fmt"
 	"time"
 
 	apicontext "github.com/flanksource/canary-checker/api/context"
@@ -18,6 +19,7 @@ import (
 	"github.com/flanksource/canary-checker/pkg/labels"
 	"github.com/flanksource/commons/logger"
 	dutyKubernetes "github.com/flanksource/duty/kubernetes"
+	"github.com/flanksource/duty/shutdown"
 	"github.com/spf13/cobra"
 	"go.opentelemetry.io/otel"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -34,7 +36,13 @@ var (
 	Operator             = &cobra.Command{
 		Use:   "operator",
 		Short: "Start the kubernetes operator",
-		Run:   run,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := run(cmd, args); err != nil {
+				shutdown.ShutdownAndExit(1, err.Error())
+			} else {
+				shutdown.ShutdownAndExit(0, err.Error())
+			}
+		},
 	}
 )
 
@@ -48,9 +56,7 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 }
 
-func run(cmd *cobra.Command, args []string) {
-	defer runner.Shutdown()
-
+func run(cmd *cobra.Command, args []string) error {
 	logger := logger.GetLogger("operator")
 	logger.SetLogLevel(k8sLogLevel)
 
@@ -61,15 +67,15 @@ func run(cmd *cobra.Command, args []string) {
 
 	ctx, err := InitContext()
 	if err != nil {
-		runner.ShutdownAndExit(1, err.Error())
+		return err
 	}
 
 	if ctx.DB() == nil {
-		runner.ShutdownAndExit(1, "operator requires a db connection")
+		return errors.New("operator requires a db connection")
 	}
 
 	if ctx.KubernetesRestConfig() == nil {
-		runner.ShutdownAndExit(1, "operator requires a kubernetes connection")
+		return errors.New("operator requires a kubernetes connection")
 	}
 
 	ctx.WithTracer(otel.GetTracerProvider().Tracer("canary-checker"))
@@ -114,7 +120,7 @@ func run(cmd *cobra.Command, args []string) {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), managerOpt)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
-		os.Exit(1)
+		return fmt.Errorf("unable to start manager: %v", err)
 	}
 
 	if runner.RunnerName == "" {
@@ -144,7 +150,7 @@ func run(cmd *cobra.Command, args []string) {
 
 	if err = canaryReconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Canary")
-		os.Exit(1)
+		return fmt.Errorf("unable to create canary controller: %v", err)
 	}
 
 	// Instantiate the canary status channel so the canary job can send updates on it.
@@ -156,10 +162,14 @@ func run(cmd *cobra.Command, args []string) {
 
 	if err = systemReconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "System")
-		os.Exit(1)
+		return fmt.Errorf("unable to create topology controller: %v", err)
 	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
+		return fmt.Errorf("problem running controller manager: %v", err)
 	}
+
+	return nil
 }

cmd/root.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"os/signal"
 	"time"
 
 	v1 "github.com/flanksource/canary-checker/api/v1"
@@ -19,6 +18,7 @@ import (
 	"github.com/flanksource/duty/connection"
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/query"
+	"github.com/flanksource/duty/shutdown"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"go.opentelemetry.io/otel"
@@ -27,10 +27,9 @@ import (
 func InitContext() (context.Context, error) {
 	ctx, closer, err := duty.Start("canary-checker", duty.SkipChangelogMigration, duty.SkipMigrationByDefaultMode)
 	if err != nil {
-		logger.Fatalf("Failed to initialize db: %v", err.Error())
-		runner.ShutdownAndExit(1, err.Error())
+		return ctx, fmt.Errorf("Failed to initialize db: %v", err.Error())
 	}
-	runner.AddShutdownHook(closer)
+	shutdown.AddHook(closer)
 
 	if err := properties.LoadFile(propertiesFile); err != nil {
 		return ctx, fmt.Errorf("failed to load properties: %v", err)
@@ -43,11 +42,9 @@ func InitContext() (context.Context, error) {
 
 var Root = &cobra.Command{
 	Use: "canary-checker",
-	PersistentPostRun: func(cmd *cobra.Command, args []string) {
-		runner.Shutdown()
-	},
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		logger.UseSlog()
+		shutdown.WaitForSignal()
 
 		canary.LogFail = logFail || logger.IsLevelEnabled(3)
 		canary.LogPass = logPass || logger.IsLevelEnabled(4)
@@ -64,7 +61,7 @@ var Root = &cobra.Command{
 		if otelcollectorURL != "" {
 			logger.Infof("Sending traces to %s", otelcollectorURL)
 
-			runner.AddShutdownHook(telemetry.InitTracer(otelServiceName, otelcollectorURL, true))
+			shutdown.AddHook(telemetry.InitTracer(otelServiceName, otelcollectorURL, true))
 		}
 		if prometheus.PrometheusURL != "" {
 			logger.Infof("Setting default prometheus: %s", prometheus.PrometheusURL)
@@ -73,15 +70,6 @@ var Root = &cobra.Command{
 				connection.HTTPConnection{URL: prometheus.PrometheusURL},
 			)
 		}
-
-		go func() {
-			quit := make(chan os.Signal, 1)
-			signal.Notify(quit, os.Interrupt)
-			<-quit
-			logger.Infof("Caught Ctrl+C")
-			// call shutdown hooks explicitly, post-run cleanup hooks will be a no-op
-			runner.Shutdown()
-		}()
 	},
 }
 

cmd/run.go

@@ -10,9 +10,9 @@ import (
 
 	"github.com/flanksource/commons/timer"
 	"github.com/flanksource/duty"
+	"github.com/flanksource/duty/shutdown"
 
 	"github.com/flanksource/canary-checker/cmd/output"
-	"github.com/flanksource/canary-checker/pkg/runner"
 	"github.com/spf13/cobra"
 
 	apicontext "github.com/flanksource/canary-checker/api/context"
@@ -37,7 +37,7 @@ var Run = &cobra.Command{
 		if err != nil {
 			logger.Fatalf("Failed to initialize db: %v", err.Error())
 		}
-		runner.AddShutdownHook(closer)
+		shutdown.AddHook(closer)
 
 		apicontext.DefaultContext = ctx
 

cmd/serve.go

@@ -24,9 +24,9 @@ import (
 	"github.com/flanksource/canary-checker/pkg/cache"
 	"github.com/flanksource/commons/logger"
 	dutyApi "github.com/flanksource/duty/api"
-	dutyContext "github.com/flanksource/duty/context"
 	dutyEcho "github.com/flanksource/duty/echo"
 	"github.com/flanksource/duty/postgrest"
+	"github.com/flanksource/duty/shutdown"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
@@ -42,28 +42,31 @@ var Serve = &cobra.Command{
 	Use:   "serve config.yaml",
 	Short: "Start a server to execute checks",
 	Run: func(cmd *cobra.Command, configFiles []string) {
-		defer runner.Shutdown()
-		canaryJobs.StartScanCanaryConfigs(setup(), dataFile, configFiles)
+		if err := setupDefaultContext(); err != nil {
+			shutdown.ShutdownAndExit(1, err.Error())
+		}
+
+		canaryJobs.StartScanCanaryConfigs(apicontext.DefaultContext, dataFile, configFiles)
 		if executor {
 			jobs.Start()
 		}
+
 		serve()
 	},
 }
 
-func setup() dutyContext.Context {
+func setupDefaultContext() error {
 	var err error
 
 	if apicontext.DefaultContext, err = InitContext(); err != nil {
-		runner.ShutdownAndExit(1, err.Error())
-		return apicontext.DefaultContext
+		return err
 	}
 
 	apicontext.DefaultContext = apicontext.DefaultContext.WithNamespace(runner.WatchNamespace)
 
 	cache.PostgresCache = cache.NewPostgresCache(apicontext.DefaultContext)
 
-	return apicontext.DefaultContext
+	return nil
 }
 
 func postgrestResponseModifier(r *http.Response) error {
@@ -105,7 +108,7 @@ func serve() {
 		logger.Tracef("No PostgREST endpoint configured, skipping proxy")
 	}
 
-	runner.AddShutdownHook(func() {
+	shutdown.AddHook(func() {
 		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 		defer cancel()
 

cmd/topology.go

@@ -9,14 +9,14 @@ import (
 
 	"github.com/flanksource/commons/timer"
 	"github.com/flanksource/duty"
+	"github.com/flanksource/duty/shutdown"
 	"github.com/google/uuid"
 
 	"github.com/spf13/cobra"
 
 	apicontext "github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/pkg"
 	"github.com/flanksource/canary-checker/pkg/db"
-	"github.com/flanksource/canary-checker/pkg/runner"
 	configSync "github.com/flanksource/canary-checker/pkg/sync"
 	"github.com/flanksource/canary-checker/pkg/topology"
 	"github.com/flanksource/commons/logger"
@@ -72,7 +72,7 @@ var RunTopology = &cobra.Command{
 			os.Exit(1)
 		}
 
-		defer runner.Shutdown()
+		defer shutdown.Shutdown()
 		var err error
 		apicontext.DefaultContext, _, err = duty.Start("canary-checker", duty.ClientOnly, duty.SkipMigrationByDefaultMode)
 		if err != nil {

go.mod

@@ -119,11 +119,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect
 	github.com/aws/smithy-go v1.20.4 // indirect
-	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
-	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
@@ -200,7 +198,6 @@ require (
 	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/invopop/jsonschema v0.12.0 // indirect
 	github.com/itchyny/gojq v0.12.16 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect
@@ -275,14 +272,10 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
-	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
-	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect

go.sum

@@ -748,8 +748,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3Co
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
 github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
-github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
-github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -761,8 +759,6 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN
 github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
@@ -1176,8 +1172,6 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
-github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
 github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
 github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
 github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
@@ -1580,8 +1574,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
-github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -1590,12 +1582,6 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
-github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=