diff --git a/cmd/weaver-kube/deploy.go b/cmd/weaver-kube/deploy.go index b3fe639..ec0bada 100644 --- a/cmd/weaver-kube/deploy.go +++ b/cmd/weaver-kube/deploy.go @@ -116,6 +116,24 @@ Container Image Names: You can also specify any combination of the various options or none. [1] https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + + e) Configure probes [1]. The kube deployer allows you to configure readiness + and liveness probes. For each probe, you can configure: + - how often to perform the probe "period_secs" + - how long to wait for a probe to respond before declaring a timeout "timeout_secs" + - minimum consecutive successes for the probe to be successful "success_threshold" + - minimum consecutive failures for the probe to be considered failed "failure_threshold" + - a probe handler that describes the probe behavior. You can use a TCP, + HTTP or a custom commands probe handler. + + E.g., + [kube.readiness_probe] + period_secs = 2 + [kube.readiness_probe.http] + path = "/health" + port = 8081 + + [1] https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ `, Flags: flags, Fn: func(ctx context.Context, args []string) error { diff --git a/internal/impl/config.go b/internal/impl/config.go index 57c1474..a2cb222 100644 --- a/internal/impl/config.go +++ b/internal/impl/config.go @@ -104,6 +104,10 @@ type kubeConfig struct { // // [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#example-MustParse. Resources resourceRequirements + + // Options for probes to check the readiness/liveness of the pods. + LivenessProbeOpts *probeOptions `toml:"liveness_probe"` + ReadinessProbeOpts *probeOptions `toml:"readiness_probe"` } // listenerConfig stores configuration options for a listener. @@ -134,3 +138,56 @@ type resourceRequirements struct { // Describes the maximum amount of memory allowed to run the pod. LimitsMem string `toml:"limits_mem"` } + +// probeOptions stores the probes [1] configuration for the pods. These options +// mirror the Kubernetes probe options available in [2]. +// +// [1] https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +// [2] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2277 +// +// TODO(rgrandl): There are a few more knobs available in the kubernetes probe +// definition. We can enable more knobs if really needed. +type probeOptions struct { + // How often to perform the probe. + PeriodSecs int32 `toml:"period_secs"` + // Number of seconds after which the probe times out. + TimeoutSecs int32 `toml:"timeout_secs"` + // Minimum consecutive successes for the probe to be considered successful after having failed. + SuccessThreshold int32 `toml:"success_threshold"` + // Minimum consecutive failures for the probe to be considered failed after having succeeded. + FailureThreshold int32 `toml:"failure_threshold"` + + // Probe behavior. Note that only one of the following should be set by the user. + + // The probe action is taken by executing commands. + Exec *execAction + // The probe action is taken by executing HTTP GET requests. + Http *httpAction + // The probe action is taken by executing TCP requests. + Tcp *tcpAction +} + +// execAction describes the probe action when using a list of commands. It mirrors +// Kubernetes ExecAction [1]. +// +// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2265 +type execAction struct { + Cmd []string // List of commands to execute inside the container. +} + +// httpAction describes the probe action when using HTTP. It mirrors Kubernetes +// HTTPGetAction [1]. +// +// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2208 +type httpAction struct { + Path string // Path to access on the HTTP server. + Port int32 // Port number to access on the container. +} + +// tcpAction describes the probe action when using TCP. It mirrors Kubernetes +// TCPSocketAction [1]. +// +// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2241 +type tcpAction struct { + Port int32 // Port number to access on the container. +} diff --git a/internal/impl/deploy.go b/internal/impl/deploy.go index 47cf3ba..5e97549 100644 --- a/internal/impl/deploy.go +++ b/internal/impl/deploy.go @@ -64,6 +64,15 @@ func Deploy(ctx context.Context, configFilename string) error { if config.ServiceAccount == "" { config.ServiceAccount = "default" } + + // Validate the probe options. + if err := checkProbeOptions(config.LivenessProbeOpts); err != nil { + return fmt.Errorf("invalid liveness probe spec: %w", err) + } + if err := checkProbeOptions(config.ReadinessProbeOpts); err != nil { + return fmt.Errorf("invalid readiness probe spec: %w", err) + } + binListeners, err := bin.ReadListeners(app.Binary) if err != nil { return fmt.Errorf("cannot read listeners from binary %s: %w", app.Binary, err) @@ -138,3 +147,36 @@ func checkVersionCompatibility(appBinary string) error { } return nil } + +// checkProbeOptions validates the configuration options for the probes. +func checkProbeOptions(opts *probeOptions) error { + if opts == nil { + return nil + } + // Check that exactly one of the probe handlers is set. + phSet := 0 + if opts.Http != nil { + phSet++ + } + if opts.Tcp != nil { + phSet++ + } + if opts.Exec != nil { + phSet++ + } + if phSet != 1 { + return fmt.Errorf("exactly one probe handler should be specified; %d provided", phSet) + } + + // Validate the handlers. + if opts.Http != nil && (opts.Http.Port < 1 || opts.Http.Port > 65535) { + return fmt.Errorf("http handler: invalid port %d", opts.Http.Port) + } + if opts.Tcp != nil && (opts.Tcp.Port < 1 || opts.Tcp.Port > 65535) { + return fmt.Errorf("tcp handler: invalid port %d", opts.Tcp.Port) + } + if opts.Exec != nil && len(opts.Exec.Cmd) == 0 { + return fmt.Errorf("exec handler: no commands specified") + } + return nil +} diff --git a/internal/impl/kube.go b/internal/impl/kube.go index 476e2c5..1e5a00b 100644 --- a/internal/impl/kube.go +++ b/internal/impl/kube.go @@ -311,7 +311,7 @@ func (r *replicaSet) buildContainer(cfg *kubeConfig) (corev1.Container, error) { return corev1.Container{}, err } - return corev1.Container{ + c := corev1.Container{ Name: appContainerName, Image: r.image, ImagePullPolicy: corev1.PullIfNotPresent, @@ -326,7 +326,49 @@ func (r *replicaSet) buildContainer(cfg *kubeConfig) (corev1.Container, error) { // for debugging. TTY: true, Stdin: true, - }, nil + } + + createProbeFn := func(opts *probeOptions) *corev1.Probe { + probe := &corev1.Probe{} + if opts.TimeoutSecs > 0 { + probe.TimeoutSeconds = opts.TimeoutSecs + } + if opts.PeriodSecs > 0 { + probe.PeriodSeconds = opts.PeriodSecs + } + if opts.SuccessThreshold > 0 { + probe.SuccessThreshold = opts.SuccessThreshold + } + if opts.FailureThreshold > 0 { + probe.FailureThreshold = opts.FailureThreshold + } + if opts.Tcp != nil { + probe.TCPSocket = &corev1.TCPSocketAction{Port: intstr.IntOrString{IntVal: opts.Tcp.Port}} + } + if opts.Http != nil { + probe.HTTPGet = &corev1.HTTPGetAction{Port: intstr.IntOrString{IntVal: opts.Http.Port}} + if opts.Http.Path != "" { + // If no path specified, the HTTPGetAction will do health checks on "/". + probe.HTTPGet.Path = opts.Http.Path + } + } + if opts.Exec != nil { + // Command is optional for an ExecAction. However, it's confusing why that's + // the case, especially that this is the only parameter to configure for an + // ExecAction. + probe.Exec = &corev1.ExecAction{Command: opts.Exec.Cmd} + } + return probe + } + + // Add probes if any. + if cfg.LivenessProbeOpts != nil { + c.LivenessProbe = createProbeFn(cfg.LivenessProbeOpts) + } + if cfg.ReadinessProbeOpts != nil { + c.LivenessProbe = createProbeFn(cfg.ReadinessProbeOpts) + } + return c, nil } // generateYAMLs generates Kubernetes YAML configurations for a given