Add options to configure probes

In kubernetes probes are used to determine when to restart a container,
to know when a container is ready to start accepting traffic, or to know
when when a container application has started.

There are 3 types of probes: liveness, readiness and startup probes.
Among there liveness and readiness are the most common and also the ones
required by the users in the Go survey.

This PR adds support to specify the liveness and readiness probes in the
weaver kube deployer. The knobs exposed to the user are not that many:
1) Knobs that define things like:
- how often to probe (optional)
- how many seconds to wait for an answer before declaring the probe
failing (optional)
- how many consecutive successful probes to declare the container as healthy
  (optional)
- how many consecutive failed probes to declare the container as
  unhealthy (optional)

2) Define what the probe should do
- should you use HTTP for probing? If so, you can specify the port and a
  path to check for probing (port is mandatory, path is optional)
- should you use TCP for probing? If so, you can specify the port
- should you use a custom list of commands?

We took the minimum number of knobs that might make sense for the user
to configure probes. I think it should cover most of the usecases and
scenarios someone might run into.

cmd/weaver-kube/deploy.go

@@ -116,6 +116,24 @@ Container Image Names:
       You can also specify any combination of the various options or none.
 
       [1] https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+
+    e) Configure probes [1]. The kube deployer allows you to configure readiness
+       and liveness probes. For each probe, you can configure:
+         - how often to perform the probe "period_secs"
+         - how long to wait for a probe to respond before declaring a timeout "timeout_secs"
+         - minimum consecutive successes for the probe to be successful "success_threshold"
+         - minimum consecutive failures for the probe to be considered failed "failure_threshold"
+         - a probe handler that describes the probe behavior. You can use a TCP,
+           HTTP or a custom commands probe handler.
+
+       E.g.,
+       [kube.readiness_probe]
+       period_secs = 2
+       [kube.readiness_probe.http]
+       path = "/health"
+       port = 8081
+
+      [1] https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
 `,
 		Flags: flags,
 		Fn: func(ctx context.Context, args []string) error {

internal/impl/config.go

@@ -104,6 +104,10 @@ type kubeConfig struct {
 	//
 	// [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#example-MustParse.
 	Resources resourceRequirements
+
+	// Options for probes to check the readiness/liveness of the pods.
+	LivenessProbeOpts  *probeOptions `toml:"liveness_probe"`
+	ReadinessProbeOpts *probeOptions `toml:"readiness_probe"`
 }
 
 // listenerConfig stores configuration options for a listener.
@@ -134,3 +138,56 @@ type resourceRequirements struct {
 	// Describes the maximum amount of memory allowed to run the pod.
 	LimitsMem string `toml:"limits_mem"`
 }
+
+// probeOptions stores the probes [1] configuration for the pods. These options
+// mirror the Kubernetes probe options available in [2].
+//
+// [1] https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+// [2] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2277
+//
+// TODO(rgrandl): There are a few more knobs available in the kubernetes probe
+// definition. We can enable more knobs if really needed.
+type probeOptions struct {
+	// How often to perform the probe.
+	PeriodSecs int32 `toml:"period_secs"`
+	// Number of seconds after which the probe times out.
+	TimeoutSecs int32 `toml:"timeout_secs"`
+	// Minimum consecutive successes for the probe to be considered successful after having failed.
+	SuccessThreshold int32 `toml:"success_threshold"`
+	// Minimum consecutive failures for the probe to be considered failed after having succeeded.
+	FailureThreshold int32 `toml:"failure_threshold"`
+
+	// Probe behavior. Note that only one of the following should be set by the user.
+
+	// The probe action is taken by executing commands.
+	Exec *execAction
+	// The probe action is taken by executing HTTP GET requests.
+	Http *httpAction
+	// The probe action is taken by executing TCP requests.
+	Tcp *tcpAction
+}
+
+// execAction describes the probe action when using a list of commands. It mirrors
+// Kubernetes ExecAction [1].
+//
+// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2265
+type execAction struct {
+	Cmd []string // List of commands to execute inside the container.
+}
+
+// httpAction describes the probe action when using HTTP. It mirrors Kubernetes
+// HTTPGetAction [1].
+//
+// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2208
+type httpAction struct {
+	Path string // Path to access on the HTTP server.
+	Port int32  // Port number to access on the container.
+}
+
+// tcpAction describes the probe action when using TCP. It mirrors Kubernetes
+// TCPSocketAction [1].
+//
+// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2241
+type tcpAction struct {
+	Port int32 // Port number to access on the container.
+}

internal/impl/deploy.go

@@ -64,6 +64,15 @@ func Deploy(ctx context.Context, configFilename string) error {
 	if config.ServiceAccount == "" {
 		config.ServiceAccount = "default"
 	}
+
+	// Validate the probe options.
+	if err := checkProbeOptions(config.LivenessProbeOpts); err != nil {
+		return fmt.Errorf("invalid liveness probe spec: %w", err)
+	}
+	if err := checkProbeOptions(config.ReadinessProbeOpts); err != nil {
+		return fmt.Errorf("invalid readiness probe spec: %w", err)
+	}
+
 	binListeners, err := bin.ReadListeners(app.Binary)
 	if err != nil {
 		return fmt.Errorf("cannot read listeners from binary %s: %w", app.Binary, err)
@@ -138,3 +147,36 @@ func checkVersionCompatibility(appBinary string) error {
 	}
 	return nil
 }
+
+// checkProbeOptions validates the configuration options for the probes.
+func checkProbeOptions(opts *probeOptions) error {
+	if opts == nil {
+		return nil
+	}
+	// Check that exactly one of the probe handlers is set.
+	phSet := 0
+	if opts.Http != nil {
+		phSet++
+	}
+	if opts.Tcp != nil {
+		phSet++
+	}
+	if opts.Exec != nil {
+		phSet++
+	}
+	if phSet != 1 {
+		return fmt.Errorf("exactly one probe handler should be specified; %d provided", phSet)
+	}
+
+	// Validate the handlers.
+	if opts.Http != nil && (opts.Http.Port < 1 || opts.Http.Port > 65535) {
+		return fmt.Errorf("http handler: invalid port %d", opts.Http.Port)
+	}
+	if opts.Tcp != nil && (opts.Tcp.Port < 1 || opts.Tcp.Port > 65535) {
+		return fmt.Errorf("tcp handler: invalid port %d", opts.Tcp.Port)
+	}
+	if opts.Exec != nil && len(opts.Exec.Cmd) == 0 {
+		return fmt.Errorf("exec handler: no commands specified")
+	}
+	return nil
+}

internal/impl/kube.go

@@ -311,7 +311,7 @@ func (r *replicaSet) buildContainer(cfg *kubeConfig) (corev1.Container, error) {
 		return corev1.Container{}, err
 	}
 
-	return corev1.Container{
+	c := corev1.Container{
 		Name:            appContainerName,
 		Image:           r.image,
 		ImagePullPolicy: corev1.PullIfNotPresent,
@@ -326,7 +326,49 @@ func (r *replicaSet) buildContainer(cfg *kubeConfig) (corev1.Container, error) {
 		// for debugging.
 		TTY:   true,
 		Stdin: true,
-	}, nil
+	}
+
+	createProbeFn := func(opts *probeOptions) *corev1.Probe {
+		probe := &corev1.Probe{}
+		if opts.TimeoutSecs > 0 {
+			probe.TimeoutSeconds = opts.TimeoutSecs
+		}
+		if opts.PeriodSecs > 0 {
+			probe.PeriodSeconds = opts.PeriodSecs
+		}
+		if opts.SuccessThreshold > 0 {
+			probe.SuccessThreshold = opts.SuccessThreshold
+		}
+		if opts.FailureThreshold > 0 {
+			probe.FailureThreshold = opts.FailureThreshold
+		}
+		if opts.Tcp != nil {
+			probe.TCPSocket = &corev1.TCPSocketAction{Port: intstr.IntOrString{IntVal: opts.Tcp.Port}}
+		}
+		if opts.Http != nil {
+			probe.HTTPGet = &corev1.HTTPGetAction{Port: intstr.IntOrString{IntVal: opts.Http.Port}}
+			if opts.Http.Path != "" {
+				// If no path specified, the HTTPGetAction will do health checks on "/".
+				probe.HTTPGet.Path = opts.Http.Path
+			}
+		}
+		if opts.Exec != nil {
+			// Command is optional for an ExecAction. However, it's confusing why that's
+			// the case, especially that this is the only parameter to configure for an
+			// ExecAction.
+			probe.Exec = &corev1.ExecAction{Command: opts.Exec.Cmd}
+		}
+		return probe
+	}
+
+	// Add probes if any.
+	if cfg.LivenessProbeOpts != nil {
+		c.LivenessProbe = createProbeFn(cfg.LivenessProbeOpts)
+	}
+	if cfg.ReadinessProbeOpts != nil {
+		c.LivenessProbe = createProbeFn(cfg.ReadinessProbeOpts)
+	}
+	return c, nil
 }
 
 // generateYAMLs generates Kubernetes YAML configurations for a given