Skip to content

Commit

Permalink
Add yaml config file with configurable k8s knobs
Browse files Browse the repository at this point in the history
  • Loading branch information
@rgrandl
rgrandl committed Nov 3, 2023
1 parent 2b300ff commit faa86d7
Show file tree
Hide file tree
Showing 8 changed files with 362 additions and 416 deletions.
18 changes: 4 additions & 14 deletions internal/impl/babysitter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,10 @@ func (b *babysitter) watchPods(ctx context.Context, component string) error {
b.mu.Unlock()

// Watch the pods running the requested component.
rs := replicaSetName(component, b.app)
rs, ok := b.cfg.Groups[component]
if !ok {
return fmt.Errorf("unable to determine group name for component %s", component)
}
name := deploymentName(b.app.Name, rs, b.cfg.DeploymentId)
opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("serviceweaver/name=%s", name)}
watcher, err := b.clientset.CoreV1().Pods(b.cfg.Namespace).Watch(ctx, opts)
Expand Down Expand Up @@ -323,16 +326,3 @@ func serveHTTP(ctx context.Context, lis net.Listener, handler http.Handler) erro
return server.Shutdown(ctx)
}
}

// replicaSetName returns the name of the replica set that hosts a given
// component.
func replicaSetName(component string, app *protos.AppConfig) string {
for _, group := range app.Colocate {
for _, c := range group.Components {
if c == component {
return group.Components[0]
}
}
}
return component
}
180 changes: 89 additions & 91 deletions internal/impl/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,17 @@

package impl

// kubeConfig contains configuration for a Service Weaver application deployed
// with `weaver kube deploy`. The contents of a kubeConfig are parsed from the
// [kube] section of a weaver.toml file.
import (
autoscalingv2 "k8s.io/api/autoscaling/v2"
corev1 "k8s.io/api/core/v1"
)

// kubeConfig contains the kubernetes configuration for a Service Weaver
// application deployed with `weaver kube deploy`.
type kubeConfig struct {
// Path to the app config file.
AppConfig string

// Image is the name of the container image hosting the Service Weaver
// application.
//
Expand Down Expand Up @@ -51,70 +58,90 @@ type kubeConfig struct {
// account for your namespace.
//
// [1] https://kubernetes.io/docs/concepts/security/service-accounts/
ServiceAccount string `toml:"service_account"`
ServiceAccount string

// If true, application listeners will use the underlying nodes' network.
// This behavior is generally discouraged, but it may be useful when running
// the application in a minikube environment, where using the underlying
// nodes' network may make it easier to access the listeners directly from
// the host machine.
UseHostNetwork bool `toml:"use_host_network"`
UseHostNetwork bool

// Options for the application listeners, keyed by listener name.
// If a listener isn't specified in the map, default options will be used.
Listeners map[string]*listenerConfig
// Options for the application listeners. If a listener isn't specified in the
// map, default options will be used.
Listeners []listenerSpec

// Observability controls how the deployer will export observability information
// such as logs, metrics and traces, keyed by service. If no options are
// specified, the deployer will launch corresponding services for exporting logs,
// metrics and traces automatically.
// Resource requirements needed to run the pods. Note that the resources should
// satisfy the format specified in [1].
//
// [1] https://pkg.go.dev/k8s.io/api/core/v1#ResourceRequirements.
//
// TODO(rgrandl): we might want to allow the user to specify the resource
// requirements differently for each group of collocated components.
ResourceSpec *corev1.ResourceRequirements

// Specs on how to scale the pods. Note that the scaling specs should satisfy
// the format specified in [1].
//
// The key must be one of the following strings:
// "prometheus_service" - to export metrics to Prometheus [1]
// "jaeger_service" - to export traces to Jaeger [2]
// "loki_service" - to export logs to Grafana Loki [3]
// "grafana_service" - to visualize/manipulate observability information [4]
// [1] https://pkg.go.dev/k8s.io/kubernetes/pkg/apis/autoscaling#HorizontalPodAutoscalerSpec.
//
// TODO(rgrandl): we might want to allow the user to specify the scaling spec
// differently for each group of collocated components.
ScalingSpec *autoscalingv2.HorizontalPodAutoscalerSpec

// Volumes that should be provided to all the running components.
Volumes []volume

// Options for probes to check the readiness/liveness/startup of the pods.
// Note that the scaling specs should satisfy the format specified in [1].
//
// [1] https://pkg.go.dev/k8s.io/api/core/v1#Probe.
ProbeSpec probes

// Groups contains kubernetes configuration for groups of collocated components.
// Note that some knobs if specified for a group will override the corresponding
// knob set for all the groups (e.g., ScalingSpec, ResourceSpec); for knobs like
// Volumes, each group will contain the sum of the volumes specified for all
// groups and the ones set for the group.
Groups []group

// Observability controls how the deployer will export observability information
// such as logs, metrics and traces. If no options are specified, the deployer
//will launch corresponding services for exporting logs, metrics and traces automatically.
//
// Possible values for each service:
// 1) do not specify a value at all; leave it empty
// 1) do not specify a value at all
// this is the default behavior; kube deployer will automatically create the
// observability service for you.
//
// 2) "none"
// kube deployer will not export the corresponding observability information to
// any service. E.g., prometheus_service = "none" means that the user will not
// any service. E.g., `prometheusService: none` means that the user will not
// be able to see any metrics at all. This can be useful for testing or
// benchmarking the performance of your application.
//
// 3) "your_observability_service_name"
// 3) `your observability service name`
// if you already have a running service to collect metrics, traces or logs,
// then you can simply specify the service name, and your application will
// automatically export the corresponding information to your service. E.g.,
// jaeger_service = "jaeger-all-in-one" will enable your running Jaeger
// `jaegerService: jaeger-all-in-one` will enable your running Jaeger
// "service/jaeger-all-in-one" to capture all the app traces.
//
// [1] - https://prometheus.io/
// [2] - https://www.jaegertracing.io/
// [3] - https://grafana.com/oss/loki/
// [4] - https://grafana.com/
Observability map[string]string

// Resources needed to run the pods. Note that the resources should satisfy
// the format specified in [1].
//
// [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#example-MustParse.
Resources resourceRequirements

// Options for probes to check the readiness/liveness of the pods.
LivenessProbeOpts *probeOptions `toml:"liveness_probe"`
ReadinessProbeOpts *probeOptions `toml:"readiness_probe"`
Observability observability
}

// listenerConfig stores configuration options for a listener.
type listenerConfig struct {
// listenerSpec stores configuration options for a listener.
type listenerSpec struct {
// Listener name.
Name string

// If specified, the listener service will have the name set to this value.
// Otherwise, we will generate a unique name for each app version.
ServiceName string `toml:"service_name"`
ServiceName string

// Is the listener public, i.e., should it receive ingress traffic
// from the public internet. If false, the listener is configured only
Expand All @@ -127,67 +154,38 @@ type listenerConfig struct {
Port int32
}

// resourceRequirements stores the resource requirements configuration for running pods.
type resourceRequirements struct {
// Describes the minimum amount of CPU required to run the pod.
RequestsCPU string `toml:"requests_cpu"`
// Describes the minimum amount of memory required to run the pod.
RequestsMem string `toml:"requests_mem"`
// Describes the maximum amount of CPU allowed to run the pod.
LimitsCPU string `toml:"limits_cpu"`
// Describes the maximum amount of memory allowed to run the pod.
LimitsMem string `toml:"limits_mem"`
}

// probeOptions stores the probes [1] configuration for the pods. These options
// mirror the Kubernetes probe options available in [2].
//
// [1] https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
// [2] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2277
// Encapsulates probe specs as defined by the user in the kubernetes config.
//
// TODO(rgrandl): There are a few more knobs available in the kubernetes probe
// definition. We can enable more knobs if really needed.
type probeOptions struct {
// How often to perform the probe.
PeriodSecs int32 `toml:"period_secs"`
// Number of seconds after which the probe times out.
TimeoutSecs int32 `toml:"timeout_secs"`
// Minimum consecutive successes for the probe to be considered successful after having failed.
SuccessThreshold int32 `toml:"success_threshold"`
// Minimum consecutive failures for the probe to be considered failed after having succeeded.
FailureThreshold int32 `toml:"failure_threshold"`

// Probe behavior. Note that only one of the following should be set by the user.

// The probe action is taken by executing commands.
Exec *execAction
// The probe action is taken by executing HTTP GET requests.
Http *httpAction
// The probe action is taken by executing TCP requests.
Tcp *tcpAction
// Note that we create this struct, so we can group the probe specs under a single
// section in the yaml config file.
type probes struct {
ReadinessProbe *corev1.Probe // Periodic probe of container service readiness.
LivenessProbe *corev1.Probe // Periodic probe of container liveness.
StartupProbe *corev1.Probe // Indicates that the pod has successfully initialized.
}

// execAction describes the probe action when using a list of commands. It mirrors
// Kubernetes ExecAction [1].
//
// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2265
type execAction struct {
Cmd []string // List of commands to execute inside the container.
// group contains kubernetes configuration for a group of colocated components.
type group struct {
Name string // name of the group
Components []string // list of components in the group
Volumes []volume // list of volumes to mount
ResourceSpec *corev1.ResourceRequirements
ScalingSpec *autoscalingv2.HorizontalPodAutoscalerSpec
listeners []listener // hosted listeners, populated by the kube deployer.
}

// httpAction describes the probe action when using HTTP. It mirrors Kubernetes
// HTTPGetAction [1].
//
// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2208
type httpAction struct {
Path string // Path to access on the HTTP server.
Port int32 // Port number to access on the container.
// volume contains kubernetes information to configure a kubernetes volume.
type volume struct {
Name string
VolumeSource *corev1.VolumeSource
VolumeMount *corev1.VolumeMount
}

// tcpAction describes the probe action when using TCP. It mirrors Kubernetes
// TCPSocketAction [1].
//
// [1] https://github.com/kubernetes/api/blob/v0.28.3/core/v1/types.go#L2241
type tcpAction struct {
Port int32 // Port number to access on the container.
// observability contains information on how metrics, traces, logs and dashboards
// should be exported.
type observability struct {
JaegerService string
PrometheusService string
LokiService string
GrafanaService string
}
Loading

0 comments on commit faa86d7

Please sign in to comment.