Merge "Add zuul-merger to sf-operator"

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 
 ## [Alpha-2]
 
+- [X] Add Zuul Merger
 - [X] Add Zuul Log Levels to CRD
 - [X] Add Zuul Bootstrap Zuul Tenant Config subcommand to sfconfig cli
 - [X] Create ADR for sfconfig binary

api/v1/softwarefactory_types.go

@@ -158,6 +158,38 @@ type ZuulSchedulerSpec struct {
 	LogLevel LogLevel `json:"logLevel,omitempty"`
 }
 
+// Some of the Zuul Merger Configurations can be found at https://zuul-ci.org/docs/zuul/latest/configuration.html#merger
+type ZuulMergerSpec struct {
+	// https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_user_name
+	// +optional
+	GitUserName string `json:"gitUserName,omitempty"`
+	// https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_user_email
+	// +optional
+	GitUserEmail string `json:"gitUserEmail,omitempty"`
+	// https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_http_low_speed_limit
+	// +kubebuilder:validation:Minimum:=0
+	GitHTTPLowSpeedLimit int32 `json:"gitHttpLowSpeedLimit,omitempty"`
+	// https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_http_low_speed_time
+	// +kubebuilder:validation:Minimum:=0
+	GitHTTPLowSpeedTime int32 `json:"gitHttpLowSpeedTime,omitempty"`
+	// https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_timeout
+	// +kubebuilder:validation:Minimum:=1
+	GitTimeout int32 `json:"gitTimeout,omitempty"`
+	// Storage-related settings
+	Storage StorageSpec `json:"storage,omitempty"`
+	// How many merger pods to run
+	// +kubebuilder:default:=1
+	MinReplicas int32 `json:"minReplicas,omitempty"`
+	// Specify the Log Level of the nodepool launcher service.
+	// Valid values are:
+	// - "INFO" (default)
+	// - "WARN"
+	// - "DEBUG"
+	// Changing this value will restart the service.
+	// +optional
+	LogLevel LogLevel `json:"logLevel,omitempty"`
+}
+
 // TODO: make sure to update the GetConnectionsName when adding new connection type.
 
 // Configuration of the Zuul service
@@ -174,6 +206,8 @@ type ZuulSpec struct {
 	Scheduler ZuulSchedulerSpec `json:"scheduler,omitempty"`
 	// Configuration of the web microservice
 	Web ZuulWebSpec `json:"web,omitempty"`
+	// Configuration of the merger microservice
+	Merger ZuulMergerSpec `json:"merger,omitempty"`
 }
 
 func GetConnectionsName(spec *ZuulSpec) []string {

config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml

@@ -398,6 +398,68 @@ spec:
                       - name
                       type: object
                     type: array
+                  merger:
+                    description: Configuration of the merger microservice
+                    properties:
+                      gitHttpLowSpeedLimit:
+                        description: https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_http_low_speed_limit
+                        format: int32
+                        minimum: 0
+                        type: integer
+                      gitHttpLowSpeedTime:
+                        description: https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_http_low_speed_time
+                        format: int32
+                        minimum: 0
+                        type: integer
+                      gitTimeout:
+                        description: https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_timeout
+                        format: int32
+                        minimum: 1
+                        type: integer
+                      gitUserEmail:
+                        description: https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_user_email
+                        type: string
+                      gitUserName:
+                        description: https://zuul-ci.org/docs/zuul/latest/configuration.html#attr-merger.git_user_name
+                        type: string
+                      logLevel:
+                        description: 'Specify the Log Level of the nodepool launcher
+                          service. Valid values are: - "INFO" (default) - "WARN" -
+                          "DEBUG" Changing this value will restart the service.'
+                        enum:
+                        - INFO
+                        - WARN
+                        - DEBUG
+                        type: string
+                      minReplicas:
+                        default: 1
+                        description: How many merger pods to run
+                        format: int32
+                        type: integer
+                      storage:
+                        description: Storage-related settings
+                        properties:
+                          className:
+                            description: Default storage class to use with Persistent
+                              Volume Claims issued by this resource. Consult your
+                              cluster's configuration to see what storage classes
+                              are available and recommended for your use case.
+                            type: string
+                          size:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            description: 'Storage space to allocate to the resource,
+                              expressed as a Quantity: https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/'
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                            x-kubernetes-validations:
+                            - message: Storage shrinking is not supported
+                              rule: self >= oldSelf
+                        required:
+                        - size
+                        type: object
+                    type: object
                   oidcAuthenticators:
                     description: A list of OpenID Connect authenticators that will
                       enable admin API access on zuul-web

controllers/static/zuul/zuul.conf

@@ -22,6 +22,12 @@ trusted_ro_paths=/etc/pki
 untrusted_ro_paths=/etc/pki
 log_config=/var/lib/zuul/zuul-executor-logging.yaml
 
+[merger]
+log_config=/var/lib/zuul/zuul-merger-logging.yaml
+state_dir=/var/lib/zuul
+command_socket=/var/lib/zuul/merger.socket
+git_dir=/var/lib/zuul/git/
+
 [auth zuul_client]
 driver=HS256
 allow_authz_override=true
@@ -30,3 +36,4 @@ client_id=zuul-client
 
 [statsd]
 server=localhost
+

controllers/zuul.go

@@ -211,10 +211,9 @@ func (r *SFController) setZuulLoggingfile() {
 	if r.cr.Spec.Zuul.Web.LogLevel != "" {
 		zuulWebLogLevel = r.cr.Spec.Zuul.Web.LogLevel
 	}
-	// TODO: Uncomment when Zuul Merger is added
-	//if r.cr.Spec.Zuul.Merger.LogLevel != "" {
-	//	zuulMerverLogLevel = r.cr.Spec.Zuul.Web.LogLevel
-	//}
+	if r.cr.Spec.Zuul.Merger.LogLevel != "" {
+		zuulMergerLogLevel = r.cr.Spec.Zuul.Web.LogLevel
+	}
 
 	loggingData["zuul-executor-logging.yaml"], _ = utils.ParseString(zuulLoggingConfig, struct {
 		LogLevel string
@@ -364,6 +363,49 @@ func (r *SFController) EnsureZuulExecutor(cfg *ini.File) bool {
 	return isStatefulSet
 }
 
+func (r *SFController) EnsureZuulMerger(cfg *ini.File) bool {
+
+	service := "zuul-merger"
+
+	sections := utils.IniGetSectionNamesByPrefix(cfg, "connection")
+	sections = append(sections, "merger")
+
+	annotations := map[string]string{
+		"zuul-common-config":    utils.IniSectionsChecksum(cfg, commonIniConfigSections),
+		"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
+		"zuul-image":            ZuulImage(service),
+		"replicas":              strconv.Itoa(int(r.cr.Spec.Zuul.Merger.MinReplicas)),
+	}
+
+	zm := r.mkHeadlessSatefulSet(service, "", r.getStorageConfOrDefault(r.cr.Spec.Zuul.Merger.Storage), int32(r.cr.Spec.Zuul.Merger.MinReplicas), apiv1.ReadWriteOnce)
+	zm.Spec.Template.ObjectMeta.Annotations = annotations
+	zm.Spec.Template.Spec.Containers = r.mkZuulContainer(service)
+	zm.Spec.Template.Spec.Volumes = mkZuulVolumes(service)
+	zm.Spec.Template.Spec.Containers[0].ReadinessProbe = base.MkReadinessHTTPProbe("/health/ready", zuulPrometheusPort)
+	zm.Spec.Template.Spec.Containers[0].LivenessProbe = base.MkReadinessHTTPProbe("/health/live", zuulPrometheusPort)
+	zm.Spec.Template.Spec.Containers[0].Ports = []apiv1.ContainerPort{
+		base.MkContainerPort(zuulPrometheusPort, zuulPrometheusPortName),
+	}
+
+	current := appsv1.StatefulSet{}
+	if r.GetM(service, &current) {
+		if !utils.MapEquals(&current.Spec.Template.ObjectMeta.Annotations, &annotations) {
+			r.log.V(1).Info("zuul-merger configuration changed, rollout zuul-merger pods ...")
+			current.Spec = zm.DeepCopy().Spec
+			r.UpdateR(&current)
+			return false
+		}
+	} else {
+		current := zm
+		r.CreateR(&current)
+	}
+
+	isStatefulSet := r.IsStatefulSetReady(&current)
+	conds.UpdateConditions(&r.cr.Status.Conditions, service, isStatefulSet)
+
+	return isStatefulSet
+}
+
 func (r *SFController) EnsureZuulWeb(cfg *ini.File) bool {
 	sections := utils.IniGetSectionNamesByPrefix(cfg, "connection")
 	authSections := utils.IniGetSectionNamesByPrefix(cfg, "auth")
@@ -425,8 +467,9 @@ func (r *SFController) EnsureZuulComponents(initContainers []apiv1.Container, cf
 	zuulServices["scheduler"] = r.EnsureZuulScheduler(initContainers, cfg)
 	zuulServices["executor"] = r.EnsureZuulExecutor(cfg)
 	zuulServices["web"] = r.EnsureZuulWeb(cfg)
+	zuulServices["merger"] = r.EnsureZuulMerger(cfg)
 
-	return zuulServices["scheduler"] && zuulServices["executor"] && zuulServices["web"]
+	return zuulServices["scheduler"] && zuulServices["executor"] && zuulServices["web"] && zuulServices["merger"]
 }
 
 func (r *SFController) EnsureZuulPodMonitor() bool {
@@ -726,12 +769,29 @@ func (r *SFController) DeployZuul() bool {
 	}
 
 	// Enable prometheus metrics
-	for _, srv := range []string{"web", "executor", "scheduler"} {
+	for _, srv := range []string{"web", "executor", "scheduler", "merger"} {
 		cfgINI.Section(srv).NewKey("prometheus_port", strconv.Itoa(zuulPrometheusPort))
 	}
 	// Set Zuul web public URL
 	cfgINI.Section("web").NewKey("root", "https://zuul."+r.cr.Spec.FQDN)
 
+	// Set Zuul Merger Configurations
+	if r.cr.Spec.Zuul.Merger.GitUserName != "" {
+		cfgINI.Section("merger").NewKey("git_user_name", r.cr.Spec.Zuul.Merger.GitUserName)
+	}
+	if r.cr.Spec.Zuul.Merger.GitUserEmail != "" {
+		cfgINI.Section("merger").NewKey("git_user_email", r.cr.Spec.Zuul.Merger.GitUserEmail)
+	}
+	if r.cr.Spec.Zuul.Merger.GitHTTPLowSpeedLimit >= 0 {
+		cfgINI.Section("merger").NewKey("git_http_low_speed_limit", fmt.Sprint(r.cr.Spec.Zuul.Merger.GitHTTPLowSpeedLimit))
+	}
+	if r.cr.Spec.Zuul.Merger.GitHTTPLowSpeedTime >= 0 {
+		cfgINI.Section("merger").NewKey("git_http_low_speed_time", fmt.Sprint(r.cr.Spec.Zuul.Merger.GitHTTPLowSpeedTime))
+	}
+	if r.cr.Spec.Zuul.Merger.GitTimeout > 0 {
+		cfgINI.Section("merger").NewKey("git_timeout", fmt.Sprint(r.cr.Spec.Zuul.Merger.GitTimeout))
+	}
+
 	// Set Database DB URI
 	cfgINI.Section("database").NewKey("dburi", fmt.Sprintf("mysql+pymysql://zuul:%s@mariadb/zuul", dbPassword.Data["password"]))
 

roles/health-check/test-monitoring/tasks/main.yaml

@@ -20,6 +20,7 @@
     - "zuul-web"
     - "zuul-executor"
     - "zuul-scheduler"
+    - "zuul-merger"
     - "logserver-nodeexporter"
 
 - name: Fetch defined alerts for logserver
@@ -54,4 +55,4 @@
 
 - fail:
     msg: Unexpected value for nodepool_launch_ready
-  when: metric_dict.nlr | float < 1
+  when: metric_dict.nlr | float < 1

roles/health-check/zuul-components/tasks/main.yaml

@@ -0,0 +1,51 @@
+- name: Get Zuul Components
+  ansible.builtin.uri:
+    url: https://{{ zuul_host }}/zuul/api/components
+    status_code: [200]
+    method: GET
+    validate_certs: "{{ validate_certs }}"
+  register: _components
+
+- name: Check if, at least, one Zuul Web instance Exists, fails otherwise
+  ansible.builtin.fail:
+    msg: "Zuul Web has no instances"
+  when: "_components.json['web'] | length  == 0"
+
+- name: Check if Zuul Web Intances are running
+  ansible.builtin.fail:
+    msg: "Zuul Web has an element {{ item.hostname }} on a non-running state: {{ item.state }}"
+  failed_when: "'running' not in item.state"
+  loop: "{{ _components.json['web'] }}"
+
+- name: Check if, at least, one Zuul Scheduler instance Exists, fails otherwise
+  ansible.builtin.fail:
+    msg: "Zuul Scheduler has no instances"
+  when: "_components.json['scheduler'] | length  == 0"
+
+- name: Check if Zuul Scheduler Instances are Running
+  ansible.builtin.fail:
+    msg: "Zuul Scheduler has an element {{ item.hostname }} on a non-running state: {{ item.state }}"
+  failed_when: "'running' not in item.state"
+  loop: "{{ _components.json['scheduler'] }}"
+
+- name: Check if, at least, one Zuul Executor Instance Exists, fails otherwise
+  ansible.builtin.fail:
+    msg: "Zuul Executor has no instances"
+  when: "_components.json['executor'] | length  == 0"
+
+- name: Check if Zuul Executor Instances are Running
+  ansible.builtin.fail:
+    msg: "Zuul Executor has an element {{ item.hostname }} on a non-running state: {{ item.state }}"
+  failed_when: "'running' not in item.state"
+  loop: "{{ _components.json['executor'] }}"
+
+- name: Check if, at least, one Zuul Merger Instances Exists, fails otherwise
+  ansible.builtin.fail:
+    msg: "Zuul Merger has no instances"
+  when: "_components.json['merger'] | length  == 0"
+
+- name: Check if Zuul Merger Components are Running
+  ansible.builtin.fail:
+    msg: "Zuul Merger has an element {{ item.hostname }} on a non-running state: {{ item.state }}"
+  failed_when: "'running' not in item.state"
+  loop: "{{ _components.json['merger'] }}"

roles/run-tests/tasks/main.yaml

@@ -30,6 +30,7 @@
     - test-custom-route-certs
     - test-cert-manager-letsencrypt
     - zuul-client-api
+    - zuul-components
     - test-monitoring
   loop_control:
     loop_var: role