Skip to content

Commit

Permalink
Add GitLab connection support to Zuul Connections
Browse files Browse the repository at this point in the history 
Change-Id: I953316c047024dd8e3fc530af36e57138c0da8a0
  • Loading branch information
@fserucas
fserucas committed Nov 7, 2023
1 parent bd892ca commit de00a83
Show file tree
Hide file tree
Showing 7 changed files with 242 additions and 48 deletions.
36 changes: 36 additions & 0 deletions api/v1/softwarefactory_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,32 @@ type ConfigLocationSpec struct {
ZuulConnectionName string `json:"zuul-connection-name"`
}

// Describes a Zuul connection using the [gitlab driver](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#gitlab).
type GitLabConnection struct {
// How the connection will be named in Zuul's configuration and appear in zuul-web
Name string `json:"name"`
// the [server](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.server)
Server string `json:"server,omitempty"`
// the [canonicalHostname](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.canonical_hostname)
CanonicalHostname string `json:"canonicalHostname,omitempty"`
// the (baseUrl)[https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.baseurl)
BaseURL string `json:"baseUrl,omitempty"`
// Name of the secret which containes the following keys:
// the [api_token](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.api_token)
// the [api_token_name](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.api_token_name)
// the [webhook_token](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.webhook_token)
Secrets string `json:"secrets,omitempty"`
// the [apiTokenName](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.api_token_name)
APITokenName string `json:"apiTokenName,omitempty"`
// the [cloneUrl](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.cloneurl)
CloneURL string `json:"cloneUrl,omitempty"`
// the [keepAlive](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.keepalive)
// +kubebuilder:validation:Minimum:=0
KeepAlive uint8 `json:"keepAlive,omitempty"`
// the [disableConnectionPool](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.disable_connection_pool)
DisableConnectionPool bool `json:"disableConnectionPool,omitempty"`
}

// Describes a Zuul connection using the [github driver](https://zuul-ci.org/docs/zuul/latest/drivers/github.html#).
type GitHubConnection struct {
// How the connection will be named in Zuul's configuration and appear in zuul-web
Expand Down Expand Up @@ -225,6 +251,8 @@ type ZuulSpec struct {
GerritConns []GerritConnection `json:"gerritconns,omitempty"`
// The list of GitHub-based connections to add to Zuul's configuration
GitHubConns []GitHubConnection `json:"githubconns,omitempty"`
// The list of GitLab-based connections to add to Zuul's configuration
GitLabConns []GitLabConnection `json:"gitlabconns,omitempty"`
// Configuration of the executor microservices
Executor ZuulExecutorSpec `json:"executor,omitempty"`
// Configuration of the scheduler microservice
Expand Down Expand Up @@ -253,6 +281,14 @@ func GetGitHubConnectionsName(spec *ZuulSpec) []string {
return res
}

func GetGitLabConnectionsName(spec *ZuulSpec) []string {
var res []string
for _, conn := range spec.GitLabConns {
res = append(res, conn.Name)
}
return res
}

// +kubebuilder:validation:Enum=INFO;WARN;DEBUG
// +kubebuilder:default:=INFO
type LogLevel string
Expand Down
20 changes: 20 additions & 0 deletions api/v1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

42 changes: 42 additions & 0 deletions config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,48 @@ spec:
- name
type: object
type: array
gitlabconns:
description: The list of GitLab-based connections to add to Zuul's
configuration
items:
description: Describes a Zuul connection using the [gitlab driver](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#gitlab).
properties:
apiTokenName:
description: the [apiTokenName](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.api_token_name)
type: string
baseUrl:
description: the (baseUrl)[https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.baseurl)
type: string
canonicalHostname:
description: the [canonicalHostname](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.canonical_hostname)
type: string
cloneUrl:
description: the [cloneUrl](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.cloneurl)
type: string
disableConnectionPool:
description: the [disableConnectionPool](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.disable_connection_pool)
type: boolean
keepAlive:
description: the [keepAlive](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.keepalive)
minimum: 0
type: integer
name:
description: How the connection will be named in Zuul's
configuration and appear in zuul-web
type: string
secrets:
description: 'Name of the secret which containes the following
keys: the [api_token](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.api_token)
the [api_token_name](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.api_token_name)
the [webhook_token](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.webhook_token)'
type: string
server:
description: the [server](https://zuul-ci.org/docs/zuul/latest/drivers/gitlab.html#attr-%3Cgitlab%20connection%3E.server)
type: string
required:
- name
type: object
type: array
merger:
description: Configuration of the merger microservice
properties:
Expand Down
7 changes: 6 additions & 1 deletion controllers/git_server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,12 @@ func makeZuulConnectionConfig(spec *sfv1.ZuulSpec) string {
}
for _, name := range sfv1.GetGitHubConnectionsName(spec) {
sb.WriteString(fmt.Sprintf("[connection %s]\n", name))
sb.WriteString("driver=git\n")
sb.WriteString("driver=gitlab\n")
sb.WriteString("baseurl=localhost\n\n")
}
for _, name := range sfv1.GetGitLabConnectionsName(spec) {
sb.WriteString(fmt.Sprintf("[connection %s]\n", name))
sb.WriteString("driver=github\n")
sb.WriteString("baseurl=localhost\n\n")
}
return sb.String()
Expand Down
131 changes: 84 additions & 47 deletions controllers/zuul.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,21 @@ func mkZuulLoggingMount(service string) apiv1.VolumeMount {
}

func mkZuulGitHubSecretsMounts(r *SFController) []apiv1.VolumeMount {
zuulGitHubMounts := []apiv1.VolumeMount{}
zuulConnectionMounts := []apiv1.VolumeMount{}
secretkey := "app_key"
for _, connection := range r.cr.Spec.Zuul.GitHubConns {
zuulGitHubMounts = append(zuulGitHubMounts, apiv1.VolumeMount{
Name: connection.Secrets,
MountPath: "/var/lib/zuul/" + connection.Secrets + "/app_key",
SubPath: "app_key",
})
secretName := connection.Secrets

_, err := r.GetSecretDataFromKey(secretName, secretkey)
if err == nil {
zuulConnectionMounts = append(zuulConnectionMounts, apiv1.VolumeMount{
Name: secretName,
MountPath: "/var/lib/zuul/" + secretName + "/" + secretkey,
SubPath: secretkey,
})
}
}
return zuulGitHubMounts
return zuulConnectionMounts
}

func (r *SFController) mkZuulContainer(service string) []apiv1.Container {
Expand Down Expand Up @@ -257,17 +263,18 @@ func (r *SFController) getZuulLoggingString(service string) string {
}

func mkZuulGitHubSecretsVolumes(r *SFController) []apiv1.Volume {
gitConnectionSecretVolumes := []apiv1.Volume{}
for _, connection := range r.cr.Spec.Zuul.GitHubConns {
secretName := connection.Secrets

gitHubSecretVolumes := []apiv1.Volume{}
for _, gitHubConnection := range r.cr.Spec.Zuul.GitHubConns {
if _, err := r.GetSecretbyNameRef(gitHubConnection.Secrets); err != nil {
r.log.V(1).Error(err, "Error while getting secret "+gitHubConnection.Secrets)
if _, err := r.GetSecretbyNameRef(secretName); err != nil {
r.log.V(1).Error(err, "Error while getting secret "+secretName)
continue
}

gitHubSecretVolumes = append(gitHubSecretVolumes, base.MkVolumeSecret(gitHubConnection.Secrets))
gitConnectionSecretVolumes = append(gitConnectionSecretVolumes, base.MkVolumeSecret(secretName))
}
return gitHubSecretVolumes
return gitConnectionSecretVolumes
}

func (r *SFController) EnsureZuulScheduler(initContainers []apiv1.Container, cfg *ini.File) bool {
Expand All @@ -278,13 +285,13 @@ func (r *SFController) EnsureZuulScheduler(initContainers []apiv1.Container, cfg
sections = append(sections, "scheduler")

annotations := map[string]string{
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage("zuul-scheduler"),
"statsd_mapping": utils.Checksum([]byte(zuulStatsdMappingConfig)),
"serial": "3",
"zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-scheduler"))),
"zuul-github-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage("zuul-scheduler"),
"statsd_mapping": utils.Checksum([]byte(zuulStatsdMappingConfig)),
"serial": "3",
"zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-scheduler"))),
"zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
}

if r.isConfigRepoSet() {
Expand Down Expand Up @@ -349,13 +356,13 @@ func (r *SFController) EnsureZuulExecutor(cfg *ini.File) bool {
sections := utils.IniGetSectionNamesByPrefix(cfg, "connection")
sections = append(sections, "executor")
annotations := map[string]string{
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage("zuul-executor"),
"replicas": strconv.Itoa(int(r.cr.Spec.Zuul.Executor.Replicas)),
"serial": "1",
"zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-executor"))),
"zuul-github-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage("zuul-executor"),
"replicas": strconv.Itoa(int(r.cr.Spec.Zuul.Executor.Replicas)),
"serial": "1",
"zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-executor"))),
"zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
}

ze := r.mkHeadlessSatefulSet("zuul-executor", "", r.getStorageConfOrDefault(r.cr.Spec.Zuul.Scheduler.Storage), int32(r.cr.Spec.Zuul.Executor.Replicas), apiv1.ReadWriteOnce)
Expand Down Expand Up @@ -400,11 +407,11 @@ func (r *SFController) EnsureZuulMerger(cfg *ini.File) bool {
sections = append(sections, "merger")

annotations := map[string]string{
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage(service),
"replicas": strconv.Itoa(int(r.cr.Spec.Zuul.Merger.MinReplicas)),
"zuul-github-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage(service),
"replicas": strconv.Itoa(int(r.cr.Spec.Zuul.Merger.MinReplicas)),
"zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
}

zm := r.mkHeadlessSatefulSet(service, "", r.getStorageConfOrDefault(r.cr.Spec.Zuul.Merger.Storage), int32(r.cr.Spec.Zuul.Merger.MinReplicas), apiv1.ReadWriteOnce)
Expand Down Expand Up @@ -442,12 +449,12 @@ func (r *SFController) EnsureZuulWeb(cfg *ini.File) bool {
sections = append(sections, authSections...)
sections = append(sections, "web")
annotations := map[string]string{
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage("zuul-web"),
"serial": "1",
"zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-web"))),
"zuul-github-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
"zuul-common-config": utils.IniSectionsChecksum(cfg, commonIniConfigSections),
"zuul-component-config": utils.IniSectionsChecksum(cfg, sections),
"zuul-image": base.ZuulImage("zuul-web"),
"serial": "1",
"zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-web"))),
"zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
}

zw := base.MkDeployment("zuul-web", r.ns, "")
Expand Down Expand Up @@ -729,7 +736,14 @@ func (r *SFController) AddGerritConnection(cfg *ini.File, conn sfv1.GerritConnec
}
}

func (r *SFController) AddGitHubConnection(cfg *ini.File, conn sfv1.GitHubConnection) error {
// addKeyToSection add a tuple to the Section if the fieldValue is not empty
func addKeyToSection(section *ini.Section, fieldKey string, fieldValue string) {
if fieldValue != "" {
section.NewKey(fieldKey, fieldValue)
}
}

func (r *SFController) AddGitHubConnection(cfg *ini.File, conn sfv1.GitHubConnection) {

appID := fmt.Sprintf("%d", conn.AppID)
appKey := "/var/lib/zuul/" + conn.Secrets + "/app_key"
Expand Down Expand Up @@ -759,12 +773,6 @@ func (r *SFController) AddGitHubConnection(cfg *ini.File, conn sfv1.GitHubConnec
section := "connection " + conn.Name
cfg.NewSection(section)

addKey := func(fieldKey string, fieldValue string) {
if fieldValue != "" {
cfg.Section(section).NewKey(fieldKey, fieldValue)
}
}

for key, value := range map[string]string{
"driver": "github",
"app_id": appID,
Expand All @@ -776,10 +784,35 @@ func (r *SFController) AddGitHubConnection(cfg *ini.File, conn sfv1.GitHubConnec
"canonical_hostname": conn.Canonicalhostname,
"verify_ssl": fmt.Sprint(conn.VerifySSL),
} {
addKey(key, value)
addKeyToSection(cfg.Section(section), key, value)
}

}

func (r *SFController) AddGitLabConnection(cfg *ini.File, conn sfv1.GitLabConnection) {

apiToken, _ := r.GetSecretDataFromKey(conn.Secrets, "api_token")
webHookToken, _ := r.GetSecretDataFromKey(conn.Secrets, "webhook_token")

section := "connection " + conn.Name
cfg.NewSection(section)

for key, value := range map[string]string{
"driver": "gitlab",
"api_token": string(apiToken),
"api_token_name": conn.APITokenName,
"webhook_token": string(webHookToken),
"server": conn.Server,
"canonical_hostname": conn.CanonicalHostname,
"baseurl": conn.BaseURL,
"sshkey": "/var/lib/zuul-ssh/..data/priv",
"cloneurl": conn.CloneURL,
"keepalive": fmt.Sprint(conn.KeepAlive),
"disable_connection_pool": fmt.Sprint(conn.DisableConnectionPool),
} {
addKeyToSection(cfg.Section(section), key, value)
}

return nil
}

func AddGitConnection(cfg *ini.File, name string, baseurl string) {
Expand Down Expand Up @@ -849,6 +882,10 @@ func (r *SFController) DeployZuul() bool {
r.AddGitHubConnection(cfgINI, conn)
}

for _, conn := range r.cr.Spec.Zuul.GitLabConns {
r.AddGitLabConnection(cfgINI, conn)
}

// Add default connections
r.AddDefaultConnections(cfgINI)

Expand Down
Loading

0 comments on commit de00a83

Please sign in to comment.