Merge "refine corporate-ca-certs and init container phases"

softwarefactory-project · Jan 12, 2024 · 35b761c · 35b761c
2 parents 065c68a + ebf7a87
commit 35b761c
Show file tree

Hide file tree

Showing 8 changed files with 212 additions and 116 deletions.
diff --git a/controllers/nodepool.go b/controllers/nodepool.go
@@ -22,6 +22,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
+//go:embed static/nodepool/init-container.sh
+var initContainerScript string
+
 //go:embed static/nodepool/generate-config.sh
 var generateConfigScript string
 
@@ -68,11 +71,19 @@ const (
 
 var NodepoolStatsdExporterPortName = monitoring.GetStatsdExporterPort(shortIdent)
 
-var configScriptVolumeMount = apiv1.VolumeMount{
-	Name:      "nodepool-tooling-vol",
-	SubPath:   "generate-config.sh",
-	MountPath: "/usr/local/bin/generate-config.sh",
-	ReadOnly:  true,
+var configScriptsVolumeMounts = []apiv1.VolumeMount{
+	{
+		Name:      "nodepool-tooling-vol",
+		SubPath:   "generate-config.sh",
+		MountPath: "/usr/local/bin/generate-config.sh",
+		ReadOnly:  true,
+	},
+	{
+		Name:      "nodepool-tooling-vol",
+		SubPath:   "fetch-config-repo.sh",
+		MountPath: "/usr/local/bin/fetch-config-repo.sh",
+		ReadOnly:  true,
+	},
 }
 
 var nodepoolFluentBitLabels = []logging.FluentBitLabel{
@@ -118,7 +129,9 @@ func createImageBuildLogForwarderSidecar(r *SFController, annotations map[string
 
 func (r *SFController) setNodepoolTooling() {
 	toolingData := make(map[string]string)
+	toolingData["init-container.sh"] = initContainerScript
 	toolingData["generate-config.sh"] = generateConfigScript
+	toolingData["fetch-config-repo.sh"] = fetchConfigRepoScript
 	toolingData["dib-ansible.py"] = dibAnsibleWrapper
 	toolingData["ssh_config"] = builderSSHConfig
 	toolingData["timestamp.py"] = timestampOutputCallback
@@ -455,7 +468,6 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 			Name:      "nodepool-ca",
 			MountPath: "/etc/pki/ca-trust/extracted",
 		},
-		configScriptVolumeMount,
 		{
 			Name:      "nodepool-tooling-vol",
 			SubPath:   "dib-ansible.py",
@@ -498,15 +510,9 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 		},
 	}...)
 
+	volumeMounts = append(volumeMounts, configScriptsVolumeMounts...)
 	volumeMounts = append(volumeMounts, nodeExporterVolumeMount...)
 
-	if corporateCMExists {
-		volumeMounts = append(volumeMounts, apiv1.VolumeMount{
-			Name:      "nodepool-builder-corporate-ca-certs",
-			MountPath: UpdateCATrustAnchorsPath,
-		})
-	}
-
 	annotations := map[string]string{
 		"nodepool.yaml":              utils.Checksum([]byte(generateConfigScript)),
 		"nodepool-logging.yaml":      utils.Checksum([]byte(loggingConfig)),
@@ -516,33 +522,49 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 		"statsd_mapping":             utils.Checksum([]byte(nodepoolStatsdMappingConfig)),
 		"image":                      base.NodepoolBuilderImage,
 		"nodepool-providers-secrets": getSecretsVersion(providersSecrets, providerSecretsExists),
-		"serial":                     "13",
+		"serial":                     "14",
 		"corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists),
 	}
 
-	initContainer := base.MkContainer("nodepool-builder-init", base.BusyboxImage)
+	initContainer := base.MkContainer("nodepool-builder-init", base.NodepoolBuilderImage)
 
-	initContainer.Command = []string{"bash", "-c", "mkdir -p ~/dib ~/nodepool/builds; /usr/local/bin/generate-config.sh"}
+	initContainer.Command = []string{"/usr/local/bin/init-container.sh"}
 	initContainer.Env = append(r.getNodepoolConfigEnvs(),
 		base.MkEnvVar("NODEPOOL_CONFIG_FILE", "nodepool-builder.yaml"),
 	)
 	initContainer.VolumeMounts = []apiv1.VolumeMount{
+		{
+			Name:      "nodepool-tooling-vol",
+			SubPath:   "init-container.sh",
+			MountPath: "/usr/local/bin/init-container.sh",
+			ReadOnly:  true,
+		},
 		{
 			Name:      "nodepool-config",
 			MountPath: "/etc/nodepool/",
 		},
-		configScriptVolumeMount,
+		{
+			Name:      "nodepool-ca",
+			MountPath: "/etc/pki/ca-trust/extracted",
+		},
 	}
+
+	initContainer.VolumeMounts = append(initContainer.VolumeMounts, configScriptsVolumeMounts...)
 	initContainer.VolumeMounts = append(initContainer.VolumeMounts, nodeExporterVolumeMount...)
 
+	if corporateCMExists {
+		initContainer.VolumeMounts = AppendCorporateCACertsVolumeMount(initContainer.VolumeMounts, "nodepool-builder-corporate-ca-certs")
+	}
+
 	nb := r.mkStatefulSet(
 		BuilderIdent, base.NodepoolBuilderImage, r.getStorageConfOrDefault(r.cr.Spec.Nodepool.Builder.Storage),
 		apiv1.ReadWriteOnce)
 
 	nb.Spec.Template.Spec.InitContainers = []apiv1.Container{initContainer}
 	nb.Spec.Template.Spec.Volumes = volumes
 	nb.Spec.Template.Spec.Containers[0].Command = []string{
-		"/usr/local/bin/dumb-init", "--", "bash", "-c", "mkdir /etc/pki/ca-trust/extracted/{pem,java,edk2,openssl} && update-ca-trust && /usr/local/bin/nodepool-builder -f -l /etc/nodepool-logging/logging.yaml",
+		"/usr/local/bin/dumb-init", "--", "bash", "-c",
+		"/usr/local/bin/nodepool-builder -f -l /etc/nodepool-logging/logging.yaml",
 	}
 	nb.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
 	nb.Spec.Template.Spec.Containers[0].Env = r.getNodepoolConfigEnvs()
@@ -566,7 +588,7 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 	nb.Spec.Template.Spec.Containers = append(nb.Spec.Template.Spec.Containers, diskUsageExporter)
 
 	// Append image build logs HTTPD sidecar
-	buildLogsContainer := base.MkContainer("build-logs-httpd", HTTPDImage)
+	buildLogsContainer := base.MkContainer("build-logs-httpd", base.HTTPDImage)
 	buildLogsContainer.VolumeMounts = []apiv1.VolumeMount{
 		{
 			Name:      BuilderIdent,
@@ -668,20 +690,15 @@ func (r *SFController) DeployNodepoolLauncher(statsdExporterVolume apiv1.Volume,
 			SubPath:   "logging.yaml",
 			MountPath: "/etc/nodepool-logging/logging.yaml",
 		},
-		configScriptVolumeMount}...,
-	)
-	if corporateCMExists {
-		volumeMounts = append(volumeMounts, apiv1.VolumeMount{
-			Name:      "nodepool-launcher-corporate-ca-certs",
-			MountPath: UpdateCATrustAnchorsPath,
-		})
-	}
+	}...)
+
+	volumeMounts = append(volumeMounts, configScriptsVolumeMounts...)
 
 	annotations := map[string]string{
 		"nodepool.yaml":         utils.Checksum([]byte(generateConfigScript)),
 		"nodepool-logging.yaml": utils.Checksum([]byte(loggingConfig)),
 		"statsd_mapping":        utils.Checksum([]byte(nodepoolStatsdMappingConfig)),
-		"serial":                "7",
+		"serial":                "8",
 		// When the Secret ResourceVersion field change (when edited) we force a nodepool-launcher restart
 		"image":                      base.NodepoolLauncherImage,
 		"nodepool-providers-secrets": getSecretsVersion(providersSecrets, providerSecretsExists),
@@ -692,23 +709,17 @@ func (r *SFController) DeployNodepoolLauncher(statsdExporterVolume apiv1.Volume,
 		annotations["config-repo-info-hash"] = r.cr.Spec.ConfigRepositoryLocation.BaseURL + r.cr.Spec.ConfigRepositoryLocation.Name
 	}
 
-	nl := base.MkDeployment("nodepool-launcher", r.ns, "")
+	initContainer := base.MkContainer("nodepool-launcher-init", base.NodepoolLauncherImage)
 
-	container := base.MkContainer("launcher", base.NodepoolLauncherImage)
-	container.VolumeMounts = volumeMounts
-	container.Command = []string{
-		"/usr/local/bin/dumb-init", "--", "bash", "-c", "mkdir /etc/pki/ca-trust/extracted/{pem,java,edk2,openssl} && update-ca-trust && /usr/local/bin/nodepool-launcher -f -l /etc/nodepool-logging/logging.yaml",
-	}
-	container.Env = r.getNodepoolConfigEnvs()
-
-	extraLoggingEnvVars := logging.SetupLogForwarding("nodepool-launcher", r.cr.Spec.FluentBitLogForwarding, nodepoolFluentBitLabels, annotations)
-	container.Env = append(container.Env, extraLoggingEnvVars...)
-
-	initContainer := base.MkContainer("nodepool-launcher-init", base.BusyboxImage)
-
-	initContainer.Command = []string{"/usr/local/bin/generate-config.sh"}
+	initContainer.Command = []string{"/usr/local/bin/init-container.sh"}
 	initContainer.Env = r.getNodepoolConfigEnvs()
 	initContainer.VolumeMounts = []apiv1.VolumeMount{
+		{
+			Name:      "nodepool-tooling-vol",
+			SubPath:   "init-container.sh",
+			MountPath: "/usr/local/bin/init-container.sh",
+			ReadOnly:  true,
+		},
 		{
 			Name:      "nodepool-config",
 			MountPath: "/etc/nodepool/",
@@ -717,9 +728,31 @@ func (r *SFController) DeployNodepoolLauncher(statsdExporterVolume apiv1.Volume,
 			Name:      "nodepool-home",
 			MountPath: "/var/lib/nodepool",
 		},
-		configScriptVolumeMount,
+		{
+			Name:      "nodepool-ca",
+			MountPath: "/etc/pki/ca-trust/extracted",
+		},
 	}
 
+	initContainer.VolumeMounts = append(initContainer.VolumeMounts, configScriptsVolumeMounts...)
+
+	if corporateCMExists {
+		initContainer.VolumeMounts = AppendCorporateCACertsVolumeMount(initContainer.VolumeMounts, "nodepool-launcher-corporate-ca-certs")
+	}
+
+	nl := base.MkDeployment("nodepool-launcher", r.ns, "")
+
+	container := base.MkContainer("launcher", base.NodepoolLauncherImage)
+	container.VolumeMounts = volumeMounts
+	container.Command = []string{
+		"/usr/local/bin/dumb-init", "--", "bash", "-c",
+		"/usr/local/bin/nodepool-launcher -f -l /etc/nodepool-logging/logging.yaml",
+	}
+	container.Env = r.getNodepoolConfigEnvs()
+
+	extraLoggingEnvVars := logging.SetupLogForwarding("nodepool-launcher", r.cr.Spec.FluentBitLogForwarding, nodepoolFluentBitLabels, annotations)
+	container.Env = append(container.Env, extraLoggingEnvVars...)
+
 	nl.Spec.Template.Spec.Volumes = volumes
 	nl.Spec.Template.Spec.InitContainers = []apiv1.Container{initContainer}
 	nl.Spec.Template.Spec.Containers = []apiv1.Container{

diff --git a/controllers/static/fetch-config-repo.sh b/controllers/static/fetch-config-repo.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -ex
+
+# config-update usage context required a specific git ref
+REF=$1
+
+# Clone or fetch config repository
+if [ -d ~/config/.git ]; then
+  pushd ~/config
+  git remote | grep origin && git remote remove origin
+  git remote add origin ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME}
+  if [ -z "$REF" ]; then
+    # Discover default remote branch ref
+    REF="origin/$(git remote show origin | sed -n '/HEAD branch/s/.*: //p')"
+  fi
+  if [ "$INIT_CONTAINER" == "1" ]; then
+    git fetch origin || true
+    git reset --hard $REF || true
+  else
+    git fetch origin
+    git reset --hard $REF
+  fi
+  popd
+else
+  pushd ~/
+  if [ "$INIT_CONTAINER" == "1" ]; then
+    git clone ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME} config || true
+  else
+    git clone ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME} config
+  fi
+  popd
+fi
diff --git a/controllers/static/nodepool/generate-config.sh b/controllers/static/nodepool/generate-config.sh
@@ -28,22 +28,10 @@ EOF
 if [ "$CONFIG_REPO_SET" == "TRUE" ]; then
   # A config repository has been set
 
+  # config-update usage context required a specific git ref
   REF=$1
-  REF=${REF:-origin/master}
-
-  # Clone or fetch config repository
-  if [ -d ~/config/.git ]; then
-    pushd ~/config
-    git remote remove origin
-    git remote add origin ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME}
-    git fetch origin
-    git reset --hard $REF
-    popd
-  else
-    pushd ~/
-    git clone ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME} config
-    popd
-  fi
+
+  /usr/local/bin/fetch-config-repo.sh $REF
 
   # Append the config repo provided config file to the default one
   if [ -f ~/config/nodepool/${NODEPOOL_CONFIG_FILE} ]; then

diff --git a/controllers/static/nodepool/init-container.sh b/controllers/static/nodepool/init-container.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -ex
+
+# Update the CA Trust chain
+mkdir -p /etc/pki/ca-trust/extracted/{pem,java,edk2,openssl}
+update-ca-trust
+
+# This create some directory expected by nodepool-builder
+mkdir -p ~/dib ~/nodepool/builds
+
+# Generate the Nodepool configuration
+/usr/local/bin/generate-config.sh
diff --git a/controllers/static/zuul/generate-tenant-config.sh b/controllers/static/zuul/generate-tenant-config.sh
@@ -22,31 +22,10 @@ EOF
 if [ "$CONFIG_REPO_SET" == "TRUE" ]; then
   # A config repository has been set
 
+  # config-update usage context required a specific git ref
   REF=$1
-  REF=${REF:-origin/master}
 
-  # Clone or fetch config repository
-  if [ -d ~/config/.git ]; then
-    pushd ~/config
-    git remote | grep origin && git remote remove origin
-    git remote add origin ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME}
-    if [ "$INIT_CONTAINER" == "1" ]; then
-      git fetch origin || true
-      git reset --hard $REF || true
-    else
-      git fetch origin
-      git reset --hard $REF
-    fi
-    popd
-  else
-    pushd ~/
-    if [ "$INIT_CONTAINER" == "1" ]; then
-      git clone ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME} config || true
-    else
-      git clone ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME} config
-    fi
-    popd
-  fi
+  /usr/local/bin/fetch-config-repo.sh $REF
 
   # Ensure the config repo enabled into the tenants config
   cat << EOF >> ~/main.yaml

diff --git a/controllers/static/zuul/scheduler-init-container.sh b/controllers/static/zuul/scheduler-init-container.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -ex
+
+# Update the CA Trust chain
+mkdir -p /etc/pki/ca-trust/extracted/{pem,java,edk2,openssl}
+update-ca-trust
+
+# Generate the Zuul tenant configuration
+/usr/local/bin/generate-zuul-tenant-yaml.sh
diff --git a/controllers/utils.go b/controllers/utils.go
@@ -44,15 +44,14 @@ import (
 )
 
 const (
-	BusyboxImage             = "quay.io/software-factory/sf-op-busybox:1.5-3"
 	CustomSSLSecretName      = "sf-ssl-cert"
 	CorporateCACerts         = "corporate-ca-certs"
 	UpdateCATrustAnchorsPath = "/usr/share/pki/ca-trust-source/anchors/"
+	UpdateCATrustCommand     = "set -x && mkdir -p /etc/pki/ca-trust/extracted/{pem,java,edk2,openssl} && update-ca-trust"
 )
 
-// HTTPDImage uses pinned/ubi8 based image for httpd
-// https://catalog.redhat.com/software/containers/ubi8/httpd-24/6065b844aee24f523c207943?q=httpd&architecture=amd64&image=651f274c8ce9242f7bb3e011
-const HTTPDImage = "registry.access.redhat.com/ubi8/httpd-24:1-284.1696531168"
+//go:embed static/fetch-config-repo.sh
+var fetchConfigRepoScript string
 
 type SFUtilContext struct {
 	Client     client.Client
@@ -777,3 +776,11 @@ func (r *SFUtilContext) CorporateCAConfigMapExists() (apiv1.ConfigMap, bool) {
 	cm, corporateCA := r.GetConfigMap(CorporateCACerts)
 	return cm, corporateCA == nil
 }
+
+func AppendCorporateCACertsVolumeMount(volumeMounts []apiv1.VolumeMount, volumeName string) []apiv1.VolumeMount {
+	volumeMounts = append(volumeMounts, apiv1.VolumeMount{
+		Name:      volumeName,
+		MountPath: UpdateCATrustAnchorsPath,
+	})
+	return volumeMounts
+}