Add tests for database and distributed architecture

Signed-off-by: Dmitry Tantsur <[email protected]>

.github/workflows/functional.yml

@@ -14,9 +14,14 @@ jobs:
     - uses: helm/kind-action@v1
       with:
         cluster_name: kind
-    - name: Prepare and run the tests
-      run: NOCLEAN=1 ./test/functional.sh
-
+    - name: Prepare tests
+      run: ./test/prepare.sh
+    - name: Run tests
+      run: cd test && go test -timeout 60m
+    # TODO(dtantsur): turn this into a normal artifact collection
+    - name: Show any local logs
+      run: cat test/*.log
+      if: always()
     - name: Show the deployment
       run: kubectl get -o yaml -n ironic-standalone-operator-system deployment/ironic-standalone-operator-controller-manager
       if: always()

test/functional.sh → test/prepare.sh

@@ -1,28 +1,9 @@
 #!/bin/bash
 
-set -x
+set -eux -o pipefail
 
 IMG="${IMG:-controller:test}"
 
-clean_up() {
-    cd "$(dirname $0)/.."
-    if [[ -z "${NOCLEAN:-}" ]]; then
-        make undeploy uninstall
-    fi
-}
-
-on_exit() {
-    EXIT=$?
-    set +e
-    clean_up
-    exit $EXIT
-}
-
-clean_up
-trap on_exit EXIT
-
-set -eu -o pipefail
-
 for image in ironic mariadb ironic-ipa-downloader; do
     docker pull "quay.io/metal3-io/${image}"
     kind load docker-image "quay.io/metal3-io/${image}"
@@ -34,4 +15,3 @@ make install deploy IMG="${IMG}"
 
 kubectl wait --for=condition=Available --timeout=60s \
     -n ironic-standalone-operator-system deployment/ironic-standalone-operator-controller-manager
-cd test && go test -timeout 60m

test/suite_test.go

@@ -20,6 +20,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"os"
 	"testing"
@@ -98,10 +99,34 @@ func WaitForIronic(name types.NamespacedName) *metal3api.Ironic {
 		if cond != nil && cond.Status == metav1.ConditionTrue {
 			return true
 		}
+		GinkgoWriter.Printf("Current status of Ironic: %+v\n", ironic.Status)
+
+		deployName := fmt.Sprintf("%s-service", name.Name)
+		if ironic.Spec.Distributed {
+			deploy, err := clientset.AppsV1().DaemonSets(name.Namespace).Get(ctx, deployName, metav1.GetOptions{})
+			if err == nil {
+				GinkgoWriter.Printf(".. status of daemon set: %+v\n", deploy.Status)
+			} else if !k8serrors.IsNotFound(err) {
+				Expect(err).NotTo(HaveOccurred())
+			}
+		} else {
+			deploy, err := clientset.AppsV1().Deployments(name.Namespace).Get(ctx, deployName, metav1.GetOptions{})
+			if err == nil {
+				GinkgoWriter.Printf(".. status of deployment: %+v\n", deploy.Status)
+			} else if !k8serrors.IsNotFound(err) {
+				Expect(err).NotTo(HaveOccurred())
+			}
+		}
+
+		pods, err := clientset.CoreV1().Pods(name.Namespace).List(ctx, metav1.ListOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		for _, pod := range pods.Items {
+			GinkgoWriter.Printf("... status of pod %s: %+v\n", pod.Name, pod.Status)
+		}
 
-		GinkgoWriter.Printf("Current status of Ironic: %+v\n", ironic)
 		return false
-	}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(BeTrue())
+	}).WithTimeout(5 * time.Minute).WithPolling(10 * time.Second).Should(BeTrue())
 
 	return ironic
 }
@@ -116,6 +141,30 @@ func VerifyIronic(ironic *metal3api.Ironic) {
 	Expect(err).NotTo(HaveOccurred())
 }
 
+func CollectLogs(namespace string) {
+	GinkgoHelper()
+
+	pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	for _, pod := range pods.Items {
+		for _, cont := range pod.Spec.Containers {
+			podLogOpts := corev1.PodLogOptions{Container: cont.Name}
+			req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts)
+			podLogs, err := req.Stream(ctx)
+			Expect(err).NotTo(HaveOccurred())
+			defer podLogs.Close()
+
+			logFile, err := os.Create(fmt.Sprintf("%s.%s.%s.log", namespace, pod.Name, cont.Name))
+			Expect(err).NotTo(HaveOccurred())
+			defer logFile.Close()
+
+			_, err = io.Copy(logFile, podLogs)
+			Expect(err).NotTo(HaveOccurred())
+		}
+	}
+}
+
 func DeleteAndWait(ironic *metal3api.Ironic) {
 	GinkgoHelper()
 
@@ -177,6 +226,46 @@ var _ = Describe("Ironic object tests", func() {
 		err := k8sClient.Create(ctx, ironic)
 		Expect(err).NotTo(HaveOccurred())
 		DeferCleanup(func() {
+			CollectLogs(namespace)
+			DeleteAndWait(ironic)
+		})
+
+		ironic = WaitForIronic(name)
+		VerifyIronic(ironic)
+	})
+
+	It("creates distributed Ironic", func() {
+		name := types.NamespacedName{
+			Name:      "test-ironic",
+			Namespace: namespace,
+		}
+
+		ironicDb := &metal3api.IronicDatabase{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-db", name.Name),
+				Namespace: name.Namespace,
+			},
+		}
+
+		err := k8sClient.Create(ctx, ironicDb)
+		Expect(err).NotTo(HaveOccurred())
+
+		ironic := &metal3api.Ironic{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name.Name,
+				Namespace: name.Namespace,
+			},
+			Spec: metal3api.IronicSpec{
+				DatabaseRef: corev1.LocalObjectReference{
+					Name: ironicDb.Name,
+				},
+				Distributed: true,
+			},
+		}
+		err = k8sClient.Create(ctx, ironic)
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(func() {
+			CollectLogs(namespace)
 			DeleteAndWait(ironic)
 		})