From 9ed17d4fcdc91795440625c073dfba4146243664 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <edigiacinto@suse.com>
Date: Fri, 6 Nov 2020 14:54:18 +0100
Subject: [PATCH] Add predrain terminationGrace e2e test

---
 .../quarks-gora-termination.yaml              | 20 ++----
 e2e/kube/terminationgraceperiod_test.go       | 63 ++++++++++++++++---
 2 files changed, 59 insertions(+), 24 deletions(-)

diff --git a/docs/examples/bosh-deployment/quarks-gora-termination.yaml b/docs/examples/bosh-deployment/quarks-gora-termination.yaml
index ad9b5d22d..09db2f4a7 100644
--- a/docs/examples/bosh-deployment/quarks-gora-termination.yaml
+++ b/docs/examples/bosh-deployment/quarks-gora-termination.yaml
@@ -1,14 +1,3 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ops-scale
-data:
-  ops: |
-    - type: replace
-      path: /instance_groups/name=quarks-gora?/instances
-      value: 2
----
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -19,7 +8,7 @@ data:
     name: quarks-gora-deployment
     releases:
     - name: quarks-gora
-      version: "0.0.15"
+      version: "0.0.16"
       url: ghcr.io/cloudfoundry-incubator
       stemcell:
         os: SLE_15_SP1
@@ -31,8 +20,10 @@ data:
         bosh:
           agent:
             settings: 
-              terminationGracePeriodSeconds: 900
+              terminationGracePeriodSeconds: 70
       jobs:
+      - name: loop-drain-job
+        release: quarks-gora
       - name: quarks-gora
         release: quarks-gora
         properties:
@@ -53,6 +44,3 @@ spec:
   manifest:
     name: quarks-gora-manifest
     type: configmap
-  ops:
-  - name: ops-scale
-    type: configmap
diff --git a/e2e/kube/terminationgraceperiod_test.go b/e2e/kube/terminationgraceperiod_test.go
index 5d99d8972..097f4bdb2 100644
--- a/e2e/kube/terminationgraceperiod_test.go
+++ b/e2e/kube/terminationgraceperiod_test.go
@@ -2,6 +2,8 @@ package kube_test
 
 import (
 	"path"
+	"strconv"
+	"time"
 
 	cmdHelper "code.cloudfoundry.org/quarks-utils/testing"
 
@@ -9,8 +11,10 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("BOSHDeployment", func() {
-	When("specifying terminationGracePeriodSeconds in bosh.env", func() {
+const graceTime = 70 // this has to match with quarks-gora-termination.yaml
+
+var _ = Describe("Instance group", func() {
+	When("specifying terminationGracePeriodSeconds in env.bosh.agent.settings", func() {
 		BeforeEach(func() {
 			By("Creating bdpl")
 			f := path.Join(examplesDir, "bosh-deployment/quarks-gora-termination.yaml")
@@ -20,16 +24,59 @@ var _ = Describe("BOSHDeployment", func() {
 			By("Checking for pods")
 			err = kubectl.Wait(namespace, "ready", "pod/quarks-gora-0", kubectl.PollTimeout)
 			Expect(err).ToNot(HaveOccurred())
-			err = kubectl.Wait(namespace, "ready", "pod/quarks-gora-1", kubectl.PollTimeout)
-			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("should create containers with the requested resources/limits", func() {
-			for _, pod := range []string{"quarks-gora-0", "quarks-gora-1"} {
-				requestedGrace, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.spec.terminationGracePeriodSeconds}")
+		It("should delay drain script termination meanwhile the instancegroup is running", func() {
+			pod := "quarks-gora-0"
+
+			uid, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.metadata.uid}")
+			Expect(err).ToNot(HaveOccurred())
+
+			// Make sure we set up sane defaults in the example and we don't change that accidentally.
+			// graceTime has to be > 30, plus additional 10 to avoid test flakyness.
+			// This test scenario is whenever we exceed Kubernetes default termination grace period which is 30s.
+			Expect(graceTime > 40).To(BeTrue())
+
+			requestedGrace, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.spec.terminationGracePeriodSeconds}")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(string(requestedGrace)).To(Equal(strconv.Itoa(graceTime)))
+
+			go func() {
+				_ = kubectl.Delete("pod", pod, "-n", namespace)
+			}()
+
+			isContainerUp := func() bool {
+				status, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.status.containerStatuses[?(@.name==\"quarks-gora-quarks-gora\")].state.running.startedAt}")
+				Expect(err).ToNot(HaveOccurred())
+				return len(status) != 0
+			}
+
+			differentUID := func() bool {
+				uidCurrent, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.metadata.uid}")
 				Expect(err).ToNot(HaveOccurred())
-				Expect(string(requestedGrace)).To(Equal("900"))
+				return (string(uid) == string(uidCurrent))
+			}
+
+			isUp := func() bool {
+				e, _ := kubectl.Exists(namespace, "pod", pod)
+				if !e {
+					return false
+				}
+
+				return differentUID() && isContainerUp()
 			}
+
+			// Make sure the gora container is up and running and it exceeds default 30s from Kubernetes.
+			// As we have setted up graceTime as terminationGrace and we expect the drain job to not terminate, the quarks-gora
+			// container should be up meanwhile the drain is running. We leave 10 extra seconds to avoid test flakyness
+			Consistently(isUp, time.Duration(time.Duration(graceTime-10)*time.Second), time.Duration(1*time.Second)).Should(BeTrue())
+
+			// Eventually, we should have a new pod as we exhaust the terminationGrace. Uses graceTime^2 as time window (not using pow for opt.)
+			Eventually(differentUID, time.Duration(time.Duration(graceTime*graceTime)*time.Second), time.Duration(1*time.Second)).Should(BeFalse())
+
+			err = kubectl.Wait(namespace, "ready", "pod/quarks-gora-0", kubectl.PollTimeout)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(isContainerUp()).To(BeTrue())
 		})
 	})
 })