diff --git a/docs/examples/bosh-deployment/quarks-gora-termination.yaml b/docs/examples/bosh-deployment/quarks-gora-termination.yaml index ad9b5d22d..09db2f4a7 100644 --- a/docs/examples/bosh-deployment/quarks-gora-termination.yaml +++ b/docs/examples/bosh-deployment/quarks-gora-termination.yaml @@ -1,14 +1,3 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ops-scale -data: - ops: | - - type: replace - path: /instance_groups/name=quarks-gora?/instances - value: 2 ---- apiVersion: v1 kind: ConfigMap metadata: @@ -19,7 +8,7 @@ data: name: quarks-gora-deployment releases: - name: quarks-gora - version: "0.0.15" + version: "0.0.16" url: ghcr.io/cloudfoundry-incubator stemcell: os: SLE_15_SP1 @@ -31,8 +20,10 @@ data: bosh: agent: settings: - terminationGracePeriodSeconds: 900 + terminationGracePeriodSeconds: 70 jobs: + - name: loop-drain-job + release: quarks-gora - name: quarks-gora release: quarks-gora properties: @@ -53,6 +44,3 @@ spec: manifest: name: quarks-gora-manifest type: configmap - ops: - - name: ops-scale - type: configmap diff --git a/e2e/kube/terminationgraceperiod_test.go b/e2e/kube/terminationgraceperiod_test.go index 5d99d8972..097f4bdb2 100644 --- a/e2e/kube/terminationgraceperiod_test.go +++ b/e2e/kube/terminationgraceperiod_test.go @@ -2,6 +2,8 @@ package kube_test import ( "path" + "strconv" + "time" cmdHelper "code.cloudfoundry.org/quarks-utils/testing" @@ -9,8 +11,10 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("BOSHDeployment", func() { - When("specifying terminationGracePeriodSeconds in bosh.env", func() { +const graceTime = 70 // this has to match with quarks-gora-termination.yaml + +var _ = Describe("Instance group", func() { + When("specifying terminationGracePeriodSeconds in env.bosh.agent.settings", func() { BeforeEach(func() { By("Creating bdpl") f := path.Join(examplesDir, "bosh-deployment/quarks-gora-termination.yaml") @@ -20,16 +24,59 @@ var _ = Describe("BOSHDeployment", func() { By("Checking for pods") err = kubectl.Wait(namespace, "ready", "pod/quarks-gora-0", kubectl.PollTimeout) Expect(err).ToNot(HaveOccurred()) - err = kubectl.Wait(namespace, "ready", "pod/quarks-gora-1", kubectl.PollTimeout) - Expect(err).ToNot(HaveOccurred()) }) - It("should create containers with the requested resources/limits", func() { - for _, pod := range []string{"quarks-gora-0", "quarks-gora-1"} { - requestedGrace, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.spec.terminationGracePeriodSeconds}") + It("should delay drain script termination meanwhile the instancegroup is running", func() { + pod := "quarks-gora-0" + + uid, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.metadata.uid}") + Expect(err).ToNot(HaveOccurred()) + + // Make sure we set up sane defaults in the example and we don't change that accidentally. + // graceTime has to be > 30, plus additional 10 to avoid test flakyness. + // This test scenario is whenever we exceed Kubernetes default termination grace period which is 30s. + Expect(graceTime > 40).To(BeTrue()) + + requestedGrace, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.spec.terminationGracePeriodSeconds}") + Expect(err).ToNot(HaveOccurred()) + Expect(string(requestedGrace)).To(Equal(strconv.Itoa(graceTime))) + + go func() { + _ = kubectl.Delete("pod", pod, "-n", namespace) + }() + + isContainerUp := func() bool { + status, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.status.containerStatuses[?(@.name==\"quarks-gora-quarks-gora\")].state.running.startedAt}") + Expect(err).ToNot(HaveOccurred()) + return len(status) != 0 + } + + differentUID := func() bool { + uidCurrent, err := cmdHelper.GetData(namespace, "pods", pod, "jsonpath={.metadata.uid}") Expect(err).ToNot(HaveOccurred()) - Expect(string(requestedGrace)).To(Equal("900")) + return (string(uid) == string(uidCurrent)) + } + + isUp := func() bool { + e, _ := kubectl.Exists(namespace, "pod", pod) + if !e { + return false + } + + return differentUID() && isContainerUp() } + + // Make sure the gora container is up and running and it exceeds default 30s from Kubernetes. + // As we have setted up graceTime as terminationGrace and we expect the drain job to not terminate, the quarks-gora + // container should be up meanwhile the drain is running. We leave 10 extra seconds to avoid test flakyness + Consistently(isUp, time.Duration(time.Duration(graceTime-10)*time.Second), time.Duration(1*time.Second)).Should(BeTrue()) + + // Eventually, we should have a new pod as we exhaust the terminationGrace. Uses graceTime^2 as time window (not using pow for opt.) + Eventually(differentUID, time.Duration(time.Duration(graceTime*graceTime)*time.Second), time.Duration(1*time.Second)).Should(BeFalse()) + + err = kubectl.Wait(namespace, "ready", "pod/quarks-gora-0", kubectl.PollTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(isContainerUp()).To(BeTrue()) }) }) })