From 14f9cbe08a4931703f0553620ba88d8fe97c2608 Mon Sep 17 00:00:00 2001 From: Kevin Cross Date: Fri, 12 Aug 2022 14:43:37 +0100 Subject: [PATCH] Fixed ordering of credential binding and app start --- Makefile | 42 ++++++++++++++++--- ci/autoscaler/scripts/deploy-autoscaler.sh | 10 ++++- ci/autoscaler/scripts/run-acceptance-tests.sh | 2 +- scripts/pr-vars.source.sh | 8 +--- scripts/update | 20 ++++----- .../app/recurring_schedule_policy_test.go | 2 +- .../app/specificdate_schedule_policy_test.go | 1 + src/acceptance/assets/app/nodeApp/app.js | 30 ++++++------- src/acceptance/helpers/helpers.go | 4 +- src/acceptance/helpers/org_space.go | 6 ++- .../pre_upgrade/pre_custom_metric_test.go | 3 +- .../pre_upgrade/pre_dynamic_policy_test.go | 2 +- 12 files changed, 85 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index db86f3991b..1bf9b91c62 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,11 @@ DBURL := $(shell case "${db_type}" in\ MYSQL_TAG := 8 POSTGRES_TAG := 12 +ACCEPTANCE_SUITES?=broker api app +AUTOSCALER_DIR?=${PWD} +CI_DIR?=${PWD}/ci CI?=false + $(shell mkdir -p target) list-modules: @@ -190,12 +194,6 @@ integration: build init-db test-certs @echo " - using DBURL=${DBURL} OPTS=${OPTS}" make -C src/autoscaler integration DBURL="${DBURL}" OPTS="${OPTS}" -.PHONY: acceptance-tests -BBL_STATE_PATH ?= ../app-autoscaler-env-bbl-state/bbl-state -acceptance-tests: - @echo " - Running acceptance tests";\ - [ -d ${BBL_STATE_PATH} ] || { echo "Did not find bbl-state folder at ${BBL_STATE_PATH}, make sure you have checked out the app-autoscaler-env-bbl-state repository next to the app-autoscaler-release repository to run this target or indicate its location via BBL_STATE_PATH"; exit 1; };\ - BBL_STATE_PATH="${BBL_STATE_PATH}" AUTOSCALER_DIR="${PWD}" ./ci/autoscaler/scripts/run-acceptance-tests.sh .PHONY:lint $(addprefix lint_,$(go_modules)) lint: $(addprefix lint_,$(go_modules)) eslint rubocop @@ -250,3 +248,35 @@ fakes: workspace: [ -e go.work ] || go work init go work use $(addprefix ./src/,$(go_modules)) + +.PHONY: update +update: + ./scripts/update + +.PHONY: deploy +uaac: + which uaac || gem install cf-uaac + +DEPLOYMENT_NAME?=app-autoscaler-test +.PHONY: deploy +deploy: update uaac + cd scripts;\ + export DEPLOYMENT_NAME="${DEPLOYMENT_NAME}";\ + source pr-vars.source.sh;\ + ${CI_DIR}/autoscaler/scripts/deploy-autoscaler.sh;\ + ${CI_DIR}/autoscaler/scripts/register-broker.sh + +.PHONY: acceptance-tests +BBL_STATE_PATH ?= ../app-autoscaler-env-bbl-state/bbl-state +acceptance-tests: + @echo " - Running acceptance tests SUITES=${ACCEPTANCE_SUITES}" + [ -d ${BBL_STATE_PATH} ] || { echo "Did not find bbl-state folder at ${BBL_STATE_PATH}, make sure you have checked out the app-autoscaler-env-bbl-state repository next to the app-autoscaler-release repository to run this target or indicate its location via BBL_STATE_PATH"; exit 1; } + NAME_PREFIX="autoscaler-test"\ + BBL_STATE_PATH="${BBL_STATE_PATH}"\ + AUTOSCALER_DIR="${PWD}"\ + SUITES="${ACCEPTANCE_SUITES}"\ + SKIP_TEARDOWN=true\ + NODES=1\ + DEPLOYMENT_NAME="${DEPLOYMENT_NAME}"\ + GINKGO_OPTS="${OPTS}"\ + ./ci/autoscaler/scripts/run-acceptance-tests.sh diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 297cfda0f8..630607a5f1 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -9,7 +9,15 @@ bbl_state_path="${BBL_STATE_PATH:-bbl-state/bbl-state}" deployment_name="${DEPLOYMENT_NAME:-app-autoscaler}" autoscaler_dir="${AUTOSCALER_DIR:-app-autoscaler-release}" deployment_manifest="${script_dir}/../../../templates/app-autoscaler.yml" -ops_files="${OPS_FILES:-''}" +ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ + ${autoscaler_dir}/operations/instance-identity-cert-from-cf.yml\ + ${autoscaler_dir}/operations/add-postgres-variables.yml\ + ${autoscaler_dir}/operations/enable-nats-tls.yml\ + ${autoscaler_dir}/operations/loggregator-certs-from-cf.yml\ + ${autoscaler_dir}/operations/add-extra-plan.yml\ + ${autoscaler_dir}/operations/set-release-version.yml\ + ${autoscaler_dir}/operations/enable-log-cache.yml\ + ${autoscaler_dir}/operations/log-cache-syslog-server.yml"} CURRENT_COMMIT_HASH=$(cd "${autoscaler_dir}"; git log -1 --pretty=format:"%H") bosh_release_version=${RELEASE_VERSION:-${CURRENT_COMMIT_HASH}-${deployment_name}} diff --git a/ci/autoscaler/scripts/run-acceptance-tests.sh b/ci/autoscaler/scripts/run-acceptance-tests.sh index b2003c30fe..a93353ef98 100755 --- a/ci/autoscaler/scripts/run-acceptance-tests.sh +++ b/ci/autoscaler/scripts/run-acceptance-tests.sh @@ -53,7 +53,7 @@ done echo "Running $suites_to_run" if [ "${suites_to_run}" != "" ]; then - SKIP_TEARDOWN=$skip_teardown CONFIG=$PWD/acceptance_config.json ./bin/test "$gingko_opts" -race -nodes="${nodes}" --slow-spec-threshold=120s -trace "${suites_to_run}" + SKIP_TEARDOWN=$skip_teardown CONFIG=$PWD/acceptance_config.json ./bin/test -race -nodes="${nodes}" --slow-spec-threshold=120s -trace "$gingko_opts" "${suites_to_run}" else echo "Nothing to run!" fi diff --git a/scripts/pr-vars.source.sh b/scripts/pr-vars.source.sh index 43758f5f7e..68e627faa5 100644 --- a/scripts/pr-vars.source.sh +++ b/scripts/pr-vars.source.sh @@ -4,16 +4,12 @@ # shellcheck disable=SC2155 export script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" pr_number=${PR_NUMBER:-44} -export DEPLOYMENT_NAME="app-autoscaler-${pr_number}" -export SERVICE_BROKER_NAME="app-autoscaler-${pr_number}servicebroker" +export DEPLOYMENT_NAME=${DEPLOYMENT_NAME:-"app-autoscaler-${pr_number}"} +export SERVICE_BROKER_NAME="${DEPLOYMENT_NAME}servicebroker" export BBL_STATE_PATH="${script_dir}/../../app-autoscaler-env-bbl-state/bbl-state" export SYSTEM_DOMAIN="autoscaler.ci.cloudfoundry.org" export AUTOSCALER_DIR="${script_dir}/../" export CI_DIR="${script_dir}/../ci" -export OPS_FILES="${AUTOSCALER_DIR}/example/operation/loggregator-certs-from-cf.yml\ - ${AUTOSCALER_DIR}/example/operation/add-extra-plan.yml\ - ${AUTOSCALER_DIR}/example/operation/set-release-version.yml\ - ${AUTOSCALER_DIR}/example/operation/enable-name-based-deployments.yml" export SERVICE_OFFERING_ENABLED=true export SKIP_SSL_VALIDATION=true export NAME_PREFIX=${NAME_PREFIX:-"${DEPLOYMENT_NAME}-TESTS"} diff --git a/scripts/update b/scripts/update index b244289939..9e97202f00 100755 --- a/scripts/update +++ b/scripts/update @@ -2,25 +2,25 @@ set -euo pipefail -pushd src/autoscaler +pushd src/autoscaler > /dev/null go mod tidy go mod vendor -popd +popd > /dev/null -pushd src/changeloglockcleaner +pushd src/changeloglockcleaner > /dev/null go mod tidy go mod vendor -popd +popd > /dev/null -pushd src/acceptance +pushd src/acceptance > /dev/null go mod tidy go mod vendor -popd +popd > /dev/null -pushd src/db +pushd src/db > /dev/null mvn --no-transfer-progress package -DskipTests -popd +popd > /dev/null -pushd src/scheduler +pushd src/scheduler > /dev/null mvn --no-transfer-progress package -DskipTests -popd +popd > /dev/null diff --git a/src/acceptance/app/recurring_schedule_policy_test.go b/src/acceptance/app/recurring_schedule_policy_test.go index 6c50d91af7..5d7f8ceaf2 100644 --- a/src/acceptance/app/recurring_schedule_policy_test.go +++ b/src/acceptance/app/recurring_schedule_policy_test.go @@ -48,10 +48,10 @@ var _ = Describe("AutoScaler recurring schedule policy", func() { Context("when scaling by recurring schedule", func() { JustBeforeEach(func() { - StartApp(appName, cfg.CfPushTimeoutDuration()) startTime, endTime = getStartAndEndTime(time.UTC, 70*time.Second, time.Duration(interval+120)*time.Second) policy = GenerateDynamicAndRecurringSchedulePolicy(1, 4, 80, "UTC", startTime, endTime, daysOfMonthOrWeek, 2, 5, 3) instanceName = CreatePolicy(cfg, appName, appGUID, policy) + StartApp(appName, cfg.CfPushTimeoutDuration()) }) scaleDown := func() { diff --git a/src/acceptance/app/specificdate_schedule_policy_test.go b/src/acceptance/app/specificdate_schedule_policy_test.go index 41f9ce9aff..83fa4d3542 100644 --- a/src/acceptance/app/specificdate_schedule_policy_test.go +++ b/src/acceptance/app/specificdate_schedule_policy_test.go @@ -49,6 +49,7 @@ var _ = Describe("AutoScaler specific date schedule policy", func() { Context("when scaling by specific date schedule", func() { JustBeforeEach(func() { + //TODO the start app needs to be after the binding but the timings require the app been up already. StartApp(appName, cfg.CfPushTimeoutDuration()) startDateTime = time.Now().In(time.UTC).Add(1 * time.Minute) endDateTime = startDateTime.Add(time.Duration(interval+120) * time.Second) diff --git a/src/acceptance/assets/app/nodeApp/app.js b/src/acceptance/assets/app/nodeApp/app.js index 71d3a08502..c945db4a6f 100644 --- a/src/acceptance/assets/app/nodeApp/app.js +++ b/src/acceptance/assets/app/nodeApp/app.js @@ -18,32 +18,34 @@ function getCredentials () { // for service offering console.log('Getting credentials...') if (process.env.VCAP_SERVICES) { + console.log(` - found vcap looking for ${serviceName}`) const vcapServices = JSON.parse(process.env.VCAP_SERVICES) const service = vcapServices[serviceName] - - if (service && service[0] && service[0].credentials) { - credentials = service[0].credentials - metricsForwarderURL = credentials.custom_metrics.url - mfUsername = credentials.custom_metrics.username - mfPassword = credentials.custom_metrics.password - console.log('consumed VCAP_SERVICES env variable (service_offering)') + if (service && service[0]) { + console.log(` - found service ${serviceName}`) + if (service[0].credentials) { + console.log(' - found credentials') + credentials = service[0].credentials + metricsForwarderURL = credentials.custom_metrics.url + mfUsername = credentials.custom_metrics.username + mfPassword = credentials.custom_metrics.password + } else { + const err = 'ERROR: no credentials in bound service env variable' + console.error(err) + throw err + } } - } else { - const err = 'ERROR: no VCAP_SERVICES env variable' - console.error(err) - throw err } // for build-in offering if (metricsForwarderURL === '' || mfUsername === '' || mfPassword === '') { + console.log(' - looking for creds in env (built in case)') if (process.env.AUTO_SCALER_CUSTOM_METRIC_ENV) { + console.log(' - found credentials in AUTO_SCALER_CUSTOM_METRIC_ENV') credentials = JSON.parse(process.env.AUTO_SCALER_CUSTOM_METRIC_ENV) metricsForwarderURL = credentials.url mfUsername = credentials.username mfPassword = credentials.password - console.log( - 'consumed AUTO_SCALER_CUSTOM_METRIC_ENV env variable (use_buildin_mode)' - ) } else { console.error('ERROR: not all credentials were provided.') console.log( diff --git a/src/acceptance/helpers/helpers.go b/src/acceptance/helpers/helpers.go index de2334b016..4122b61d13 100644 --- a/src/acceptance/helpers/helpers.go +++ b/src/acceptance/helpers/helpers.go @@ -136,7 +136,7 @@ func CheckServiceExists(cfg *config.Config, spaceName, serviceName string) { Expect(spaceCmd).To(Exit(0), fmt.Sprintf("Space, %s, does not exist", spaceName)) spaceGuid := strings.TrimSpace(strings.Trim(string(spaceCmd.Out.Contents()), "\n")) - serviceCmd := cf.Cf("curl", "-f", ServicePlansUrl(cfg, spaceGuid)).Wait(cfg.DefaultTimeoutDuration()) + serviceCmd := cf.CfSilent("curl", "-f", ServicePlansUrl(cfg, spaceGuid)).Wait(cfg.DefaultTimeoutDuration()) if serviceCmd.ExitCode() != 0 { Fail(fmt.Sprintf("Failed get broker information for serviceName=%s spaceName=%s", cfg.ServiceName, spaceName)) } @@ -151,11 +151,13 @@ func CheckServiceExists(cfg *config.Config, spaceName, serviceName string) { if err != nil { AbortSuite(fmt.Sprintf("Failed to parse service plan json: %s\n\n'%s'", err.Error(), string(contents))) } + GinkgoWriter.Printf("\nFound services: %s\n", services.Included.ServiceOfferings) for _, service := range services.Included.ServiceOfferings { if service.Name == serviceName { return } } + cf.Cf("marketplace", "-e", cfg.ServiceName).Wait(cfg.DefaultTimeoutDuration()) Fail(fmt.Sprintf("Could not find service %s in space %s", serviceName, spaceName)) } diff --git a/src/acceptance/helpers/org_space.go b/src/acceptance/helpers/org_space.go index 84ef9a70a2..4127af8a10 100644 --- a/src/acceptance/helpers/org_space.go +++ b/src/acceptance/helpers/org_space.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" + "github.com/onsi/ginkgo/v2" + "github.com/KevinJCross/cf-test-helpers/v2/cf" . "github.com/onsi/gomega" @@ -33,7 +35,7 @@ type cfSpaces struct { } func GetTestOrgs(cfg *config.Config) []string { - rawOrgs := cf.Cf("curl", "/v3/organizations").Wait(cfg.DefaultTimeoutDuration()) + rawOrgs := cf.CfSilent("curl", "/v3/organizations").Wait(cfg.DefaultTimeoutDuration()) Expect(rawOrgs).To(Exit(0), "unable to get orgs") var orgs cfOrgs @@ -46,7 +48,7 @@ func GetTestOrgs(cfg *config.Config) []string { orgNames = append(orgNames, org.Name) } } - + ginkgo.GinkgoWriter.Printf("\nGot orgs: %s\n", orgNames) return orgNames } diff --git a/src/acceptance/pre_upgrade/pre_custom_metric_test.go b/src/acceptance/pre_upgrade/pre_custom_metric_test.go index 051a0aa3e6..8931ad5f12 100644 --- a/src/acceptance/pre_upgrade/pre_custom_metric_test.go +++ b/src/acceptance/pre_upgrade/pre_custom_metric_test.go @@ -19,10 +19,9 @@ var _ = Describe("AutoScaler custom metrics policy", func() { policy = helpers.GenerateDynamicScaleOutAndInPolicy(1, 2, "test_metric", 500, 500) appName = helpers.CreateTestApp(cfg, "node-custom-metric", 1) appGUID = helpers.GetAppGuid(cfg, appName) - helpers.StartApp(appName, cfg.CfPushTimeoutDuration()) _ = helpers.CreatePolicy(cfg, appName, appGUID, policy) helpers.CreateCustomMetricCred(cfg, appName, appGUID) - helpers.WaitForNInstancesRunning(appGUID, 1, cfg.DefaultTimeoutDuration()) + helpers.StartApp(appName, cfg.CfPushTimeoutDuration()) }) Context("when scaling by custom metrics", func() { diff --git a/src/acceptance/pre_upgrade/pre_dynamic_policy_test.go b/src/acceptance/pre_upgrade/pre_dynamic_policy_test.go index 1ac5ee30bf..40e1c9488b 100644 --- a/src/acceptance/pre_upgrade/pre_dynamic_policy_test.go +++ b/src/acceptance/pre_upgrade/pre_dynamic_policy_test.go @@ -19,8 +19,8 @@ var _ = Describe("AutoScaler dynamic policy", func() { JustBeforeEach(func() { appName = helpers.CreateTestApp(cfg, "nodeapp-cpu", initialInstanceCount) appGUID = helpers.GetAppGuid(cfg, appName) - helpers.StartApp(appName, cfg.CfPushTimeoutDuration()) _ = helpers.CreatePolicy(cfg, appName, appGUID, policy) + helpers.StartApp(appName, cfg.CfPushTimeoutDuration()) }) Context("when scaling by cpu", func() {