diff --git a/.github/workflows/mysql.yaml b/.github/workflows/mysql.yaml index f19bb7766e..57a328100f 100644 --- a/.github/workflows/mysql.yaml +++ b/.github/workflows/mysql.yaml @@ -42,6 +42,7 @@ jobs: - uses: actions/checkout@v4 - name: make build run: | + make generate-openapi-generated-clients-and-servers make build db_type=mysql - name: make ${{ matrix.suite }} diff --git a/.github/workflows/postgres.yaml b/.github/workflows/postgres.yaml index 107b7db988..0987d596b9 100644 --- a/.github/workflows/postgres.yaml +++ b/.github/workflows/postgres.yaml @@ -48,6 +48,7 @@ jobs: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 run: | + make generate-openapi-generated-clients-and-servers make build - name: make ${{ matrix.suite }} diff --git a/.gitignore b/.gitignore index 033e998c9b..57fb3f6afb 100644 --- a/.gitignore +++ b/.gitignore @@ -44,7 +44,8 @@ src/acceptance/assets/app/go_app/internal/openapi-specs.bundled src/acceptance/assets/app/go_app/internal/applicationmetric/oas*gen.go src/acceptance/assets/app/go_app/internal/custommetrics/oas*gen.go src/acceptance/assets/app/go_app/internal/policy/oas*gen.go -src/autoscaler/helpers/apis/scalinghistory/oas*gen.go +src/autoscaler/api/apis/scalinghistory/oas*gen.go +src/autoscaler/scalingengine/apis/scalinghistory/oas*gen.go # https://go.dev/ref/mod#go-work-file go.work* diff --git a/Makefile b/Makefile index 61598ce5c0..abb06d282c 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ clean-acceptance: build: $(all_modules) build-tests: build-test build-test: $(addprefix test_,$(go_modules)) -build-all: build build-test build-test-app mta-build ## Build all modules and tests +build-all: generate-openapi-generated-clients-and-servers build build-test build-test-app mta-build ## Build all modules and tests db: target/db target/db: @echo "# building $@" diff --git a/api/internal-scaling-history-api.openapi.yaml b/api/internal-scaling-history-api.openapi.yaml new file mode 100644 index 0000000000..ee6614de43 --- /dev/null +++ b/api/internal-scaling-history-api.openapi.yaml @@ -0,0 +1,213 @@ +openapi: 3.0.0 +info: + title: Scaling History API + description: List scaling history of an application + version: 1.0.0 + license: + name: "Apache License Version 2.0" + # identifier: "Apache-2.0" # Requires at least OpenAPI 3.1.0 + url: "http://www.apache.org/licenses/LICENSE-2.0.html" +tags: +- name: Scaling History API V1 + description: List the scaling history of an Application +paths: + /v1/apps/{guid}/scaling_histories: + parameters: + - name: guid + in: path + required: true + description: | + The GUID identifying the application for which the scaling history is fetched. + + It can be found in the `application_id` property of the JSON object stored in the + `VCAP_APPLICATION` environment variable. + schema: + $ref: "./shared_definitions.yaml#/schemas/GUID" + - name: start-time + in: query + description: | + The start time in the number of nanoseconds elapsed since January 1, 1970 UTC. + schema: + type: integer + default: 0 + example: start-time=1494989539138350432 + - name: end-time + in: query + description: | + The end time in the number of nanoseconds elapsed since January 1, 1970 UTC. + schema: + type: integer + default: -1 + example: end-time=1494989549117047288 + - name: order-direction + in: query + description: | + The sorting order. The scaling history will be order by timestamp ascending or descending. + schema: + type: string + enum: ["asc", "desc"] + default: desc + example: order-direction=desc + - name: order + in: query + description: | + Deprecated: Use order-direction instead. + schema: + type: string + enum: ["asc", "desc"] + deprecated: true + example: order=desc + - name: page + in: query + description: The page number to query + schema: + type: integer + minimum: 1 + default: 1 + example: page=1 + - name: results-per-page + in: query + description: Number of entries shown per page. + schema: + type: integer + minimum: 0 + default: 50 + example: results-per-page=10 + get: + summary: Retrieves the scaling history of an application. + description: | + Use to retrieve scaling history for an app. + tags: + - Scaling History API V1 + responses: + "200": + description: "OK" + content: + application/json: + schema: + $ref: "#/components/schemas/History" + default: + $ref: "./shared_definitions.yaml#/responses/Error" + security: [] + x-codegen-request-body-name: body +components: + schemas: + History: + description: Object containing scaling history. + type: object + properties: + total_results: + type: integer + format: int64 + description: Number of history entries found for the given query. + example: 2 + total_pages: + type: integer + format: int64 + description: Number of Pages from the query + example: 1 + page: + type: integer + format: int64 + description: Number of the current page. + example: 1 + prev_url: + type: string + format: uri + next_url: + type: string + format: uri + resources: + type: array + items: + $ref: '#/components/schemas/HistoryEntry' + HistoryEntry: + description: "Properties common for each entry in the scaling history." + type: object + oneOf: + - $ref: "#/components/schemas/HistoryErrorEntry" + - $ref: "#/components/schemas/HistoryIgnoreEntry" + - $ref: "#/components/schemas/HistorySuccessEntry" +# Unfortunately, we cannot use a discriminator here, as the property MUST be a string, see also https://github.com/OAI/OpenAPI-Specification/issues/2731 +# discriminator: +# propertyName: status +# mapping: +# 0: "#/components/schemas/HistorySuccessEntry" +# 1: "#/components/schemas/HistoryErrorEntry" +# 2: "#/components/schemas/HistoryIgnoreEntry" + properties: + status: + type: integer + format: int64 + enum: [0, 1, 2] + description: | + Following stati are possible: + + 0: The scaling was done successfully. + + 1: The scaling failed explicitly. + + 2: The scaling was ignored. + This field is as well a selector of which of the other ones are used and which not. + example: 0 + app_id: + $ref: "./shared_definitions.yaml#/schemas/GUID" + timestamp: + type: integer + description: | + The scaling time in the number of nanoseconds elapsed since January 1, 1970 UTC. + example: 1494989539138350432 + scaling_type: + type: integer + format: int64 + enum: [0, 1] + description: | + There are two different scaling types: + + 0: This represents `ScalingTypeDynamic`. The scaling has been done due to a dynamic + scaling rule, reacting on metrics provided by the app. + + 1: This represents `ScalingTypeSchedule`. The scaling has been done due to a + scheduled period changing the default instance limits. + example: 0 + old_instances: + type: integer + format: int64 + minimum: -1 + description: The number of instances before the scaling. -1 means that the value is not applicable. + example: 1 + new_instances: + type: integer + format: int64 + minimum: -1 + description: The number of instances after the scaling. -1 means that the value is not applicable. + example: 2 + reason: + type: string + description: Textual information about what triggered the scaling event. + example: -1 instance(s) because cpu < 20% for 60 seconds + message: + type: string + description: Textual information about the scaling event. + example: app + HistoryErrorEntry: + description: Description of a failed scaling even in history. + type: object + properties: + error: + type: string + description: | + In case the scaling failed, the reason is provided in this field. + example: failed to compute new app instances + HistoryIgnoreEntry: + description: Description of an ignored scaling event in history. + type: object + properties: + ignore_reason: + type: string + description: | + In case the scaling was ignored, the reason is provided in this field. + example: app in cooldown period + HistorySuccessEntry: + description: Description of a successful scaling event event in history. + type: object + properties: {} # No extra fields needed in this variant. + securitySchemes: + basicAuth: + type: http + scheme: basic diff --git a/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl b/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl new file mode 100644 index 0000000000..2c25984c38 --- /dev/null +++ b/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl @@ -0,0 +1,13 @@ +--- +admin_password: ((/bosh-autoscaler/cf/cf_admin_password)) +routing_api_tls_client_cert: ((/bosh-autoscaler/cf/routing_api_tls_client.certificate)) +routing_api_ca_certs: ((/bosh-autoscaler/cf/router_ssl.ca)) +routing_api_client_secret: ((/bosh-autoscaler/cf/uaa_clients_routing_api_client_secret)) +routing_api_tls_client_private_key: ((/bosh-autoscaler/cf/routing_api_tls_client.private_key)) +routing_api_server_ca_cert: ((/bosh-autoscaler/cf/router_ssl.ca)) +log_cache_syslog_tls_ca: ((/bosh-autoscaler/cf/log_cache_syslog_tls.ca)) +syslog_agent_log_cache_tls_certificate: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.certificate)) +syslog_agent_log_cache_tls_key: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.private_key)) +metricscollector_ca_cert: ((/bosh-autoscaler/cf/log_cache.ca)) +metricscollector_client_cert: ((/bosh-autoscaler/cf/log_cache.certificate)) +metricscollector_client_key: ((/bosh-autoscaler/cf/log_cache.private_key)) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 7cdbc0c6c8..8c087616f6 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -17,8 +17,8 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/add-extra-plan.yml\ ${autoscaler_dir}/operations/set-release-version.yml\ ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\ - ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} - + ${autoscaler_dir}/operations/enable-scheduler-logging.yml\ + ${autoscaler_dir}/operations/use-cf-services.yml"} case "${cpu_upper_threshold}" in @@ -89,6 +89,9 @@ function create_manifest(){ # on MacOS mktemp does not know the --tmpdir option tmp_manifest_file="$(mktemp "${tmp_dir}/${deployment_name}.bosh-manifest.yaml.XXX")" + + credhub interpolate -f "${autoscaler_dir}/ci/autoscaler/scripts/autoscaler-secrets.yml.tpl" > /tmp/autoscaler-secrets.yml + bosh -n -d "${deployment_name}" \ interpolate "${deployment_manifest}" \ ${OPS_FILES_TO_USE} \ @@ -96,22 +99,10 @@ function create_manifest(){ -v system_domain="${system_domain}" \ -v deployment_name="${deployment_name}" \ -v app_autoscaler_version="${bosh_release_version}" \ - -v admin_password="$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q)"\ - -v routing_api_ca_certs="$(credhub get -n /bosh-autoscaler/cf/router_ssl --key ca --quiet)"\ - -v routing_api_client_secret="$(credhub get -n /bosh-autoscaler/cf/uaa_clients_routing_api_client_secret --quiet)"\ - -v routing_api_tls_client_cert="$(credhub get -n /bosh-autoscaler/cf/routing_api_tls_client --key certificate --quiet)"\ - -v routing_api_tls_client_private_key="$(credhub get -n /bosh-autoscaler/cf/routing_api_tls_client --key private_key --quiet)"\ - -v routing_api_server_ca_cert="$(credhub get -n /bosh-autoscaler/cf/router_ssl --key ca --quiet)"\ -v cf_client_id=autoscaler_client_id \ -v cf_client_secret=autoscaler_client_secret \ - -v log_cache_syslog_tls_ca="$(credhub get -n /bosh-autoscaler/cf/log_cache_syslog_tls --key ca --quiet)"\ - -v syslog_agent_log_cache_tls_certificate="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)"\ - -v syslog_agent_log_cache_tls_key="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)"\ - -v metricscollector_ca_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key ca --quiet)"\ - -v metricscollector_client_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key certificate --quiet)"\ - -v metricsforwarder_host="${metricsforwarder_host}"\ -v postgres_external_port="$(get_postgres_external_port)"\ - -v metricscollector_client_key="$(credhub get -n /bosh-autoscaler/cf/log_cache --key private_key --quiet)"\ + --vars-file=/tmp/autoscaler-secrets.yml \ -v skip_ssl_validation=true \ > "${tmp_manifest_file}" @@ -134,9 +125,9 @@ function check_ops_files(){ function deploy() { # Try to silence Prometheus but do not fail deployment if there's an error - ${script_dir}/silence_prometheus_alert.sh "BOSHJobEphemeralDiskPredictWillFill" || true - ${script_dir}/silence_prometheus_alert.sh "BOSHJobProcessUnhealthy" || true - ${script_dir}/silence_prometheus_alert.sh "BOSHJobUnhealthy" || true +# ${script_dir}/silence_prometheus_alert.sh "BOSHJobEphemeralDiskPredictWillFill" || true +# ${script_dir}/silence_prometheus_alert.sh "BOSHJobProcessUnhealthy" || true +# ${script_dir}/silence_prometheus_alert.sh "BOSHJobUnhealthy" || true create_manifest diff --git a/ci/autoscaler/scripts/release-autoscaler.sh b/ci/autoscaler/scripts/release-autoscaler.sh index eb88e94ee0..7f12236ba0 100755 --- a/ci/autoscaler/scripts/release-autoscaler.sh +++ b/ci/autoscaler/scripts/release-autoscaler.sh @@ -33,7 +33,11 @@ function create_release() { echo " - creating release '${version}' in '${build_path}' as ${release_file}" yq eval -i ".properties.\"autoscaler.apiserver.info.build\".default = \"${version}\"" jobs/golangapiserver/spec + yq eval -i ".build = \"${version}\"" src/autoscaler/api/default_info.json + git add jobs/golangapiserver/spec + git add src/autoscaler/api/default_info.json + [ "${CI}" = "true" ] && git commit -S -m "Updated release version to ${version} in golangapiserver" # shellcheck disable=SC2086 diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index b7b925a2ac..b1f58cfeb7 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -67,17 +67,6 @@ export SYSTEM_DOMAIN="${SYSTEM_DOMAIN:-"autoscaler.app-runtime-interfaces.ci.clo debug "SYSTEM_DOMAIN: ${SYSTEM_DOMAIN}" system_domain="${SYSTEM_DOMAIN}" -# Configure cloudfoundry app variables -export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}" -debug "METRICSFORWARDER_APPNAME: ${METRICSFORWARDER_APPNAME}" -log "set up vars: METRICSFORWRDER_APPNAME=${METRICSFORWARDER_APPNAME}" -metricsforwarder_appname="${METRICSFORWARDER_APPNAME}" - -export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"${METRICSFORWARDER_APPNAME}.${SYSTEM_DOMAIN}"}" -debug "METRICSFORWARDER_HOST: ${METRICSFORWARDER_HOST}" -log "set up vars: METRICSFORWARDER_HOST=${METRICSFORWARDER_HOST}" -metricsforwarder_host="${METRICSFORWARDER_HOST}" - BBL_STATE_PATH="${BBL_STATE_PATH:-$( realpath -e "${root_dir}/../app-autoscaler-env-bbl-state/bbl-state" 2> /dev/null || echo "${root_dir}/../bbl-state/bbl-state" )}" BBL_STATE_PATH="$(realpath -e "${BBL_STATE_PATH}" || echo "ERR_invalid_state_path" )" export BBL_STATE_PATH diff --git a/devbox.json b/devbox.json index 1dc861fecd..feb5ae44a5 100644 --- a/devbox.json +++ b/devbox.json @@ -9,6 +9,7 @@ "bundix": "latest", "coreutils": "latest", "delve": "latest", + "direnv": "2.34.0", "fly": "7.10.0", "gh": "latest", "gnumake": "4.4", @@ -35,12 +36,11 @@ "ginkgo": "2.20.2", "bundler": "latest", "golangci-lint": "1.61.0", - "actionlint": "1.7.3", - "go": "1.22.5", "act": "0.2.67", + "actionlint": "latest", "cloudfoundry-cli": "8.8.1", - "direnv": "2.35.0", "google-cloud-sdk": "latest", + "go": "1.22.3", "temurin-bin-21": "latest", "ruby": "latest" }, diff --git a/devbox.lock b/devbox.lock index 5412f9bd2d..a0d42be71f 100644 --- a/devbox.lock +++ b/devbox.lock @@ -49,51 +49,51 @@ } } }, - "actionlint@1.7.3": { - "last_modified": "2024-10-06T12:21:13Z", - "resolved": "github:NixOS/nixpkgs/50b3bd3fed0442bcbf7f58355e990da84af1749d#actionlint", + "actionlint@latest": { + "last_modified": "2024-09-24T10:20:15Z", + "resolved": "github:NixOS/nixpkgs/965289e5e07243f1cde3212d8bcaf726d36c5c46#actionlint", "source": "devbox-search", - "version": "1.7.3", + "version": "1.7.2", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/33rbqx21gcp3imbrr2i0ykhz7x9ixsj5-actionlint-1.7.3", + "path": "/nix/store/qv1fnp2ifk37jlr1lbzf5r63vr3q6k41-actionlint-1.7.2", "default": true } ], - "store_path": "/nix/store/33rbqx21gcp3imbrr2i0ykhz7x9ixsj5-actionlint-1.7.3" + "store_path": "/nix/store/qv1fnp2ifk37jlr1lbzf5r63vr3q6k41-actionlint-1.7.2" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/xa1zzhm3szslprnknblyhvp4lbagk6nx-actionlint-1.7.3", + "path": "/nix/store/nw0xpacz21b9rszn4k6ay452pf8amf1p-actionlint-1.7.2", "default": true } ], - "store_path": "/nix/store/xa1zzhm3szslprnknblyhvp4lbagk6nx-actionlint-1.7.3" + "store_path": "/nix/store/nw0xpacz21b9rszn4k6ay452pf8amf1p-actionlint-1.7.2" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/z37rjxbvy5z9pk0vbimw2gw2rqdxjjc3-actionlint-1.7.3", + "path": "/nix/store/rh6f40c82jp5z3irxsq6lfapg6ckc9br-actionlint-1.7.2", "default": true } ], - "store_path": "/nix/store/z37rjxbvy5z9pk0vbimw2gw2rqdxjjc3-actionlint-1.7.3" + "store_path": "/nix/store/rh6f40c82jp5z3irxsq6lfapg6ckc9br-actionlint-1.7.2" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/48asng2d6lcma3rblhq56zwzbp7c7hbf-actionlint-1.7.3", + "path": "/nix/store/ggpghzgdd1cv04hh93cmqjb7mm3zqc7m-actionlint-1.7.2", "default": true } ], - "store_path": "/nix/store/48asng2d6lcma3rblhq56zwzbp7c7hbf-actionlint-1.7.3" + "store_path": "/nix/store/ggpghzgdd1cv04hh93cmqjb7mm3zqc7m-actionlint-1.7.2" } } }, @@ -457,51 +457,51 @@ } } }, - "direnv@2.35.0": { - "last_modified": "2024-10-08T08:54:18Z", - "resolved": "github:NixOS/nixpkgs/a2eacc0c62c0537bd1a7a60c1f91d1f3a59fd013#direnv", + "direnv@2.34.0": { + "last_modified": "2024-08-31T10:12:23Z", + "resolved": "github:NixOS/nixpkgs/5629520edecb69630a3f4d17d3d33fc96c13f6fe#direnv", "source": "devbox-search", - "version": "2.35.0", + "version": "2.34.0", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/b862impi4vy6hx7irfhnp3y5padbw0jl-direnv-2.35.0", + "path": "/nix/store/439f5yi8i1akxl2669f5mam6iacisycv-direnv-2.34.0", "default": true } ], - "store_path": "/nix/store/b862impi4vy6hx7irfhnp3y5padbw0jl-direnv-2.35.0" + "store_path": "/nix/store/439f5yi8i1akxl2669f5mam6iacisycv-direnv-2.34.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/z2kgy6bcdn3h0rplawv2nj09hmffknhx-direnv-2.35.0", + "path": "/nix/store/1fj0mbpggzc91ykg1n5q86gd5ih7azws-direnv-2.34.0", "default": true } ], - "store_path": "/nix/store/z2kgy6bcdn3h0rplawv2nj09hmffknhx-direnv-2.35.0" + "store_path": "/nix/store/1fj0mbpggzc91ykg1n5q86gd5ih7azws-direnv-2.34.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/djl7k7rvngmjr8pbhgmzhccv82hlaxms-direnv-2.35.0", + "path": "/nix/store/v05bzbn98fq9qg6fpr5znpkqxgyx0xx4-direnv-2.34.0", "default": true } ], - "store_path": "/nix/store/djl7k7rvngmjr8pbhgmzhccv82hlaxms-direnv-2.35.0" + "store_path": "/nix/store/v05bzbn98fq9qg6fpr5znpkqxgyx0xx4-direnv-2.34.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/0cc1cyiifsb6g6vr2271448va439q73b-direnv-2.35.0", + "path": "/nix/store/ml0yghnf9h5rb4dmv1brkfi60ils8gz6-direnv-2.34.0", "default": true } ], - "store_path": "/nix/store/0cc1cyiifsb6g6vr2271448va439q73b-direnv-2.35.0" + "store_path": "/nix/store/ml0yghnf9h5rb4dmv1brkfi60ils8gz6-direnv-2.34.0" } } }, @@ -853,51 +853,51 @@ } } }, - "go@1.22.5": { - "last_modified": "2024-08-14T11:41:26Z", - "resolved": "github:NixOS/nixpkgs/0cb2fd7c59fed0cd82ef858cbcbdb552b9a33465#go", + "go@1.22.3": { + "last_modified": "2024-06-12T20:55:33Z", + "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#go", "source": "devbox-search", - "version": "1.22.5", + "version": "1.22.3", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/p2i1kd6n12qj3s8kx65l3199mmjcffwz-go-1.22.5", + "path": "/nix/store/m4ip3yyqg4k2pilbyfczgiy8xqvh4554-go-1.22.3", "default": true } ], - "store_path": "/nix/store/p2i1kd6n12qj3s8kx65l3199mmjcffwz-go-1.22.5" + "store_path": "/nix/store/m4ip3yyqg4k2pilbyfczgiy8xqvh4554-go-1.22.3" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/xvwr7mp9yqafl8pclayhrhdcxkszaf6d-go-1.22.5", + "path": "/nix/store/iz2bp36nbgp2l7day14186nmw2v4v0qg-go-1.22.3", "default": true } ], - "store_path": "/nix/store/xvwr7mp9yqafl8pclayhrhdcxkszaf6d-go-1.22.5" + "store_path": "/nix/store/iz2bp36nbgp2l7day14186nmw2v4v0qg-go-1.22.3" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/vkrmfnargqkzg5amlfl4yh8vgxja7pli-go-1.22.5", + "path": "/nix/store/gl1vr9lznqg5zv2f0bvh0q469k82mjz0-go-1.22.3", "default": true } ], - "store_path": "/nix/store/vkrmfnargqkzg5amlfl4yh8vgxja7pli-go-1.22.5" + "store_path": "/nix/store/gl1vr9lznqg5zv2f0bvh0q469k82mjz0-go-1.22.3" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/4ay992wzksf59aapkkh5lflv4rkbmdjy-go-1.22.5", + "path": "/nix/store/vz8d6wmfcf38l3h3vymwqr6c5zxp5jmp-go-1.22.3", "default": true } ], - "store_path": "/nix/store/4ay992wzksf59aapkkh5lflv4rkbmdjy-go-1.22.5" + "store_path": "/nix/store/vz8d6wmfcf38l3h3vymwqr6c5zxp5jmp-go-1.22.3" } } }, @@ -950,50 +950,50 @@ } }, "google-cloud-sdk@latest": { - "last_modified": "2024-10-01T01:43:25Z", - "resolved": "github:NixOS/nixpkgs/9682b2197dabc185fcca802ac1ac21136e48fcc2#google-cloud-sdk", + "last_modified": "2024-09-16T10:17:16Z", + "resolved": "github:NixOS/nixpkgs/20f9370d5f588fb8c72e844c54511cab054b5f40#google-cloud-sdk", "source": "devbox-search", - "version": "494.0.0", + "version": "492.0.0", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/vj189b7g7a47m410n7i5218i339zrv1r-google-cloud-sdk-494.0.0", + "path": "/nix/store/j9baimnbg147szh8ws2hc1hmkb423riq-google-cloud-sdk-492.0.0", "default": true } ], - "store_path": "/nix/store/vj189b7g7a47m410n7i5218i339zrv1r-google-cloud-sdk-494.0.0" + "store_path": "/nix/store/j9baimnbg147szh8ws2hc1hmkb423riq-google-cloud-sdk-492.0.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/fcy6s62xsbgnzdmg42wsd84a9066hf8r-google-cloud-sdk-494.0.0", + "path": "/nix/store/r6dgpw8nqg3l8x78i527hb2slz7chk8g-google-cloud-sdk-492.0.0", "default": true } ], - "store_path": "/nix/store/fcy6s62xsbgnzdmg42wsd84a9066hf8r-google-cloud-sdk-494.0.0" + "store_path": "/nix/store/r6dgpw8nqg3l8x78i527hb2slz7chk8g-google-cloud-sdk-492.0.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/0k1wcz2h1pixyas5acpxpcvrlfr178m7-google-cloud-sdk-494.0.0", + "path": "/nix/store/5pywqc481xpqgxn89lzxa1lf0p2h13zv-google-cloud-sdk-492.0.0", "default": true } ], - "store_path": "/nix/store/0k1wcz2h1pixyas5acpxpcvrlfr178m7-google-cloud-sdk-494.0.0" + "store_path": "/nix/store/5pywqc481xpqgxn89lzxa1lf0p2h13zv-google-cloud-sdk-492.0.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/h5xqqp2y3cp3nkf2jy2ysc1cyapzq7mf-google-cloud-sdk-494.0.0", + "path": "/nix/store/dqpw1myxbj5zk0f8w0lsfzqq98g23hj9-google-cloud-sdk-492.0.0", "default": true } ], - "store_path": "/nix/store/h5xqqp2y3cp3nkf2jy2ysc1cyapzq7mf-google-cloud-sdk-494.0.0" + "store_path": "/nix/store/dqpw1myxbj5zk0f8w0lsfzqq98g23hj9-google-cloud-sdk-492.0.0" } } }, diff --git a/jobs/eventgenerator/templates/eventgenerator.yml.erb b/jobs/eventgenerator/templates/eventgenerator.yml.erb index 2b302a7115..46bcae0ef5 100644 --- a/jobs/eventgenerator/templates/eventgenerator.yml.erb +++ b/jobs/eventgenerator/templates/eventgenerator.yml.erb @@ -69,15 +69,17 @@ logging: level: <%= p("autoscaler.eventgenerator.logging.level") %> http_client_timeout: <%= p("autoscaler.eventgenerator.http_client_timeout") %> health: - port: <%= p("autoscaler.eventgenerator.health.port") %> - username: <%= p("autoscaler.eventgenerator.health.username") %> - password: <%= p("autoscaler.eventgenerator.health.password") %> - <% if_p("autoscaler.eventgenerator.health.ca_cert", "autoscaler.eventgenerator.health.server_cert", "autoscaler.eventgenerator.health.server_key") do %> - tls: - ca_file: /var/vcap/jobs/eventgenerator/config/certs/healthendpoint/ca.crt - cert_file: /var/vcap/jobs/eventgenerator/config/certs/healthendpoint/server.crt - key_file: /var/vcap/jobs/eventgenerator/config/certs/healthendpoint/server.key - <% end %> + server_config: + port: <%= p("autoscaler.eventgenerator.health.port") %> + <% if_p("autoscaler.eventgenerator.health.ca_cert", "autoscaler.eventgenerator.health.server_cert", "autoscaler.eventgenerator.health.server_key") do %> + tls: + ca_file: /var/vcap/jobs/eventgenerator/config/certs/healthendpoint/ca.crt + cert_file: /var/vcap/jobs/eventgenerator/config/certs/healthendpoint/server.crt + key_file: /var/vcap/jobs/eventgenerator/config/certs/healthendpoint/server.key + <% end %> + basic_auth: + username: <%= p("autoscaler.eventgenerator.health.username") %> + password: <%= p("autoscaler.eventgenerator.health.password") %> db: policy_db: diff --git a/jobs/golangapiserver/templates/apiserver.yml.erb b/jobs/golangapiserver/templates/apiserver.yml.erb index ece70ac31d..629bb34b15 100644 --- a/jobs/golangapiserver/templates/apiserver.yml.erb +++ b/jobs/golangapiserver/templates/apiserver.yml.erb @@ -91,13 +91,14 @@ dashboard_redirect_uri: <%= p("autoscaler.apiserver.broker.server.dashboard_redi default_credential_type: <%= p("autoscaler.apiserver.broker.default_credential_type") %> health: - port: <%= p("autoscaler.apiserver.health.port") %> - <% if_p("autoscaler.apiserver.health.ca_cert", "autoscaler.apiserver.health.server_cert", "autoscaler.apiserver.health.server_key") do %> - tls: - ca_file: /var/vcap/jobs/golangapiserver/config/certs/healthendpoint/ca.crt - cert_file: /var/vcap/jobs/golangapiserver/config/certs/healthendpoint/server.crt - key_file: /var/vcap/jobs/golangapiserver/config/certs/healthendpoint/server.key - <% end %> + server_config: + port: <%= p("autoscaler.apiserver.health.port") %> + <% if_p("autoscaler.apiserver.health.ca_cert", "autoscaler.apiserver.health.server_cert", "autoscaler.apiserver.health.server_key") do %> + tls: + ca_file: /var/vcap/jobs/golangapiserver/config/certs/healthendpoint/ca.crt + cert_file: /var/vcap/jobs/golangapiserver/config/certs/healthendpoint/server.crt + key_file: /var/vcap/jobs/golangapiserver/config/certs/healthendpoint/server.key + <% end %> db: policy_db: diff --git a/jobs/metricsforwarder/templates/metricsforwarder.yml.erb b/jobs/metricsforwarder/templates/metricsforwarder.yml.erb index e518676955..e6713c8643 100644 --- a/jobs/metricsforwarder/templates/metricsforwarder.yml.erb +++ b/jobs/metricsforwarder/templates/metricsforwarder.yml.erb @@ -91,15 +91,17 @@ cache_ttl: <%= p("autoscaler.metricsforwarder.cache_ttl") %> cache_cleanup_interval: <%= p("autoscaler.metricsforwarder.cache_cleanup_interval") %> policy_poller_interval: <%= p("autoscaler.metricsforwarder.policy_poller_interval") %> health: - port: <%= p("autoscaler.metricsforwarder.health.port") %> - username: <%= p("autoscaler.metricsforwarder.health.username") %> - password: <%= p("autoscaler.metricsforwarder.health.password") %> - <% if_p("autoscaler.metricsforwarder.health.ca_cert", "autoscaler.metricsforwarder.health.server_cert", "autoscaler.metricsforwarder.health.server_key") do %> - tls: - ca_file: /var/vcap/jobs/metricsforwarder/config/certs/healthendpoint/ca.crt - cert_file: /var/vcap/jobs/metricsforwarder/config/certs/healthendpoint/server.crt - key_file: /var/vcap/jobs/metricsforwarder/config/certs/healthendpoint/server.key - <% end %> + server_config: + port: <%= p("autoscaler.metricsforwarder.health.port") %> + <% if_p("autoscaler.metricsforwarder.health.ca_cert", "autoscaler.metricsforwarder.health.server_cert", "autoscaler.metricsforwarder.health.server_key") do %> + tls: + ca_file: /var/vcap/jobs/metricsforwarder/config/certs/healthendpoint/ca.crt + cert_file: /var/vcap/jobs/metricsforwarder/config/certs/healthendpoint/server.crt + key_file: /var/vcap/jobs/metricsforwarder/config/certs/healthendpoint/server.key + <% end %> + basic_auth: + username: <%= p("autoscaler.metricsforwarder.health.username") %> + password: <%= p("autoscaler.metricsforwarder.health.password") %> rate_limit: valid_duration: <%= p("autoscaler.metricsforwarder.rate_limit.valid_duration") %> diff --git a/jobs/operator/templates/operator.yml.erb b/jobs/operator/templates/operator.yml.erb index c96a6890a0..83b53699d9 100644 --- a/jobs/operator/templates/operator.yml.erb +++ b/jobs/operator/templates/operator.yml.erb @@ -58,15 +58,17 @@ cf: logging: level: <%= p("autoscaler.operator.logging.level") %> health: - port: <%= p("autoscaler.operator.health.port") %> - username: <%= p("autoscaler.operator.health.username") %> - password: <%= p("autoscaler.operator.health.password") %> - <% if_p("autoscaler.operator.health.ca_cert", "autoscaler.operator.health.server_cert", "autoscaler.operator.health.server_key") do %> - tls: - ca_file: /var/vcap/jobs/operator/config/certs/healthendpoint/ca.crt - cert_file: /var/vcap/jobs/operator/config/certs/healthendpoint/server.crt - key_file: /var/vcap/jobs/operator/config/certs/healthendpoint/server.key - <% end %> + server_config: + port: <%= p("autoscaler.operator.health.port") %> + <% if_p("autoscaler.operator.health.ca_cert", "autoscaler.operator.health.server_cert", "autoscaler.operator.health.server_key") do %> + tls: + ca_file: /var/vcap/jobs/operator/config/certs/healthendpoint/ca.crt + cert_file: /var/vcap/jobs/operator/config/certs/healthendpoint/server.crt + key_file: /var/vcap/jobs/operator/config/certs/healthendpoint/server.key + <% end %> + basic_auth: + username: <%= p("autoscaler.operator.health.username") %> + password: <%= p("autoscaler.operator.health.password") %> http_client_timeout: <%= p("autoscaler.operator.http_client_timeout") %> diff --git a/jobs/scalingengine/templates/scalingengine.yml.erb b/jobs/scalingengine/templates/scalingengine.yml.erb index 38c913d67a..562b65596e 100644 --- a/jobs/scalingengine/templates/scalingengine.yml.erb +++ b/jobs/scalingengine/templates/scalingengine.yml.erb @@ -63,15 +63,17 @@ logging: level: <%= p("autoscaler.scalingengine.logging.level") %> http_client_timeout: <%= p("autoscaler.scalingengine.http_client_timeout") %> health: - port: <%= p("autoscaler.scalingengine.health.port") %> - username: <%= p("autoscaler.scalingengine.health.username") %> - password: <%= p("autoscaler.scalingengine.health.password") %> - <% if_p("autoscaler.scalingengine.health.ca_cert", "autoscaler.scalingengine.health.server_cert", "autoscaler.scalingengine.health.server_key") do %> - tls: - ca_file: /var/vcap/jobs/scalingengine/config/certs/healthendpoint/ca.crt - cert_file: /var/vcap/jobs/scalingengine/config/certs/healthendpoint/server.crt - key_file: /var/vcap/jobs/scalingengine/config/certs/healthendpoint/server.key - <% end %> + basic_auth: + username: <%= p("autoscaler.scalingengine.health.username") %> + password: <%= p("autoscaler.scalingengine.health.password") %> + server_config: + port: <%= p("autoscaler.scalingengine.health.port") %> + <% if_p("autoscaler.scalingengine.health.ca_cert", "autoscaler.scalingengine.health.server_cert", "autoscaler.scalingengine.health.server_key") do %> + tls: + ca_file: /var/vcap/jobs/scalingengine/config/certs/healthendpoint/ca.crt + cert_file: /var/vcap/jobs/scalingengine/config/certs/healthendpoint/server.crt + key_file: /var/vcap/jobs/scalingengine/config/certs/healthendpoint/server.key + <% end %> db: diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml index b12de16443..33c0a7268a 100644 --- a/operations/use-cf-services.yml +++ b/operations/use-cf-services.yml @@ -1,9 +1,3 @@ -- type: replace - path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/apiserver/metrics_forwarder - value: - host: ((metricsforwarder_host)) - mtls_host: ((metricsforwarder_host)) - ## add router tcp route for postgres - type: replace path: /instance_groups/name=postgres/jobs/- diff --git a/packages/golangapiserver/spec b/packages/golangapiserver/spec index 4b88477766..db12046d87 100644 --- a/packages/golangapiserver/spec +++ b/packages/golangapiserver/spec @@ -11,6 +11,8 @@ files: - autoscaler/api/policyvalidator/* - autoscaler/api/schemas/* - autoscaler/api/* # gosub +- autoscaler/api/apis/* # gosub +- autoscaler/api/apis/scalinghistory/* # gosub - autoscaler/api/broker/* # gosub - autoscaler/api/brokerserver/* # gosub - autoscaler/api/cmd/api/* # gosub @@ -26,12 +28,12 @@ files: - autoscaler/db/sqldb/* # gosub - autoscaler/healthendpoint/* # gosub - autoscaler/helpers/* # gosub -- autoscaler/helpers/apis/scalinghistory/* # gosub - autoscaler/helpers/handlers/* # gosub - autoscaler/metricsforwarder/server/common/* # gosub - autoscaler/models/* # gosub - autoscaler/ratelimiter/* # gosub - autoscaler/routes/* # gosub +- autoscaler/scalingengine/apis/scalinghistory/* # gosub - autoscaler/vendor/code.cloudfoundry.org/cfhttp/v2/* # gosub - autoscaler/vendor/code.cloudfoundry.org/clock/* # gosub - autoscaler/vendor/code.cloudfoundry.org/lager/v3/* # gosub diff --git a/packages/scalingengine/spec b/packages/scalingengine/spec index 3769e5739a..aeff80f066 100644 --- a/packages/scalingengine/spec +++ b/packages/scalingengine/spec @@ -14,12 +14,14 @@ files: - autoscaler/db/sqldb/* # gosub - autoscaler/healthendpoint/* # gosub - autoscaler/helpers/* # gosub -- autoscaler/helpers/apis/scalinghistory/* # gosub - autoscaler/helpers/handlers/* # gosub - autoscaler/metricsforwarder/server/common/* # gosub - autoscaler/models/* # gosub - autoscaler/routes/* # gosub - autoscaler/scalingengine/* # gosub +- autoscaler/scalingengine/apis/* # gosub +- autoscaler/scalingengine/apis/scalinghistory/* # gosub +- autoscaler/scalingengine/client/* # gosub - autoscaler/scalingengine/cmd/scalingengine/* # gosub - autoscaler/scalingengine/config/* # gosub - autoscaler/scalingengine/schedule/* # gosub diff --git a/spec/jobs/common/health_endpoint_spec.rb b/spec/jobs/common/health_endpoint_spec.rb index 98a96b2898..097e9e14b6 100644 --- a/spec/jobs/common/health_endpoint_spec.rb +++ b/spec/jobs/common/health_endpoint_spec.rb @@ -13,40 +13,38 @@ %w[operator operator config/operator.yml operator.yml], %w[scalingengine scalingengine config/scalingengine.yml scalingengine.yml] ].each do |service, release_job, config_file, properties_file| - context service do - context "health endpoint" do - before(:each) do - @properties = YAML.safe_load(fixture(properties_file).read) - @template = release.job(release_job).template(config_file) - @links = case service - when "eventgenerator" - [Bosh::Template::Test::Link.new(name: "eventgenerator")] - else - [] - end - @rendered_template = YAML.safe_load(@template.render(@properties, consumes: @links)) - end - it "by default TLS is not configured" do - expect(@rendered_template["health"]["tls"]).to be_nil + describe "service #{service} health endpoint" do + before(:each) do + @properties = YAML.safe_load(fixture(properties_file).read) + @template = release.job(release_job).template(config_file) + @links = case service + when "eventgenerator" + [Bosh::Template::Test::Link.new(name: "eventgenerator")] + else + [] end + @rendered_template = YAML.safe_load(@template.render(@properties, consumes: @links)) + end + it "by default TLS is not configured" do + expect(@rendered_template["health"]["server_config"]["tls"]).to be_nil + end - it "TLS can be enabled" do - service_config = (@properties["autoscaler"][service] ||= {}) - service_config["health"] = { - "ca_cert" => "SOME_CA", - "server_cert" => "SOME_CERT", - "server_key" => "SOME_KEY" - } + it "TLS can be enabled" do + service_config = (@properties["autoscaler"][service] ||= {}) + service_config["health"] = { + "ca_cert" => "SOME_CA", + "server_cert" => "SOME_CERT", + "server_key" => "SOME_KEY" + } - rendered_template = YAML.safe_load(@template.render(@properties, consumes: @links)) + rendered_template = YAML.safe_load(@template.render(@properties, consumes: @links)) - expect(rendered_template["health"]["tls"]).not_to be_nil - expect(rendered_template["health"]["tls"]).to include({ - "key_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.key", - "ca_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/ca.crt", - "cert_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.crt" - }) - end + expect(rendered_template["health"]["server_config"]["tls"]).not_to be_nil + expect(rendered_template["health"]["server_config"]["tls"]).to include({ + "key_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.key", + "ca_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/ca.crt", + "cert_file" => "/var/vcap/jobs/#{release_job}/config/certs/healthendpoint/server.crt" + }) end end end diff --git a/spec/jobs/eventgenerator/eventgenerator_spec.rb b/spec/jobs/eventgenerator/eventgenerator_spec.rb index e868cb2cd5..3fe85e2ff1 100644 --- a/spec/jobs/eventgenerator/eventgenerator_spec.rb +++ b/spec/jobs/eventgenerator/eventgenerator_spec.rb @@ -29,10 +29,7 @@ "port" => 1234 } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["eventgenerator"]["health"]["port"]) end it "check eventgenerator username and password" do @@ -44,12 +41,9 @@ } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234, - "username" => "test-user", - "password" => "test-user-password"} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["eventgenerator"]["health"]["port"]) + expect(rendered_template["health"]["basic_auth"]["username"]).to eq(properties["autoscaler"]["eventgenerator"]["health"]["username"]) + expect(rendered_template["health"]["basic_auth"]["password"]).to eq(properties["autoscaler"]["eventgenerator"]["health"]["password"]) end describe "when using log-cache via https/uaa" do diff --git a/spec/jobs/metricsforwarder/metricsforwarder_spec.rb b/spec/jobs/metricsforwarder/metricsforwarder_spec.rb index 140ca46a92..0d66108e17 100644 --- a/spec/jobs/metricsforwarder/metricsforwarder_spec.rb +++ b/spec/jobs/metricsforwarder/metricsforwarder_spec.rb @@ -42,10 +42,7 @@ } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["metricsforwarder"]["health"]["port"]) end it "check metricsforwarder basic auth username and password" do @@ -57,12 +54,9 @@ } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234, - "username" => "test-user", - "password" => "test-user-password"} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["metricsforwarder"]["health"]["port"]) + expect(rendered_template["health"]["basic_auth"]["username"]).to eq(properties["autoscaler"]["metricsforwarder"]["health"]["username"]) + expect(rendered_template["health"]["basic_auth"]["password"]).to eq(properties["autoscaler"]["metricsforwarder"]["health"]["password"]) end it "has a cred helper impl by default" do diff --git a/spec/jobs/operator/operator_spec.rb b/spec/jobs/operator/operator_spec.rb index fb7ebb2264..a3dc12f463 100644 --- a/spec/jobs/operator/operator_spec.rb +++ b/spec/jobs/operator/operator_spec.rb @@ -20,10 +20,7 @@ } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["operator"]["health"]["port"]) end it "check operator basic auth username and password" do @@ -35,12 +32,9 @@ } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234, - "username" => "test-user", - "password" => "test-user-password"} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["operator"]["health"]["port"]) + expect(rendered_template["health"]["basic_auth"]["username"]).to eq(properties["autoscaler"]["operator"]["health"]["username"]) + expect(rendered_template["health"]["basic_auth"]["password"]).to eq(properties["autoscaler"]["operator"]["health"]["password"]) end context "uses tls" do diff --git a/spec/jobs/scalingengine/scalingengine_spec.rb b/spec/jobs/scalingengine/scalingengine_spec.rb index 1c40ad7d3b..39d4c13891 100644 --- a/spec/jobs/scalingengine/scalingengine_spec.rb +++ b/spec/jobs/scalingengine/scalingengine_spec.rb @@ -21,10 +21,7 @@ } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["scalingengine"]["health"]["port"]) end it "check scalingengine basic auth username and password" do @@ -36,12 +33,9 @@ } } - expect(rendered_template["health"]) - .to include( - {"port" => 1234, - "username" => "test-user", - "password" => "test-user-password"} - ) + expect(rendered_template["health"]["server_config"]["port"]).to eq(properties["autoscaler"]["scalingengine"]["health"]["port"]) + expect(rendered_template["health"]["basic_auth"]["username"]).to eq(properties["autoscaler"]["scalingengine"]["health"]["username"]) + expect(rendered_template["health"]["basic_auth"]["password"]).to eq(properties["autoscaler"]["scalingengine"]["health"]["password"]) end end diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index 4a8ed8c098..d74ddc228a 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -33,19 +33,23 @@ GINKGO_VERSION = v$(shell cat ../../.tool-versions | grep ginkgo | cut --delimi # ogen generated OpenAPI clients and servers -openapi-generated-clients-and-servers-dir := ./helpers/apis/scalinghistory +openapi-generated-clients-and-servers-api-dir := ./api/apis/scalinghistory +openapi-generated-clients-and-servers-scalingengine-dir := ./scalingengine/apis/scalinghistory + openapi-spec-path := ../../api openapi-specs-list = $(wildcard ${openapi-spec-path}/*.yaml) -openapi-generated-clients-and-servers-files = $(wildcard ${openapi-generated-clients-and-servers-dir}/*.go) +openapi-generated-clients-and-servers-api-files = $(wildcard ${openapi-generated-clients-and-servers-api-dir}/*.go) +openapi-generated-clients-and-servers-scalingengine-files = $(wildcard ${openapi-generated-clients-and-servers-scalingengine-dir}/*.go) .PHONY: generate-openapi-generated-clients-and-servers -generate-openapi-generated-clients-and-servers: ${openapi-generated-clients-and-servers-dir} ${openapi-generated-clients-and-servers-files} -${openapi-generated-clients-and-servers-dir} ${openapi-generated-clients-and-servers-files} &: $(wildcard ./helpers/apis/generate.go) ${openapi-specs-list} ./go.mod ./go.sum +generate-openapi-generated-clients-and-servers: ${openapi-generated-clients-and-servers-api-dir} ${openapi-generated-clients-and-servers-api-files} ${openapi-generated-clients-and-servers-scalingengine-dir} ${openapi-generated-clients-and-servers-scalingengine-files} +${openapi-generated-clients-and-servers-api-dir} ${openapi-generated-clients-and-servers-api-files} ${openapi-generated-clients-and-servers-scalingengine-dir} ${openapi-generated-clients-and-servers-scalingengine-files} &: $(wildcard ./scalingengine/apis/generate.go) $(wildcard ./api/apis/generate.go) ${openapi-specs-list} ./go.mod ./go.sum @echo "# Generating OpenAPI clients and servers" - # $(wildcard ./helpers/apis/generate.go) causes the target to always being executed, no matter if file exists or not. + # $(wildcard ./api/apis/generate.go) causes the target to always being executed, no matter if file exists or not. # so let's don't fail if file can't be found, e.g. the eventgenerator bosh package does not contain it. - go generate ./helpers/apis/generate.go || true + go generate ./api/apis/generate.go || true + go generate ./scalingengine/apis/generate.go || true # The presence of the subsequent directory indicates whether the fakes still need to be generated # or not. @@ -92,8 +96,9 @@ go-mod-vendor: ${go-vendoring-folder} ${go-vendored-files} ${go-vendoring-folder} ${go-vendored-files} &: ${app-fakes-dir} ${app-fakes-files} go mod vendor + # CGO_ENABLED := 1 is required to enforce dynamic linking which is a requirement of dynatrace. -build-%: ${openapi-generated-clients-and-servers-dir} ${openapi-generated-clients-and-servers-files} +build-%: @echo "# building $*" @CGO_ENABLED=1 go build $(BUILDTAGS) $(BUILDFLAGS) -o build/$* $*/cmd/$*/main.go @@ -146,7 +151,8 @@ clean: @rm --force --recursive 'build' @rm --force --recursive 'fakes' @rm --force --recursive 'vendor' - @rm --force --recursive "${openapi-generated-clients-and-servers-dir}" + @rm --force --recursive "${openapi-generated-clients-and-servers-api-dir}" + @rm --force --recursive "${openapi-generated-clients-and-servers-scalingengine-dir}" .PHONY: mta-deploy mta-deploy: mta-build build-extension-file @@ -167,11 +173,10 @@ mta-logs: .PHONY: mta-build mta-build: mta-build-clean @echo "bulding mtar file for version: $(VERSION)" - cp mta.tpl.yaml mta.yaml - sed -i 's/VERSION/$(VERSION)/g' mta.yaml - mkdir -p $(DEST) - mbt build - @mv mta_archives/com.github.cloudfoundry.app-autoscaler-release_$(VERSION).mtar $(DEST)/app-autoscaler-release-v$(VERSION).mtar + @cp mta.tpl.yaml mta.yaml + @sed -i 's/VERSION/$(VERSION)/g' mta.yaml + @mbt build + @mkdir -p "${PWD}/$(DEST)" && mv ${PWD}/mta_archives/com.github.cloudfoundry.app-autoscaler-release_$(VERSION).mtar ${PWD}/$(DEST)/app-autoscaler-release-v$(VERSION).mtar mta-build-clean: rm -rf mta_archives diff --git a/src/autoscaler/api/apis/generate.go b/src/autoscaler/api/apis/generate.go new file mode 100644 index 0000000000..dfb2141a54 --- /dev/null +++ b/src/autoscaler/api/apis/generate.go @@ -0,0 +1,3 @@ +package apis + +//go:generate go run github.com/ogen-go/ogen/cmd/ogen --config ogen-config.yaml --package scalinghistory --target scalinghistory --clean ../../../../api/scaling-history-api.openapi.yaml diff --git a/src/autoscaler/api/apis/ogen-config.yaml b/src/autoscaler/api/apis/ogen-config.yaml new file mode 100644 index 0000000000..5b85450eaa --- /dev/null +++ b/src/autoscaler/api/apis/ogen-config.yaml @@ -0,0 +1,2 @@ +parser: + allow_remote: true diff --git a/src/autoscaler/api/cmd/api/api_suite_test.go b/src/autoscaler/api/cmd/api/api_suite_test.go index 88d4d5b67a..b545e2f0ff 100644 --- a/src/autoscaler/api/cmd/api/api_suite_test.go +++ b/src/autoscaler/api/cmd/api/api_suite_test.go @@ -3,7 +3,6 @@ package main_test import ( "database/sql" "encoding/json" - "net/http" "os" "os/exec" "path/filepath" @@ -12,8 +11,6 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf/mocks" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -24,6 +21,8 @@ import ( "github.com/onsi/gomega/gexec" "gopkg.in/yaml.v3" + . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,18 +36,16 @@ const ( ) var ( - apPath string - cfg config.Config - configFile *os.File - apiHttpClient *http.Client - healthHttpClient *http.Client - catalogBytes string - schedulerServer *ghttp.Server - brokerPort int - publicApiPort int - healthport int - infoBytes string - ccServer *mocks.Server + apPath string + cfg config.Config + configFile *os.File + schedulerServer *ghttp.Server + catalogBytes string + brokerPort int + publicApiPort int + healthport int + infoBytes string + ccServer *mocks.Server ) func TestApi(t *testing.T) { @@ -125,7 +122,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { CACertFile: filepath.Join(testCertDir, "autoscaler-ca.crt"), }, } - cfg.PublicApiServer = helpers.ServerConfig{ + cfg.Server = helpers.ServerConfig{ Port: publicApiPort, TLS: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "api.key"), @@ -134,15 +131,15 @@ var _ = SynchronizedBeforeSuite(func() []byte { }, } cfg.Logging.Level = "info" - cfg.DB = make(map[string]db.DatabaseConfig) + cfg.Db = make(map[string]db.DatabaseConfig) dbUrl := GetDbUrl() - cfg.DB[db.BindingDb] = db.DatabaseConfig{ + cfg.Db[db.BindingDb] = db.DatabaseConfig{ URL: dbUrl, MaxOpenConnections: 10, MaxIdleConnections: 5, ConnectionMaxLifetime: 10 * time.Second, } - cfg.DB[db.PolicyDb] = db.DatabaseConfig{ + cfg.Db[db.PolicyDb] = db.DatabaseConfig{ URL: dbUrl, MaxOpenConnections: 10, MaxIdleConnections: 5, @@ -201,8 +198,10 @@ var _ = SynchronizedBeforeSuite(func() []byte { ServerConfig: helpers.ServerConfig{ Port: healthport, }, - HealthCheckUsername: "healthcheckuser", - HealthCheckPassword: "healthcheckpassword", + BasicAuth: models.BasicAuth{ + Username: "healthcheckuser", + Password: "healthcheckpassword", + }, } cfg.RateLimit.MaxAmount = 10 cfg.RateLimit.ValidDuration = 1 * time.Second @@ -211,9 +210,6 @@ var _ = SynchronizedBeforeSuite(func() []byte { configFile = writeConfig(&cfg) - apiHttpClient = NewApiClient() - - healthHttpClient = &http.Client{} }) var _ = SynchronizedAfterSuite(func() { diff --git a/src/autoscaler/api/cmd/api/api_test.go b/src/autoscaler/api/cmd/api/api_test.go index 4a2b35689b..89196c0b47 100644 --- a/src/autoscaler/api/cmd/api/api_test.go +++ b/src/autoscaler/api/cmd/api/api_test.go @@ -4,13 +4,14 @@ import ( "fmt" "io" "net/http" + "net/url" "os" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" + . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -20,14 +21,36 @@ import ( var _ = Describe("Api", func() { var ( - runner *ApiRunner - rsp *http.Response + runner *ApiRunner + rsp *http.Response + brokerHttpClient *http.Client + healthHttpClient *http.Client + apiHttpClient *http.Client + + serverURL *url.URL + brokerURL *url.URL + healthURL *url.URL + + err error ) BeforeEach(func() { - brokerHttpClient = NewServiceBrokerClient() runner = NewApiRunner() + + brokerHttpClient = NewServiceBrokerClient() + healthHttpClient = &http.Client{} + apiHttpClient = NewPublicApiClient() + + serverURL, err = url.Parse(fmt.Sprintf("https://127.0.0.1:%d", cfg.Server.Port)) + Expect(err).NotTo(HaveOccurred()) + + brokerURL, err = url.Parse(fmt.Sprintf("https://127.0.0.1:%d", cfg.BrokerServer.Port)) + Expect(err).NotTo(HaveOccurred()) + + healthURL, err = url.Parse(fmt.Sprintf("http://127.0.0.1:%d", cfg.Health.ServerConfig.Port)) + Expect(err).NotTo(HaveOccurred()) + }) Describe("Api configuration check", func() { @@ -72,9 +95,9 @@ var _ = Describe("Api", func() { runner.startCheck = "" missingConfig := cfg - missingConfig.DB = make(map[string]db.DatabaseConfig) - missingConfig.DB[db.PolicyDb] = db.DatabaseConfig{URL: ""} - missingConfig.DB[db.BindingDb] = db.DatabaseConfig{URL: ""} + missingConfig.Db = make(map[string]db.DatabaseConfig) + missingConfig.Db[db.PolicyDb] = db.DatabaseConfig{URL: ""} + missingConfig.Db[db.BindingDb] = db.DatabaseConfig{URL: ""} var brokerCreds []config.BrokerCredentialsConfig missingConfig.BrokerCredentials = brokerCreds @@ -118,14 +141,17 @@ var _ = Describe("Api", func() { BeforeEach(func() { runner.Start() }) + It("succeeds with a 200", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d/v2/catalog", brokerPort), nil) + brokerURL.Path = "/v2/catalog" + req, err := http.NewRequest(http.MethodGet, brokerURL.String(), nil) Expect(err).NotTo(HaveOccurred()) req.SetBasicAuth(username, password) rsp, err = brokerHttpClient.Do(req) Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) if rsp.StatusCode != http.StatusOK { Fail(fmt.Sprintf("Not ok:%d", rsp.StatusCode)) @@ -153,7 +179,8 @@ var _ = Describe("Api", func() { runner.Start() }) It("succeeds with a 200", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d/v1/info", publicApiPort), nil) + serverURL.Path = "/v1/info" + req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil) Expect(err).NotTo(HaveOccurred()) rsp, err = apiHttpClient.Do(req) @@ -169,8 +196,8 @@ var _ = Describe("Api", func() { Describe("when Health server is ready to serve RESTful API", func() { BeforeEach(func() { basicAuthConfig := cfg - basicAuthConfig.Health.HealthCheckUsername = "" - basicAuthConfig.Health.HealthCheckPassword = "" + basicAuthConfig.Health.BasicAuth.Username = "" + basicAuthConfig.Health.BasicAuth.Password = "" runner.configPath = writeConfig(&basicAuthConfig).Name() runner.Start() }) @@ -180,7 +207,8 @@ var _ = Describe("Api", func() { }) Context("when a request to query health comes", func() { It("returns with a 200", func() { - rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d", healthport)) + rsp, err := healthHttpClient.Get(healthURL.String()) + Expect(err).NotTo(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) raw, _ := io.ReadAll(rsp.Body) @@ -224,7 +252,7 @@ var _ = Describe("Api", func() { req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth(cfg.Health.HealthCheckUsername, cfg.Health.HealthCheckPassword) + req.SetBasicAuth(cfg.Health.BasicAuth.Username, cfg.Health.BasicAuth.Password) rsp, err := healthHttpClient.Do(req) Expect(err).ToNot(HaveOccurred()) @@ -246,7 +274,7 @@ var _ = Describe("Api", func() { }) Context("when a request to query health comes", func() { It("returns with a 200", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d/v1/info", publicApiPort), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/v1/info", serverURL), nil) Expect(err).NotTo(HaveOccurred()) rsp, err = apiHttpClient.Do(req) diff --git a/src/autoscaler/api/cmd/api/main.go b/src/autoscaler/api/cmd/api/main.go index ee9c782c2c..c183faba6f 100644 --- a/src/autoscaler/api/cmd/api/main.go +++ b/src/autoscaler/api/cmd/api/main.go @@ -6,11 +6,11 @@ import ( "os" "time" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/brokerserver" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/publicapiserver" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cred_helper" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db/sqldb" @@ -20,7 +20,6 @@ import ( "code.cloudfoundry.org/clock" "code.cloudfoundry.org/lager/v3" - "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" "github.com/tedsuo/ifrit/grouper" "github.com/tedsuo/ifrit/sigmon" @@ -28,26 +27,23 @@ import ( func main() { var path string + var err error + var conf *config.Config + flag.StringVar(&path, "c", "", "config file") flag.Parse() - if path == "" { - _, _ = fmt.Fprintln(os.Stderr, "missing config file") - os.Exit(1) - } - configFile, err := os.Open(path) + vcapConfiguration, err := configutil.NewVCAPConfigurationReader() if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to open config file '%s' : %s\n", path, err.Error()) + _, _ = fmt.Fprintf(os.Stdout, "failed to read vcap configuration : %s\n", err.Error()) os.Exit(1) } - var conf *config.Config - conf, err = config.LoadConfig(configFile) + conf, err = config.LoadConfig(path, vcapConfiguration) if err != nil { _, _ = fmt.Fprintf(os.Stdout, "failed to read config file '%s' : %s\n", path, err.Error()) os.Exit(1) } - _ = configFile.Close() err = conf.Validate() if err != nil { @@ -61,18 +57,14 @@ func main() { members := grouper.Members{} - policyDb := sqldb.CreatePolicyDb(conf.DB[db.PolicyDb], logger) + policyDb := sqldb.CreatePolicyDb(conf.Db[db.PolicyDb], logger) defer func() { _ = policyDb.Close() }() - logger.Debug("Connected to PolicyDB", lager.Data{"dbConfig": conf.DB[db.PolicyDb]}) + logger.Debug("Connected to PolicyDB", lager.Data{"dbConfig": conf.Db[db.PolicyDb]}) - credentialProvider := cred_helper.CredentialsProvider(conf.CredHelperImpl, conf.StoredProcedureConfig, conf.DB, 10*time.Second, 10*time.Minute, logger, policyDb) + credentialProvider := cred_helper.CredentialsProvider(conf.CredHelperImpl, conf.StoredProcedureConfig, conf.Db, 10*time.Second, 10*time.Minute, logger, policyDb) defer func() { _ = credentialProvider.Close() }() httpStatusCollector := healthendpoint.NewHTTPStatusCollector("autoscaler", "golangapiserver") - prometheusCollectors := []prometheus.Collector{ - healthendpoint.NewDatabaseStatusCollector("autoscaler", "golangapiserver", "policyDB", policyDb), - httpStatusCollector, - } paClock := clock.NewClock() cfClient := cf.NewCFClient(&conf.CF, logger.Session("cf"), paClock) @@ -83,13 +75,12 @@ func main() { } logger.Debug("Successfully logged into CF", lager.Data{"API": conf.CF.API}) - bindingDB, err := sqldb.NewBindingSQLDB(conf.DB[db.BindingDb], logger.Session("bindingdb-db")) + bindingDB, err := sqldb.NewBindingSQLDB(conf.Db[db.BindingDb], logger.Session("bindingdb-db")) if err != nil { - logger.Error("failed to connect bindingdb database", err, lager.Data{"dbConfig": conf.DB[db.BindingDb]}) + logger.Error("failed to connect bindingdb database", err, lager.Data{"dbConfig": conf.Db[db.BindingDb]}) os.Exit(1) } defer func() { _ = bindingDB.Close() }() - prometheusCollectors = append(prometheusCollectors, healthendpoint.NewDatabaseStatusCollector("autoscaler", "golangapiserver", "bindingDB", bindingDB)) checkBindingFunc := func(appId string) bool { return bindingDB.CheckServiceBinding(appId) } @@ -99,21 +90,26 @@ func main() { logger.Error("failed to create broker http server", err) os.Exit(1) } - members = append(members, grouper.Member{"broker_http_server", brokerHttpServer}) - promRegistry := prometheus.NewRegistry() - healthendpoint.RegisterCollectors(promRegistry, prometheusCollectors, true, logger.Session("golangapiserver-prometheus")) + rateLimiter := ratelimiter.DefaultRateLimiter(conf.RateLimit.MaxAmount, conf.RateLimit.ValidDuration, logger.Session("api-ratelimiter")) + + publicApiHttpServer := publicapiserver.NewPublicApiServer( + logger.Session("public_api_http_server"), conf, policyDb, bindingDB, + credentialProvider, checkBindingFunc, cfClient, httpStatusCollector, + rateLimiter) - publicApiHttpServer := createApiServer(conf, logger, policyDb, credentialProvider, checkBindingFunc, cfClient, httpStatusCollector, bindingDB) - healthServer, err := healthendpoint.NewServerWithBasicAuth(conf.Health, []healthendpoint.Checker{}, logger.Session("health-server"), promRegistry, time.Now) + vmServer, err := publicApiHttpServer.GetMtlsServer() if err != nil { - logger.Fatal("Failed to create health server", err) + logger.Error("failed to create public api http server", err) os.Exit(1) } + logger.Debug("Successfully created health server") + healthServer, _ := publicApiHttpServer.GetHealthServer() members = append(members, - grouper.Member{"public_api_http_server", publicApiHttpServer}, + grouper.Member{"public_api_http_server", vmServer}, + grouper.Member{"broker", brokerHttpServer}, grouper.Member{"health_server", healthServer}, ) @@ -130,13 +126,3 @@ func main() { logger.Info("exited") } - -func createApiServer(conf *config.Config, logger lager.Logger, policyDb *sqldb.PolicySQLDB, credentialProvider cred_helper.Credentials, checkBindingFunc api.CheckBindingFunc, cfClient cf.CFClient, httpStatusCollector healthendpoint.HTTPStatusCollector, bindingDB db.BindingDB) ifrit.Runner { - rateLimiter := ratelimiter.DefaultRateLimiter(conf.RateLimit.MaxAmount, conf.RateLimit.ValidDuration, logger.Session("api-ratelimiter")) - publicApiHttpServer, err := publicapiserver.NewPublicApiServer(logger.Session("public_api_http_server"), conf, policyDb, credentialProvider, checkBindingFunc, cfClient, httpStatusCollector, rateLimiter, bindingDB) - if err != nil { - logger.Error("failed to create public api http server", err) - os.Exit(1) - } - return publicApiHttpServer -} diff --git a/src/autoscaler/api/config/config.go b/src/autoscaler/api/config/config.go index 5d8439bc0b..9006b7cb28 100644 --- a/src/autoscaler/api/config/config.go +++ b/src/autoscaler/api/config/config.go @@ -3,11 +3,12 @@ package config import ( "errors" "fmt" - "io" + "os" "strings" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "golang.org/x/crypto/bcrypt" @@ -33,6 +34,11 @@ const ( DefaultDiskUpperThreshold = 2 * 1024 ) +var ( + ErrReadYaml = errors.New("failed to read config file") + ErrPublicApiServerConfigNotFound = errors.New("metricsforwarder config service not found") +) + var defaultBrokerServerConfig = helpers.ServerConfig{ Port: 8080, } @@ -92,8 +98,8 @@ type LowerUpperThresholdConfig struct { type Config struct { Logging helpers.LoggingConfig `yaml:"logging"` BrokerServer helpers.ServerConfig `yaml:"broker_server"` - PublicApiServer helpers.ServerConfig `yaml:"public_api_server"` - DB map[string]db.DatabaseConfig `yaml:"db"` + Server helpers.ServerConfig `yaml:"public_api_server"` + Db map[string]db.DatabaseConfig `yaml:"db"` BrokerCredentials []BrokerCredentialsConfig `yaml:"broker_credentials"` APIClientId string `yaml:"api_client_id"` PlanCheck *PlanCheckConfig `yaml:"plan_check"` @@ -119,13 +125,15 @@ type PlanCheckConfig struct { PlanDefinitions map[string]PlanDefinition `yaml:"plan_definitions"` } -func LoadConfig(reader io.Reader) (*Config, error) { - conf := &Config{ - Logging: defaultLoggingConfig, - BrokerServer: defaultBrokerServerConfig, - PublicApiServer: defaultPublicApiServerConfig, +func defaultConfig() Config { + return Config{ + Logging: defaultLoggingConfig, + BrokerServer: defaultBrokerServerConfig, + Server: defaultPublicApiServerConfig, CF: cf.Config{ - ClientConfig: cf.ClientConfig{SkipSSLValidation: false}, + ClientConfig: cf.ClientConfig{ + SkipSSLValidation: false, + }, }, RateLimit: models.RateLimitConfig{ MaxAmount: DefaultMaxAmount, @@ -151,17 +159,102 @@ func LoadConfig(reader io.Reader) (*Config, error) { }, } - dec := yaml.NewDecoder(reader) +} +func loadYamlFile(filepath string, conf *Config) error { + if filepath == "" { + return nil + } + file, err := os.Open(filepath) + if err != nil { + fmt.Fprintf(os.Stdout, "failed to open config file '%s': %s\n", filepath, err) + return ErrReadYaml + } + defer file.Close() + + dec := yaml.NewDecoder(file) dec.KnownFields(true) - err := dec.Decode(conf) + if err := dec.Decode(conf); err != nil { + return fmt.Errorf("%w: %v", ErrReadYaml, err) + } + return nil +} +func loadPublicApiServerConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + data, err := vcapReader.GetServiceCredentialContent("metricsforwarder-config", "metricsforwarder") + if err != nil { + return fmt.Errorf("%w: %v", ErrPublicApiServerConfigNotFound, err) + } + return yaml.Unmarshal(data, conf) +} + +func loadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + if !vcapReader.IsRunningOnCF() { + return nil + } + + conf.Server.Port = vcapReader.GetPort() + if err := loadPublicApiServerConfig(conf, vcapReader); err != nil { + return err + } + if conf.Db == nil { + conf.Db = make(map[string]db.DatabaseConfig) + } + + if err := configurePolicyDb(conf, vcapReader); err != nil { + return err + } + + if err := configureBindingDb(conf, vcapReader); err != nil { + return err + } + return nil +} + +func configurePolicyDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + currentPolicyDb, ok := conf.Db[db.PolicyDb] + if !ok { + conf.Db[db.PolicyDb] = db.DatabaseConfig{} + } + + dbURL, err := vcapReader.MaterializeDBFromService(db.PolicyDb) + currentPolicyDb.URL = dbURL + if err != nil { + return err + } + conf.Db[db.PolicyDb] = currentPolicyDb + return nil +} + +func configureBindingDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + currentBindingDb, ok := conf.Db[db.BindingDb] + if !ok { + conf.Db[db.BindingDb] = db.DatabaseConfig{} + } + + dbURL, err := vcapReader.MaterializeDBFromService(db.BindingDb) + currentBindingDb.URL = dbURL if err != nil { + return err + } + conf.Db[db.BindingDb] = currentBindingDb + + return nil +} + +func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) { + conf := defaultConfig() + + if err := loadYamlFile(filepath, &conf); err != nil { + return nil, err + } + + if err := loadVcapConfig(&conf, vcapReader); err != nil { return nil, err } conf.Logging.Level = strings.ToLower(conf.Logging.Level) - return conf, nil + return &conf, nil } func (c *Config) Validate() error { @@ -170,7 +263,7 @@ func (c *Config) Validate() error { return err } - if c.DB[db.PolicyDb].URL == "" { + if c.Db[db.PolicyDb].URL == "" { return fmt.Errorf("Configuration error: PolicyDB URL is empty") } if c.Scheduler.SchedulerURL == "" { @@ -210,7 +303,7 @@ func (c *Config) Validate() error { return fmt.Errorf("Configuration error: ScalingRules.CPU.UpperThreshold is less than zero") } - if c.DB[db.BindingDb].URL == "" { + if c.Db[db.BindingDb].URL == "" { return fmt.Errorf("Configuration error: BindingDB URL is empty") } diff --git a/src/autoscaler/api/config/config_test.go b/src/autoscaler/api/config/config_test.go index 8bc5389ec3..8b642b170b 100644 --- a/src/autoscaler/api/config/config_test.go +++ b/src/autoscaler/api/config/config_test.go @@ -1,10 +1,11 @@ package config_test import ( - "bytes" + "fmt" "time" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" @@ -13,510 +14,540 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" ) var _ = Describe("Config", func() { var ( - conf *Config - err error - configBytes string + conf *Config + err error + configBytes []byte + configFile string + mockVCAPConfigurationReader *fakes.FakeVCAPConfigurationReader ) + BeforeEach(func() { + mockVCAPConfigurationReader = &fakes.FakeVCAPConfigurationReader{} + }) + Describe("Load Config", func() { - JustBeforeEach(func() { - conf, err = LoadConfig(bytes.NewReader([]byte(configBytes))) - }) - Context("with invalid yaml", func() { - BeforeEach(func() { - configBytes = LoadFile("invalid_config.yml") - }) + When("config is read from env", func() { + var expectedDbUrl string - It("returns an error", func() { - Expect(err).To(MatchError(MatchRegexp("yaml: .*"))) - }) - }) - Context("with valid yaml", func() { - BeforeEach(func() { - configBytes = LoadFile("valid_config.yml") - }) - - It("It returns the config", func() { - Expect(err).NotTo(HaveOccurred()) - - Expect(conf.Logging.Level).To(Equal("debug")) - Expect(conf.BrokerServer.Port).To(Equal(8080)) - Expect(conf.BrokerServer.TLS).To(Equal( - models.TLSCerts{ - KeyFile: "/var/vcap/jobs/autoscaler/config/certs/broker.key", - CACertFile: "/var/vcap/jobs/autoscaler/config/certs/autoscaler-ca.crt", - CertFile: "/var/vcap/jobs/autoscaler/config/certs/broker.crt", - }, - )) - Expect(conf.PublicApiServer.Port).To(Equal(8081)) - Expect(conf.PublicApiServer.TLS).To(Equal( - models.TLSCerts{ - KeyFile: "/var/vcap/jobs/autoscaler/config/certs/api.key", - CACertFile: "/var/vcap/jobs/autoscaler/config/certs/autoscaler-ca.crt", - CertFile: "/var/vcap/jobs/autoscaler/config/certs/api.crt", - }, - )) - Expect(conf.DB[db.BindingDb]).To(Equal( - db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - })) - Expect(conf.DB[db.PolicyDb]).To(Equal( - db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - })) - Expect(conf.BrokerCredentials[0].BrokerUsername).To(Equal("broker_username")) - Expect(conf.BrokerCredentials[0].BrokerPassword).To(Equal("broker_password")) - Expect(conf.CatalogPath).To(Equal("../exampleconfig/catalog-example.json")) - Expect(conf.CatalogSchemaPath).To(Equal("../schemas/catalog.schema.json")) - Expect(conf.PolicySchemaPath).To(Equal("../exampleconfig/policy.schema.json")) - Expect(conf.Scheduler).To(Equal( - SchedulerConfig{ - SchedulerURL: "https://localhost:8083", - TLSClientCerts: models.TLSCerts{ - KeyFile: "/var/vcap/jobs/autoscaler/config/certs/sc.key", - CACertFile: "/var/vcap/jobs/autoscaler/config/certs/autoscaler-ca.crt", - CertFile: "/var/vcap/jobs/autoscaler/config/certs/sc.crt", - }, - }, - )) - Expect(conf.MetricsForwarder).To(Equal( - MetricsForwarderConfig{ - MetricsForwarderUrl: "https://localhost:8088", - MetricsForwarderMtlsUrl: "https://mtlssdsdds:8084", - }, - )) - Expect(conf.InfoFilePath).To(Equal("/var/vcap/jobs/autoscaer/config/info-file.json")) - Expect(conf.CF).To(Equal( - cf.Config{ - API: "https://api.example.com", - ClientID: "client-id", - Secret: "client-secret", - ClientConfig: cf.ClientConfig{ - SkipSSLValidation: false, - MaxRetries: 3, - MaxRetryWaitMs: 27, - }, - }, - )) - Expect(conf.CredHelperImpl).To(Equal("default")) - Expect(conf.ScalingRules.CPU.LowerThreshold).To(Equal(22)) - Expect(conf.ScalingRules.CPU.UpperThreshold).To(Equal(33)) - Expect(conf.ScalingRules.CPUUtil.LowerThreshold).To(Equal(22)) - Expect(conf.ScalingRules.CPUUtil.UpperThreshold).To(Equal(33)) - Expect(conf.ScalingRules.DiskUtil.LowerThreshold).To(Equal(22)) - Expect(conf.ScalingRules.DiskUtil.UpperThreshold).To(Equal(33)) - Expect(conf.ScalingRules.Disk.LowerThreshold).To(Equal(22)) - Expect(conf.ScalingRules.Disk.UpperThreshold).To(Equal(33)) + JustBeforeEach(func() { + mockVCAPConfigurationReader.IsRunningOnCFReturns(true) + mockVCAPConfigurationReader.MaterializeDBFromServiceReturns(expectedDbUrl, nil) + conf, err = LoadConfig("", mockVCAPConfigurationReader) }) - }) - Context("with partial config", func() { - BeforeEach(func() { - configBytes = LoadFile("partial_config.yml") - }) - It("It returns the default values", func() { - Expect(err).NotTo(HaveOccurred()) - - Expect(conf.Logging.Level).To(Equal("info")) - Expect(conf.BrokerServer.Port).To(Equal(8080)) - Expect(conf.PublicApiServer.Port).To(Equal(8081)) - Expect(conf.DB[db.BindingDb]).To(Equal( - db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 0, - MaxIdleConnections: 0, - ConnectionMaxLifetime: 0 * time.Second, - })) - Expect(conf.DB[db.PolicyDb]).To(Equal( - db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 0, - MaxIdleConnections: 0, - ConnectionMaxLifetime: 0 * time.Second, - })) - Expect(conf.ScalingRules.CPU.LowerThreshold).To(Equal(1)) - Expect(conf.ScalingRules.CPU.UpperThreshold).To(Equal(100)) - Expect(conf.ScalingRules.CPUUtil.LowerThreshold).To(Equal(1)) - Expect(conf.ScalingRules.CPUUtil.UpperThreshold).To(Equal(100)) - Expect(conf.ScalingRules.DiskUtil.LowerThreshold).To(Equal(1)) - Expect(conf.ScalingRules.DiskUtil.UpperThreshold).To(Equal(100)) - Expect(conf.ScalingRules.Disk.LowerThreshold).To(Equal(1)) - Expect(conf.ScalingRules.Disk.UpperThreshold).To(Equal(2 * 1024)) - }) - }) + When("vcap PORT is set to a number", func() { + BeforeEach(func() { + mockVCAPConfigurationReader.GetPortReturns(3333) + }) - Context("when it gives a non integer broker_server port", func() { - BeforeEach(func() { - configBytes = ` -broker_server: - port: port -` + It("sets env variable over config file", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Server.Port).To(Equal(3333)) + }) }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - }) - Context("when it gives a non integer public_api_server port", func() { - BeforeEach(func() { - configBytes = ` -public_api_server: - port: port -` - }) + When("service is empty", func() { + var expectedErr error + BeforeEach(func() { + expectedErr = fmt.Errorf("publicapiserver config service not found") + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(""), expectedErr) + }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - }) - Context("when it gives a non integer health server port", func() { - BeforeEach(func() { - configBytes = ` -health: - port: port -` + It("should error with config service not found", func() { + Expect(err).To(MatchError(MatchRegexp("publicapiserver config service not found"))) + }) }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - }) + When("VCAP_SERVICES has relational db service bind to app for policy db", func() { + BeforeEach(func() { + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(`{ "cred_helper_impl": "default" }`), nil) // #nosec G101 + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 + }) - Context("when max_amount of rate_limit is not an integer", func() { - BeforeEach(func() { - configBytes = ` -rate_limit: - max_amount: NOT-INTEGER -` - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - }) - Context("when valid_duration of rate_limit is not a time duration", func() { - BeforeEach(func() { - configBytes = ` -rate_limit: - valid_duration: NOT-TIME-DURATION -` + It("loads the db config from VCAP_SERVICES successfully", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Db[db.PolicyDb].URL).To(Equal(expectedDbUrl)) + actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(0) + Expect(actualDbName).To(Equal(db.PolicyDb)) + }) }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into time.Duration"))) - }) - }) - - }) - - Describe("Validate", func() { - BeforeEach(func() { - conf = &Config{} - conf.DB = make(map[string]db.DatabaseConfig) - conf.DB[db.BindingDb] = db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - } - conf.DB[db.PolicyDb] = db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - } - - brokerCred1 := BrokerCredentialsConfig{ - BrokerUsernameHash: []byte("$2a$10$WNO1cPko4iDAT6MkhaDojeJMU8ZdNH6gt.SapsFOsC0OF4cQ9qQwu"), // ruby -r bcrypt -e 'puts BCrypt::Password.create("broker_username")' - BrokerPasswordHash: []byte("$2a$10$evLviRLcIPKnWQqlBl3DJOvBZir9vJ4gdEeyoGgvnK/CGBnxIAFRu"), // ruby -r bcrypt -e 'puts BCrypt::Password.create("broker_password")' - } - var brokerCreds []BrokerCredentialsConfig - brokerCreds = append(brokerCreds, brokerCred1) - conf.BrokerCredentials = brokerCreds - - conf.CatalogSchemaPath = "../schemas/catalog.schema.json" - conf.CatalogPath = "../exampleconfig/catalog-example.json" - conf.PolicySchemaPath = "../exampleconfig/policy.schema.json" - - conf.Scheduler.SchedulerURL = "https://localhost:8083" - - conf.ScalingEngine.ScalingEngineUrl = "https://localhost:8084" - conf.EventGenerator.EventGeneratorUrl = "https://localhost:8085" - conf.MetricsForwarder.MetricsForwarderUrl = "https://localhost:8088" - - conf.CF.API = "https://api.bosh-lite.com" - conf.CF.ClientID = "client-id" - conf.CF.Secret = "secret" - - conf.InfoFilePath = "../exampleconfig/info-file.json" - - conf.RateLimit.MaxAmount = 10 - conf.RateLimit.ValidDuration = 1 * time.Second - - conf.CredHelperImpl = "path/to/plugin" - }) - JustBeforeEach(func() { - err = conf.Validate() - }) - Context("When all the configs are valid", func() { - It("should not error", func() { - Expect(err).NotTo(HaveOccurred()) - }) - }) + When("VCAP_SERVICES has relational db service bind to app for binding db", func() { + BeforeEach(func() { + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(`{ "cred_helper_impl": "default" }`), nil) // #nosec G101 + expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/binding_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 + }) - Context("when bindingdb url is not set", func() { - BeforeEach(func() { - conf.DB[db.BindingDb] = db.DatabaseConfig{URL: ""} - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: BindingDB URL is empty"))) + It("loads the db config from VCAP_SERVICES successfully", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(conf.Db[db.BindingDb].URL).To(Equal(expectedDbUrl)) + actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(1) + Expect(actualDbName).To(Equal(db.BindingDb)) + }) }) }) - Context("when policydb url is not set", func() { - BeforeEach(func() { - conf.DB[db.PolicyDb] = db.DatabaseConfig{URL: ""} - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: PolicyDB URL is empty"))) + When("config is read from file", func() { + JustBeforeEach(func() { + configFile = testhelpers.BytesToFile(configBytes) + conf, err = LoadConfig(configFile, mockVCAPConfigurationReader) }) - }) - Context("when scheduler url is not set", func() { - BeforeEach(func() { - conf.Scheduler.SchedulerURL = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: scheduler.scheduler_url is empty"))) - }) - }) + Context("with invalid yaml", func() { + BeforeEach(func() { + configBytes = []byte(testhelpers.LoadFile("invalid_config.yml")) + }) - Context("when neither the broker username nor its hash is set", func() { - BeforeEach(func() { - brokerCred1 := BrokerCredentialsConfig{ - BrokerPasswordHash: []byte(""), - BrokerPassword: "", - } - var brokerCreds []BrokerCredentialsConfig - brokerCreds = append(brokerCreds, brokerCred1) - conf.BrokerCredentials = brokerCreds - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_username and broker_username_hash are empty, please provide one of them"))) + It("returns an error", func() { + Expect(err).To(MatchError(MatchRegexp("yaml: .*"))) + }) }) - }) + Context("with valid yaml", func() { + BeforeEach(func() { + configBytes = []byte(testhelpers.LoadFile("valid_config.yml")) + }) - Context("when both the broker username and its hash are set", func() { - BeforeEach(func() { - brokerCred1 := BrokerCredentialsConfig{ - BrokerUsername: "broker_username", - BrokerUsernameHash: []byte("$2a$10$WNO1cPko4iDAT6MkhaDojeJMU8ZdNH6gt.SapsFOsC0OF4cQ9qQwu"), - } - var brokerCreds []BrokerCredentialsConfig - brokerCreds = append(brokerCreds, brokerCred1) - conf.BrokerCredentials = brokerCreds - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_username and broker_username_hash are set, please provide only one of them"))) - }) - }) + It("It returns the config", func() { + Expect(err).NotTo(HaveOccurred()) - Context("when just the broker username is set", func() { - BeforeEach(func() { - conf.BrokerCredentials[0].BrokerUsername = "broker_username" - conf.BrokerCredentials[0].BrokerUsernameHash = []byte("") - }) - It("should not error", func() { - Expect(err).NotTo(HaveOccurred()) + Expect(conf.Logging.Level).To(Equal("debug")) + Expect(conf.BrokerServer.Port).To(Equal(8080)) + Expect(conf.BrokerServer.TLS).To(Equal( + models.TLSCerts{ + KeyFile: "/var/vcap/jobs/autoscaler/config/certs/broker.key", + CACertFile: "/var/vcap/jobs/autoscaler/config/certs/autoscaler-ca.crt", + CertFile: "/var/vcap/jobs/autoscaler/config/certs/broker.crt", + }, + )) + Expect(conf.Server.Port).To(Equal(8081)) + Expect(conf.Server.TLS).To(Equal( + models.TLSCerts{ + KeyFile: "/var/vcap/jobs/autoscaler/config/certs/api.key", + CACertFile: "/var/vcap/jobs/autoscaler/config/certs/autoscaler-ca.crt", + CertFile: "/var/vcap/jobs/autoscaler/config/certs/api.crt", + }, + )) + Expect(conf.Db[db.BindingDb]).To(Equal( + db.DatabaseConfig{ + URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", + MaxOpenConnections: 10, + MaxIdleConnections: 5, + ConnectionMaxLifetime: 60 * time.Second, + })) + Expect(conf.Db[db.PolicyDb]).To(Equal( + db.DatabaseConfig{ + URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", + MaxOpenConnections: 10, + MaxIdleConnections: 5, + ConnectionMaxLifetime: 60 * time.Second, + })) + Expect(conf.BrokerCredentials[0].BrokerUsername).To(Equal("broker_username")) + Expect(conf.BrokerCredentials[0].BrokerPassword).To(Equal("broker_password")) + Expect(conf.CatalogPath).To(Equal("../exampleconfig/catalog-example.json")) + Expect(conf.CatalogSchemaPath).To(Equal("../schemas/catalog.schema.json")) + Expect(conf.PolicySchemaPath).To(Equal("../exampleconfig/policy.schema.json")) + Expect(conf.Scheduler).To(Equal( + SchedulerConfig{ + SchedulerURL: "https://localhost:8083", + TLSClientCerts: models.TLSCerts{ + KeyFile: "/var/vcap/jobs/autoscaler/config/certs/sc.key", + CACertFile: "/var/vcap/jobs/autoscaler/config/certs/autoscaler-ca.crt", + CertFile: "/var/vcap/jobs/autoscaler/config/certs/sc.crt", + }, + }, + )) + Expect(conf.MetricsForwarder).To(Equal( + MetricsForwarderConfig{ + MetricsForwarderUrl: "https://localhost:8088", + MetricsForwarderMtlsUrl: "https://mtlssdsdds:8084", + }, + )) + Expect(conf.InfoFilePath).To(Equal("/var/vcap/jobs/autoscaer/config/info-file.json")) + Expect(conf.CF).To(Equal( + cf.Config{ + API: "https://api.example.com", + ClientID: "client-id", + Secret: "client-secret", + ClientConfig: cf.ClientConfig{ + SkipSSLValidation: false, + MaxRetries: 3, + MaxRetryWaitMs: 27, + }, + }, + )) + Expect(conf.CredHelperImpl).To(Equal("default")) + Expect(conf.ScalingRules.CPU.LowerThreshold).To(Equal(22)) + Expect(conf.ScalingRules.CPU.UpperThreshold).To(Equal(33)) + Expect(conf.ScalingRules.CPUUtil.LowerThreshold).To(Equal(22)) + Expect(conf.ScalingRules.CPUUtil.UpperThreshold).To(Equal(33)) + Expect(conf.ScalingRules.DiskUtil.LowerThreshold).To(Equal(22)) + Expect(conf.ScalingRules.DiskUtil.UpperThreshold).To(Equal(33)) + Expect(conf.ScalingRules.Disk.LowerThreshold).To(Equal(22)) + Expect(conf.ScalingRules.Disk.UpperThreshold).To(Equal(33)) + }) + }) + + Context("with partial config", func() { + BeforeEach(func() { + configBytes = []byte(testhelpers.LoadFile("partial_config.yml")) + }) + It("It returns the default values", func() { + Expect(err).NotTo(HaveOccurred()) + + Expect(conf.Logging.Level).To(Equal("info")) + Expect(conf.BrokerServer.Port).To(Equal(8080)) + Expect(conf.Server.Port).To(Equal(8081)) + Expect(conf.Db[db.BindingDb]).To(Equal( + db.DatabaseConfig{ + URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", + MaxOpenConnections: 0, + MaxIdleConnections: 0, + ConnectionMaxLifetime: 0 * time.Second, + })) + Expect(conf.Db[db.PolicyDb]).To(Equal( + db.DatabaseConfig{ + URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", + MaxOpenConnections: 0, + MaxIdleConnections: 0, + ConnectionMaxLifetime: 0 * time.Second, + })) + Expect(conf.ScalingRules.CPU.LowerThreshold).To(Equal(1)) + Expect(conf.ScalingRules.CPU.UpperThreshold).To(Equal(100)) + Expect(conf.ScalingRules.CPUUtil.LowerThreshold).To(Equal(1)) + Expect(conf.ScalingRules.CPUUtil.UpperThreshold).To(Equal(100)) + Expect(conf.ScalingRules.DiskUtil.LowerThreshold).To(Equal(1)) + Expect(conf.ScalingRules.DiskUtil.UpperThreshold).To(Equal(100)) + Expect(conf.ScalingRules.Disk.LowerThreshold).To(Equal(1)) + Expect(conf.ScalingRules.Disk.UpperThreshold).To(Equal(2 * 1024)) + }) + }) + + Context("when max_amount of rate_limit is not an integer", func() { + BeforeEach(func() { + configBytes = []byte(` +rate_limit: + max_amount: NOT-INTEGER +`) + }) + It("should error", func() { + Expect(err).To(MatchError(MatchRegexp("failed to read config file"))) + }) }) - }) - Context("when the broker username hash is set to an invalid value", func() { - BeforeEach(func() { - conf.BrokerCredentials[0].BrokerUsernameHash = []byte("not a bcrypt hash") - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: broker_username_hash is not a valid bcrypt hash"))) - }) - }) + Context("when valid_duration of rate_limit is not a time duration", func() { + BeforeEach(func() { + configBytes = []byte(` +rate_limit: + valid_duration: NOT-TIME-DURATION +`) + }) - Context("when neither the broker password nor its hash is set", func() { - BeforeEach(func() { - conf.BrokerCredentials[0].BrokerPassword = "" - conf.BrokerCredentials[0].BrokerPasswordHash = []byte("") + It("should error", func() { + Expect(err).To(MatchError(MatchRegexp("failed to read config file"))) + }) }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_password and broker_password_hash are empty, please provide one of them"))) - }) - }) - Context("when both the broker password and its hash are set", func() { - BeforeEach(func() { - conf.BrokerCredentials[0].BrokerPassword = "broker_password" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_password and broker_password_hash are set, please provide only one of them"))) - }) }) - Context("when just the broker password is set", func() { + Describe("Validate", func() { BeforeEach(func() { - conf.BrokerCredentials[0].BrokerPassword = "broker_password" - conf.BrokerCredentials[0].BrokerPasswordHash = []byte("") - }) - It("should not error", func() { - Expect(err).NotTo(HaveOccurred()) - }) - }) + conf = &Config{} + conf.Db = make(map[string]db.DatabaseConfig) + conf.Db[db.BindingDb] = db.DatabaseConfig{ + URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", + MaxOpenConnections: 10, + MaxIdleConnections: 5, + ConnectionMaxLifetime: 60 * time.Second, + } + conf.Db[db.PolicyDb] = db.DatabaseConfig{ + URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", + MaxOpenConnections: 10, + MaxIdleConnections: 5, + ConnectionMaxLifetime: 60 * time.Second, + } - Context("when the broker password hash is set to an invalid value", func() { - BeforeEach(func() { brokerCred1 := BrokerCredentialsConfig{ - BrokerUsername: "broker_username", - BrokerPasswordHash: []byte("not a bcrypt hash"), + BrokerUsernameHash: []byte("$2a$10$WNO1cPko4iDAT6MkhaDojeJMU8ZdNH6gt.SapsFOsC0OF4cQ9qQwu"), // ruby -r bcrypt -e 'puts BCrypt::Password.create("broker_username")' + BrokerPasswordHash: []byte("$2a$10$evLviRLcIPKnWQqlBl3DJOvBZir9vJ4gdEeyoGgvnK/CGBnxIAFRu"), // ruby -r bcrypt -e 'puts BCrypt::Password.create("broker_password")' } var brokerCreds []BrokerCredentialsConfig brokerCreds = append(brokerCreds, brokerCred1) conf.BrokerCredentials = brokerCreds - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: broker_password_hash is not a valid bcrypt hash"))) - }) - }) - Context("when eventgenerator url is not set", func() { - BeforeEach(func() { - conf.EventGenerator.EventGeneratorUrl = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: event_generator.event_generator_url is empty"))) - }) - }) - - Context("when scalingengine url is not set", func() { - BeforeEach(func() { - conf.ScalingEngine.ScalingEngineUrl = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: scaling_engine.scaling_engine_url is empty"))) - }) - }) - - Context("when metricsforwarder url is not set", func() { - BeforeEach(func() { - conf.MetricsForwarder.MetricsForwarderUrl = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: metrics_forwarder.metrics_forwarder_url is empty"))) - }) - }) + conf.CatalogSchemaPath = "../schemas/catalog.schema.json" + conf.CatalogPath = "../exampleconfig/catalog-example.json" + conf.PolicySchemaPath = "../exampleconfig/policy.schema.json" + + conf.Scheduler.SchedulerURL = "https://localhost:8083" + + conf.ScalingEngine.ScalingEngineUrl = "https://localhost:8084" + conf.EventGenerator.EventGeneratorUrl = "https://localhost:8085" + conf.MetricsForwarder.MetricsForwarderUrl = "https://localhost:8088" + + conf.CF.API = "https://api.bosh-lite.com" + conf.CF.ClientID = "client-id" + conf.CF.Secret = "secret" + + conf.InfoFilePath = "../exampleconfig/info-file.json" + + conf.RateLimit.MaxAmount = 10 + conf.RateLimit.ValidDuration = 1 * time.Second + + conf.CredHelperImpl = "path/to/plugin" + }) + JustBeforeEach(func() { + err = conf.Validate() + }) + + Context("When all the configs are valid", func() { + It("should not error", func() { + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when bindingdb url is not set", func() { + BeforeEach(func() { + conf.Db[db.BindingDb] = db.DatabaseConfig{URL: ""} + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: BindingDB URL is empty"))) + }) + }) + + Context("when policydb url is not set", func() { + BeforeEach(func() { + conf.Db[db.PolicyDb] = db.DatabaseConfig{URL: ""} + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: PolicyDB URL is empty"))) + }) + }) + + Context("when scheduler url is not set", func() { + BeforeEach(func() { + conf.Scheduler.SchedulerURL = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: scheduler.scheduler_url is empty"))) + }) + }) + + Context("when neither the broker username nor its hash is set", func() { + BeforeEach(func() { + brokerCred1 := BrokerCredentialsConfig{ + BrokerPasswordHash: []byte(""), + BrokerPassword: "", + } + var brokerCreds []BrokerCredentialsConfig + brokerCreds = append(brokerCreds, brokerCred1) + conf.BrokerCredentials = brokerCreds + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_username and broker_username_hash are empty, please provide one of them"))) + }) + }) + + Context("when both the broker username and its hash are set", func() { + BeforeEach(func() { + brokerCred1 := BrokerCredentialsConfig{ + BrokerUsername: "broker_username", + BrokerUsernameHash: []byte("$2a$10$WNO1cPko4iDAT6MkhaDojeJMU8ZdNH6gt.SapsFOsC0OF4cQ9qQwu"), + } + var brokerCreds []BrokerCredentialsConfig + brokerCreds = append(brokerCreds, brokerCred1) + conf.BrokerCredentials = brokerCreds + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_username and broker_username_hash are set, please provide only one of them"))) + }) + }) + + Context("when just the broker username is set", func() { + BeforeEach(func() { + conf.BrokerCredentials[0].BrokerUsername = "broker_username" + conf.BrokerCredentials[0].BrokerUsernameHash = []byte("") + }) + It("should not error", func() { + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when the broker username hash is set to an invalid value", func() { + BeforeEach(func() { + conf.BrokerCredentials[0].BrokerUsernameHash = []byte("not a bcrypt hash") + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: broker_username_hash is not a valid bcrypt hash"))) + }) + }) + + Context("when neither the broker password nor its hash is set", func() { + BeforeEach(func() { + conf.BrokerCredentials[0].BrokerPassword = "" + conf.BrokerCredentials[0].BrokerPasswordHash = []byte("") + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_password and broker_password_hash are empty, please provide one of them"))) + }) + }) + + Context("when both the broker password and its hash are set", func() { + BeforeEach(func() { + conf.BrokerCredentials[0].BrokerPassword = "broker_password" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: both broker_password and broker_password_hash are set, please provide only one of them"))) + }) + }) + + Context("when just the broker password is set", func() { + BeforeEach(func() { + conf.BrokerCredentials[0].BrokerPassword = "broker_password" + conf.BrokerCredentials[0].BrokerPasswordHash = []byte("") + }) + It("should not error", func() { + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when the broker password hash is set to an invalid value", func() { + BeforeEach(func() { + brokerCred1 := BrokerCredentialsConfig{ + BrokerUsername: "broker_username", + BrokerPasswordHash: []byte("not a bcrypt hash"), + } + var brokerCreds []BrokerCredentialsConfig + brokerCreds = append(brokerCreds, brokerCred1) + conf.BrokerCredentials = brokerCreds + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: broker_password_hash is not a valid bcrypt hash"))) + }) + }) + + Context("when eventgenerator url is not set", func() { + BeforeEach(func() { + conf.EventGenerator.EventGeneratorUrl = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: event_generator.event_generator_url is empty"))) + }) + }) + + Context("when scalingengine url is not set", func() { + BeforeEach(func() { + conf.ScalingEngine.ScalingEngineUrl = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: scaling_engine.scaling_engine_url is empty"))) + }) + }) + + Context("when metricsforwarder url is not set", func() { + BeforeEach(func() { + conf.MetricsForwarder.MetricsForwarderUrl = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: metrics_forwarder.metrics_forwarder_url is empty"))) + }) + }) + + Context("when catalog schema path is not set", func() { + BeforeEach(func() { + conf.CatalogSchemaPath = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: CatalogSchemaPath is empty"))) + }) + }) + + Context("when catalog path is not set", func() { + BeforeEach(func() { + conf.CatalogPath = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: CatalogPath is empty"))) + }) + }) + + Context("when policy schema path is not set", func() { + BeforeEach(func() { + conf.PolicySchemaPath = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: PolicySchemaPath is empty"))) + }) + }) + + Context("when catalog is not valid json", func() { + BeforeEach(func() { + conf.CatalogPath = "../exampleconfig/catalog-invalid-json-example.json" + }) + It("should err", func() { + Expect(err).To(MatchError("invalid character '[' after object key")) + }) + }) + + Context("when catalog is missing required fields", func() { + BeforeEach(func() { + conf.CatalogPath = "../exampleconfig/catalog-missing-example.json" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("{\"name is required\"}"))) + }) + }) + + Context("when catalog has invalid type fields", func() { + BeforeEach(func() { + conf.CatalogPath = "../exampleconfig/catalog-invalid-example.json" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("{\"Invalid type. Expected: boolean, given: integer\"}"))) + }) + }) + + Context("when info_file_path is not set", func() { + BeforeEach(func() { + conf.InfoFilePath = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: InfoFilePath is empty"))) + }) + }) + + Context("when cf.client_id is not set", func() { + BeforeEach(func() { + conf.CF.ClientID = "" + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: client_id is empty"))) + }) + }) + + Context("when rate_limit.max_amount is <= zero", func() { + BeforeEach(func() { + conf.RateLimit.MaxAmount = 0 + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.MaxAmount is equal or less than zero"))) + }) + }) - Context("when catalog schema path is not set", func() { - BeforeEach(func() { - conf.CatalogSchemaPath = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: CatalogSchemaPath is empty"))) + Context("when rate_limit.valid_duration is <= 0 ns", func() { + BeforeEach(func() { + conf.RateLimit.ValidDuration = 0 * time.Nanosecond + }) + It("should err", func() { + Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.ValidDuration is equal or less than zero nanosecond"))) + }) }) - }) - Context("when catalog path is not set", func() { - BeforeEach(func() { - conf.CatalogPath = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: CatalogPath is empty"))) - }) }) - - Context("when policy schema path is not set", func() { - BeforeEach(func() { - conf.PolicySchemaPath = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: PolicySchemaPath is empty"))) - }) - }) - - Context("when catalog is not valid json", func() { - BeforeEach(func() { - conf.CatalogPath = "../exampleconfig/catalog-invalid-json-example.json" - }) - It("should err", func() { - Expect(err).To(MatchError("invalid character '[' after object key")) - }) - }) - - Context("when catalog is missing required fields", func() { - BeforeEach(func() { - conf.CatalogPath = "../exampleconfig/catalog-missing-example.json" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("{\"name is required\"}"))) - }) - }) - - Context("when catalog has invalid type fields", func() { - BeforeEach(func() { - conf.CatalogPath = "../exampleconfig/catalog-invalid-example.json" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("{\"Invalid type. Expected: boolean, given: integer\"}"))) - }) - }) - - Context("when info_file_path is not set", func() { - BeforeEach(func() { - conf.InfoFilePath = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: InfoFilePath is empty"))) - }) - }) - - Context("when cf.client_id is not set", func() { - BeforeEach(func() { - conf.CF.ClientID = "" - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: client_id is empty"))) - }) - }) - - Context("when rate_limit.max_amount is <= zero", func() { - BeforeEach(func() { - conf.RateLimit.MaxAmount = 0 - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.MaxAmount is equal or less than zero"))) - }) - }) - - Context("when rate_limit.valid_duration is <= 0 ns", func() { - BeforeEach(func() { - conf.RateLimit.ValidDuration = 0 * time.Nanosecond - }) - It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.ValidDuration is equal or less than zero nanosecond"))) - }) - }) - }) }) diff --git a/src/autoscaler/api/default_catalog.json b/src/autoscaler/api/default_catalog.json new file mode 100644 index 0000000000..a1d6f7f753 --- /dev/null +++ b/src/autoscaler/api/default_catalog.json @@ -0,0 +1,29 @@ +{ + "services": [ + { + "bindable": true, + "bindings_retrievable": true, + "description": "Automatically increase or decrease the number of application instances based on a policy you define.", + "id": "autoscaler-guid", + "instances_retrievable": true, + "name": "autoscaler-3119", + "plans": [ + { + "description": "This is the free service plan for the Auto-Scaling service.", + "id": "autoscaler-free-plan-id", + "name": "autoscaler-free-plan", + "plan_updateable": true + }, + { + "description": "This is the standard service plan for the Auto-Scaling service.", + "id": "acceptance-standard", + "name": "acceptance-standard", + "plan_updateable": false + } + ], + "tags": [ + "app-autoscaler" + ] + } + ] +} diff --git a/src/autoscaler/api/default_config.json b/src/autoscaler/api/default_config.json new file mode 100644 index 0000000000..3d68b2eef0 --- /dev/null +++ b/src/autoscaler/api/default_config.json @@ -0,0 +1,85 @@ +{ + "publicapiserver": { + "cf": { + "api": "https://api.autoscaler.app-runtime-interfaces.ci.cloudfoundry.org", + "client_id": "autoscaler_client_id", + "secret": "autoscaler_client_secret", + "skip_ssl_validation": false, + "max_retries": 3, + "max_retry_wait_ms": 0, + "idle_connection_timeout_ms": 5000, + "max_idle_conns_per_host_ms": 200 + }, + "broker_server": { + "port": 6102 + }, + "broker_credentials": [ + { + "broker_password": "REPLACE_ME", + "broker_username": "autoscaler-broker-user" + }, + { + "broker_password": "REPLACE_ME", + "broker_username": "autoscaler-broker-user-blue" + } + ], + "catalog_path": "/home/vcap/app/api/default_catalog.json", + "catalog_schema_path": "/home/vcap/app/api/schemas/catalog.schema.json", + "info_file_path": "/home/vcap/app/api/default_info.json", + "policy_schema_path": "/home/vcap/app/api/policyvalidator/policy_json.schema.json", + "dashboard_redirect_uri": null, + "default_credential_type": "binding-secret", + "health": { + "server_config": { + "port": 1080 + } + }, + "scaling_engine": { + "scaling_engine_url": "https://autoscaler-3119.scalingengine.service.cf.internal:6104", + "tls": { + "key_file": "/var/vcap/jobs/golangapiserver/config/certs/scalingengine/client.key", + "cert_file": "/var/vcap/jobs/golangapiserver/config/certs/scalingengine/client.crt", + "ca_file": "/var/vcap/jobs/golangapiserver/config/certs/scalingengine/ca.crt" + } + }, + "scheduler": { + "scheduler_url": "https://autoscaler-3119.autoscalerscheduler.service.cf.internal:6102", + "tls": { + "key_file": "/var/vcap/jobs/golangapiserver/config/certs/scheduler/client.key", + "cert_file": "/var/vcap/jobs/golangapiserver/config/certs/scheduler/client.crt", + "ca_file": "/var/vcap/jobs/golangapiserver/config/certs/scheduler/ca.crt" + } + }, + "event_generator": { + "event_generator_url": "https://autoscaler-3119.eventgenerator.service.cf.internal:6105", + "tls": { + "key_file": "/var/vcap/jobs/golangapiserver/config/certs/eventgenerator/client.key", + "cert_file": "/var/vcap/jobs/golangapiserver/config/certs/eventgenerator/client.crt", + "ca_file": "/var/vcap/jobs/golangapiserver/config/certs/eventgenerator/ca.crt" + } + }, + "scaling_rules": { + "cpu": { + "lower_threshold": 1, + "upper_threshold": 100 + }, + "cpuutil": { + "lower_threshold": 1, + "upper_threshold": 100 + }, + "diskutil": { + "lower_threshold": 1, + "upper_threshold": 100 + }, + "disk": { + "lower_threshold": 1, + "upper_threshold": 2048 + } + }, + "rate_limit": { + "valid_duration": "1s", + "max_amount": 10 + }, + "cred_helper_impl": "default" + } +} diff --git a/src/autoscaler/api/default_info.json b/src/autoscaler/api/default_info.json new file mode 100644 index 0000000000..918bd6550b --- /dev/null +++ b/src/autoscaler/api/default_info.json @@ -0,0 +1,8 @@ + + +{ + "name": "Autoscaler", + "build": "14.3.0", + "support": "https://github.com/cloudfoundry/app-autoscaler-release", + "description": "Automatically increase or decrease the number of application instances based on a policy you define." +} diff --git a/src/autoscaler/api/publicapiserver/public_api_handler.go b/src/autoscaler/api/publicapiserver/public_api_handler.go index e8c7c485f1..f764d710f2 100644 --- a/src/autoscaler/api/publicapiserver/public_api_handler.go +++ b/src/autoscaler/api/publicapiserver/public_api_handler.go @@ -29,7 +29,6 @@ type PublicApiHandler struct { conf *config.Config policydb db.PolicyDB bindingdb db.BindingDB - scalingEngineClient *http.Client eventGeneratorClient *http.Client policyValidator *policyvalidator.PolicyValidator schedulerUtil *schedulerclient.Client @@ -43,13 +42,7 @@ const ( ) func NewPublicApiHandler(logger lager.Logger, conf *config.Config, policydb db.PolicyDB, bindingdb db.BindingDB, credentials cred_helper.Credentials) *PublicApiHandler { - seClient, err := helpers.CreateHTTPClient(&conf.ScalingEngine.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scaling_client")) - if err != nil { - logger.Error("Failed to create http client for ScalingEngine", err, lager.Data{"scalingengine": conf.ScalingEngine.TLSClientCerts}) - os.Exit(1) - } - - egClient, err := helpers.CreateHTTPClient(&conf.EventGenerator.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("event_client")) + egClient, err := helpers.CreateHTTPSClient(&conf.EventGenerator.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("event_client")) if err != nil { logger.Error("Failed to create http client for EventGenerator", err, lager.Data{"eventgenerator": conf.EventGenerator.TLSClientCerts}) os.Exit(1) @@ -60,7 +53,6 @@ func NewPublicApiHandler(logger lager.Logger, conf *config.Config, policydb db.P conf: conf, policydb: policydb, bindingdb: bindingdb, - scalingEngineClient: seClient, eventGeneratorClient: egClient, policyValidator: policyvalidator.NewPolicyValidator( conf.PolicySchemaPath, @@ -233,44 +225,6 @@ func (h *PublicApiHandler) DetachScalingPolicy(w http.ResponseWriter, r *http.Re } } -func (h *PublicApiHandler) GetScalingHistories(w http.ResponseWriter, req *http.Request, vars map[string]string) { - appId := vars["appId"] - logger := h.logger.Session("GetScalingHistories", lager.Data{"appId": appId}) - logger.Info("Get ScalingHistories") - - // be careful about removing this call! There's some backwards compatibility being done in this function - parameters, err := parseParameter(req, vars) - if err != nil { - logger.Error("bad-request", err, lager.Data{"appId": appId}) - writeErrorResponse(w, http.StatusBadRequest, err.Error()) - return - } - path, _ := routes.ScalingEngineRoutes().Get(routes.GetScalingHistoriesRouteName).URLPath("guid", appId) - targetURL := h.conf.ScalingEngine.ScalingEngineUrl + path.RequestURI() + "?" + parameters.Encode() - - targetRequest, _ := http.NewRequest(http.MethodGet, targetURL, nil) - targetRequest.Header.Set("Authorization", "Bearer none") - - response, err := h.scalingEngineClient.Do(targetRequest) - - if err != nil { - logger.Error("error-getting-scaling-history", err, lager.Data{"url": targetURL}) - writeErrorResponse(w, http.StatusInternalServerError, "Error retrieving scaling history from scaling engine") - return - } - w.Header().Set("Content-Type", response.Header.Get("Content-Type")) - w.Header().Set("Content-Length", response.Header.Get("Content-Length")) - - if _, err := io.Copy(w, response.Body); err != nil { - logger.Error("copy-response", err) - return - } - err = response.Body.Close() - if err != nil { - logger.Error("body-close", err) - } -} - func proxyRequest(pathFn func() string, call func(url string) (*http.Response, error), w http.ResponseWriter, reqUrl *url.URL, parameters *url.Values, requestDescription string, logger lager.Logger) { aUrl := pathFn() resp, err := call(aUrl) diff --git a/src/autoscaler/api/publicapiserver/public_api_server.go b/src/autoscaler/api/publicapiserver/public_api_server.go index 40891dc747..6d52c219ab 100644 --- a/src/autoscaler/api/publicapiserver/public_api_server.go +++ b/src/autoscaler/api/publicapiserver/public_api_server.go @@ -3,13 +3,14 @@ package publicapiserver import ( "fmt" "net/http" + "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cred_helper" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/apis/scalinghistory" "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/apis/scalinghistory" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" @@ -19,6 +20,7 @@ import ( "code.cloudfoundry.org/lager/v3" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" ) @@ -29,22 +31,43 @@ func (vh VarsFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { vh(w, r, vars) } -func NewPublicApiServer(logger lager.Logger, conf *config.Config, policydb db.PolicyDB, credentials cred_helper.Credentials, - checkBindingFunc api.CheckBindingFunc, cfclient cf.CFClient, httpStatusCollector healthendpoint.HTTPStatusCollector, - rateLimiter ratelimiter.Limiter, bindingdb db.BindingDB) (ifrit.Runner, error) { - pah := NewPublicApiHandler(logger, conf, policydb, bindingdb, credentials) +type PublicApiServer struct { + logger lager.Logger + conf *config.Config + policyDB db.PolicyDB + bindingDB db.BindingDB + credentials cred_helper.Credentials + checkBindingFunc api.CheckBindingFunc + cfClient cf.CFClient + httpStatusCollector healthendpoint.HTTPStatusCollector + rateLimiter ratelimiter.Limiter +} + +func (s *PublicApiServer) GetHealthServer() (ifrit.Runner, error) { + healthRouter, err := createHealthRouter(s.logger, s.conf, s.policyDB, s.bindingDB, s.httpStatusCollector) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) + } + + return helpers.NewHTTPServer(s.logger, s.conf.Health.ServerConfig, healthRouter) +} + +func (s *PublicApiServer) GetMtlsServer() (ifrit.Runner, error) { + pah := NewPublicApiHandler(s.logger, s.conf, s.policyDB, s.bindingDB, s.credentials) - scalingHistoryHandler, err := newScalingHistoryHandler(logger, conf) + scalingHistoryHandler, err := newScalingHistoryHandler(s.logger, s.conf) if err != nil { return nil, err } - mw := NewMiddleware(logger, cfclient, checkBindingFunc, conf.APIClientId) - rateLimiterMiddleware := ratelimiter.NewRateLimiterMiddleware("appId", rateLimiter, logger.Session("api-ratelimiter-middleware")) - httpStatusCollectMiddleware := healthendpoint.NewHTTPStatusCollectMiddleware(httpStatusCollector) + mw := NewMiddleware(s.logger, s.cfClient, s.checkBindingFunc, s.conf.APIClientId) + rateLimiterMiddleware := ratelimiter.NewRateLimiterMiddleware("appId", s.rateLimiter, s.logger.Session("api-ratelimiter-middleware")) + httpStatusCollectMiddleware := healthendpoint.NewHTTPStatusCollectMiddleware(s.httpStatusCollector) + r := routes.ApiOpenRoutes() r.Use(otelmux.Middleware("apiserver")) r.Use(httpStatusCollectMiddleware.Collect) + r.Get(routes.PublicApiInfoRouteName).Handler(VarsFunc(pah.GetApiInfo)) r.Get(routes.PublicApiHealthRouteName).Handler(VarsFunc(pah.GetHealth)) @@ -69,15 +92,72 @@ func NewPublicApiServer(logger lager.Logger, conf *config.Config, policydb db.Po rpolicy.Get(routes.PublicApiAttachPolicyRouteName).Handler(VarsFunc(pah.AttachScalingPolicy)) rpolicy.Get(routes.PublicApiDetachPolicyRouteName).Handler(VarsFunc(pah.DetachScalingPolicy)) - return helpers.NewHTTPServer(logger, conf.PublicApiServer, r) + healthRouter, err := createHealthRouter(s.logger, s.conf, s.policyDB, s.bindingDB, s.httpStatusCollector) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) + } + + mainRouter := setupMainRouter(r, healthRouter) + + return helpers.NewHTTPServer(s.logger, s.conf.Server, mainRouter) +} + +func NewPublicApiServer(logger lager.Logger, conf *config.Config, policyDB db.PolicyDB, + bindingDB db.BindingDB, credentials cred_helper.Credentials, checkBindingFunc api.CheckBindingFunc, + cfClient cf.CFClient, httpStatusCollector healthendpoint.HTTPStatusCollector, + rateLimiter ratelimiter.Limiter) *PublicApiServer { + return &PublicApiServer{ + logger: logger, + conf: conf, + policyDB: policyDB, + bindingDB: bindingDB, + credentials: credentials, + checkBindingFunc: checkBindingFunc, + cfClient: cfClient, + httpStatusCollector: httpStatusCollector, + rateLimiter: rateLimiter, + } +} + +func setupMainRouter(r *mux.Router, healthRouter *mux.Router) *mux.Router { + mainRouter := mux.NewRouter() + mainRouter.PathPrefix("/v1").Handler(r) + mainRouter.PathPrefix("/health").Handler(healthRouter) + mainRouter.PathPrefix("/").Handler(healthRouter) + return mainRouter +} + +func createPrometheusRegistry(policyDB db.PolicyDB, bindingDB db.BindingDB, httpStatusCollector healthendpoint.HTTPStatusCollector, logger lager.Logger) *prometheus.Registry { + promRegistry := prometheus.NewRegistry() + healthendpoint.RegisterCollectors(promRegistry, + []prometheus.Collector{ + healthendpoint.NewDatabaseStatusCollector("autoscaler", "golangapiserver", "policyDB", policyDB), + healthendpoint.NewDatabaseStatusCollector("autoscaler", "golangapiserver", "bindingDB", bindingDB), + httpStatusCollector, + }, + true, logger.Session("golangapiserver-prometheus")) + return promRegistry +} + +func createHealthRouter(logger lager.Logger, conf *config.Config, policyDB db.PolicyDB, bindingDB db.BindingDB, httpStatusCollector healthendpoint.HTTPStatusCollector) (*mux.Router, error) { + checkers := []healthendpoint.Checker{} + gatherer := createPrometheusRegistry(policyDB, bindingDB, httpStatusCollector, logger) + healthRouter, err := healthendpoint.NewHealthRouter(conf.Health, checkers, logger.Session("health-server"), gatherer, time.Now) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) + } + + logger.Debug("Successfully created health server") + return healthRouter, nil } func newScalingHistoryHandler(logger lager.Logger, conf *config.Config) (http.Handler, error) { + ss := SecuritySource{} scalingHistoryHandler, err := NewScalingHistoryHandler(logger, conf) if err != nil { return nil, fmt.Errorf("error creating scaling history handler: %w", err) } - scalingHistoryServer, err := scalinghistory.NewServer(scalingHistoryHandler, scalingHistoryHandler) + scalingHistoryServer, err := scalinghistory.NewServer(scalingHistoryHandler, ss) if err != nil { return nil, fmt.Errorf("error creating ogen scaling history server: %w", err) } diff --git a/src/autoscaler/api/publicapiserver/public_api_server_test.go b/src/autoscaler/api/publicapiserver/public_api_server_test.go index 7b17a46518..1751a49ab2 100644 --- a/src/autoscaler/api/publicapiserver/public_api_server_test.go +++ b/src/autoscaler/api/publicapiserver/public_api_server_test.go @@ -6,7 +6,7 @@ import ( "net/url" "strings" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/apis/scalinghistory" + internalscalinghistory "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/apis/scalinghistory" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -42,24 +42,24 @@ var _ = Describe("PublicApiServer", func() { ) BeforeEach(func() { - scalingHistoryEntry := []scalinghistory.HistoryEntry{ + scalingHistoryEntry := []internalscalinghistory.HistoryEntry{ { - Status: scalinghistory.NewOptHistoryEntryStatus(scalinghistory.HistoryEntryStatus0), - AppID: scalinghistory.NewOptGUID(TEST_APP_ID), - Timestamp: scalinghistory.NewOptInt(300), - ScalingType: scalinghistory.NewOptHistoryEntryScalingType(scalinghistory.HistoryEntryScalingType0), - OldInstances: scalinghistory.NewOptInt64(2), - NewInstances: scalinghistory.NewOptInt64(4), - Reason: scalinghistory.NewOptString("a reason"), + Status: internalscalinghistory.NewOptHistoryEntryStatus(internalscalinghistory.HistoryEntryStatus0), + AppID: internalscalinghistory.NewOptGUID(TEST_APP_ID), + Timestamp: internalscalinghistory.NewOptInt(300), + ScalingType: internalscalinghistory.NewOptHistoryEntryScalingType(internalscalinghistory.HistoryEntryScalingType0), + OldInstances: internalscalinghistory.NewOptInt64(2), + NewInstances: internalscalinghistory.NewOptInt64(4), + Reason: internalscalinghistory.NewOptString("a reason"), }, } - scalingEngineResponse = scalinghistory.History{ - TotalResults: scalinghistory.NewOptInt64(1), - TotalPages: scalinghistory.NewOptInt64(1), - Page: scalinghistory.NewOptInt64(1), - PrevURL: scalinghistory.OptURI{}, - NextURL: scalinghistory.OptURI{}, + scalingEngineResponse = internalscalinghistory.History{ + TotalResults: internalscalinghistory.NewOptInt64(1), + TotalPages: internalscalinghistory.NewOptInt64(1), + Page: internalscalinghistory.NewOptInt64(1), + PrevURL: internalscalinghistory.OptURI{}, + NextURL: internalscalinghistory.OptURI{}, Resources: scalingHistoryEntry, } @@ -425,6 +425,7 @@ var _ = Describe("PublicApiServer", func() { }) }) }) + Describe("UnProtected Routes", func() { Context("when calling info endpoint", func() { It("should succeed", func() { @@ -437,12 +438,6 @@ var _ = Describe("PublicApiServer", func() { }) }) }) - - Context("when requesting non existing path", func() { - It("should get 404", func() { - verifyResponse(httpClient, serverUrl, "/non-existing-path", nil, http.MethodGet, "", http.StatusNotFound) - }) - }) }) func verifyResponse(httpClient *http.Client, serverUrl *url.URL, path string, headers map[string]string, httpRequestMethod string, httpRequestBody string, expectResponseStatusCode int) { diff --git a/src/autoscaler/api/publicapiserver/publicapiserver_suite_test.go b/src/autoscaler/api/publicapiserver/publicapiserver_suite_test.go index 4c73bc9d28..455f6c23f6 100644 --- a/src/autoscaler/api/publicapiserver/publicapiserver_suite_test.go +++ b/src/autoscaler/api/publicapiserver/publicapiserver_suite_test.go @@ -11,7 +11,8 @@ import ( "strconv" "testing" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/apis/scalinghistory" + internalscalinghistory "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/apis/scalinghistory" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" "code.cloudfoundry.org/lager/v3/lagertest" . "github.com/onsi/ginkgo/v2" @@ -62,18 +63,18 @@ var ( schedulerStatus int schedulerErrJson string - scalingEngineResponse scalinghistory.History + scalingEngineResponse internalscalinghistory.History metricsCollectorResponse []models.AppInstanceMetric eventGeneratorResponse []models.AppMetric fakeCFClient *fakes.FakeCFClient fakePolicyDB *fakes.FakePolicyDB + fakeBindingDB *fakes.FakeBindingDB fakeRateLimiter *fakes.FakeLimiter fakeCredentials *fakes.FakeCredentials checkBindingFunc api.CheckBindingFunc hasBinding = true apiPort = 0 - testCertDir = "../../../../test-certs" ) func TestPublicapiserver(t *testing.T) { @@ -111,6 +112,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) fakePolicyDB = &fakes.FakePolicyDB{} + fakeBindingDB = &fakes.FakeBindingDB{} checkBindingFunc = func(appId string) bool { return hasBinding } @@ -118,10 +120,13 @@ var _ = BeforeSuite(func() { httpStatusCollector := &fakes.FakeHTTPStatusCollector{} fakeRateLimiter = &fakes.FakeLimiter{} fakeCredentials = &fakes.FakeCredentials{} - httpServer, err := publicapiserver.NewPublicApiServer(lagertest.NewTestLogger("public_apiserver"), conf, - fakePolicyDB, fakeCredentials, - checkBindingFunc, fakeCFClient, - httpStatusCollector, fakeRateLimiter, nil) + + publicApiServer := publicapiserver.NewPublicApiServer( + lagertest.NewTestLogger("public_apiserver"), conf, fakePolicyDB, + fakeBindingDB, fakeCredentials, checkBindingFunc, fakeCFClient, + httpStatusCollector, fakeRateLimiter) + + httpServer, err := publicApiServer.GetMtlsServer() Expect(err).NotTo(HaveOccurred()) serverUrl, err = url.Parse("http://127.0.0.1:" + strconv.Itoa(apiPort)) @@ -177,6 +182,7 @@ func CheckResponse(resp *httptest.ResponseRecorder, statusCode int, errResponse } func CreateConfig(apiServerPort int) *config.Config { + testCertDir := testhelpers.TestCertFolder() return &config.Config{ Logging: helpers.LoggingConfig{ Level: "debug", diff --git a/src/autoscaler/api/publicapiserver/scaling_history_handler.go b/src/autoscaler/api/publicapiserver/scaling_history_handler.go index 2e8ad706d0..3ba6d60e86 100644 --- a/src/autoscaler/api/publicapiserver/scaling_history_handler.go +++ b/src/autoscaler/api/publicapiserver/scaling_history_handler.go @@ -5,37 +5,46 @@ import ( "fmt" "net/http" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/apis/scalinghistory" + internalscalingenginehistory "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/apis/scalinghistory" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/apis/scalinghistory" "code.cloudfoundry.org/lager/v3" ) var ( - _ = scalinghistory.SecurityHandler(&ScalingHistoryHandler{}) - _ = scalinghistory.SecuritySource(&ScalingHistoryHandler{}) + _ = scalinghistory.SecurityHandler(&SecuritySource{}) + _ = scalinghistory.SecuritySource(&SecuritySource{}) ) +type SecuritySource struct{} + +func (h *SecuritySource) BearerAuth(_ context.Context, _ string) (scalinghistory.BearerAuth, error) { + // We are calling the scalingengine server authenticated via mTLS, so no bearer token is necessary. + // Having this function is required by the interface `SecuritySource`in “oas_security_gen”. + return scalinghistory.BearerAuth{Token: "none"}, nil +} + +func (h SecuritySource) HandleBearerAuth(ctx context.Context, operationName string, t scalinghistory.BearerAuth) (context.Context, error) { + // this handler is a no-op, as this handler shall only be available used behind our own auth middleware. + // having this handler is required by the interface `securityhandler` in “oas_security_gen”. + return ctx, nil +} + type ScalingHistoryHandler struct { - logger lager.Logger - conf *config.Config - scalingEngineClient *http.Client - client *scalinghistory.Client + logger lager.Logger + conf *config.Config + client *internalscalingenginehistory.Client } func NewScalingHistoryHandler(logger lager.Logger, conf *config.Config) (*ScalingHistoryHandler, error) { - seClient, err := helpers.CreateHTTPClient(&conf.ScalingEngine.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scaling_client")) - if err != nil { - return nil, fmt.Errorf("error creating scaling history HTTP client: %w", err) - } - newHandler := &ScalingHistoryHandler{ - logger: logger.Session("scaling-history-handler"), - conf: conf, - scalingEngineClient: seClient, + logger: logger.Session("scaling-history-handler"), + conf: conf, } - if client, err := scalinghistory.NewClient(conf.ScalingEngine.ScalingEngineUrl, newHandler, scalinghistory.WithClient(seClient)); err != nil { + if client, err := internalscalingenginehistory.NewClient(conf.ScalingEngine.ScalingEngineUrl); err != nil { return nil, fmt.Errorf("error creating ogen scaling history client: %w", err) } else { newHandler.client = client @@ -54,26 +63,39 @@ func (h *ScalingHistoryHandler) NewError(_ context.Context, _ error) *scalinghis return result } -func (h *ScalingHistoryHandler) HandleBearerAuth(ctx context.Context, operationName string, t scalinghistory.BearerAuth) (context.Context, error) { - // This handler is a no-op, as this handler shall only be available used behind our own auth middleware. - // Having this handler is required by the interface `SecurityHandler` in “oas_security_gen”. - return ctx, nil -} - func (h *ScalingHistoryHandler) V1AppsGUIDScalingHistoriesGet(ctx context.Context, params scalinghistory.V1AppsGUIDScalingHistoriesGetParams) (*scalinghistory.History, error) { + result := &scalinghistory.History{} logger := h.logger.Session("get-scaling-histories", helpers.AddTraceID(ctx, lager.Data{"app_guid": params.GUID})) logger.Info("start") defer logger.Info("end") - result, err := h.client.V1AppsGUIDScalingHistoriesGet(ctx, params) + internalParams := internalscalingenginehistory.V1AppsGUIDScalingHistoriesGetParams{ + GUID: internalscalingenginehistory.GUID(params.GUID), + StartTime: internalscalingenginehistory.OptInt(params.StartTime), + EndTime: internalscalingenginehistory.OptInt(params.EndTime), + OrderDirection: internalscalingenginehistory.OptV1AppsGUIDScalingHistoriesGetOrderDirection{ + Value: internalscalingenginehistory.V1AppsGUIDScalingHistoriesGetOrderDirection(params.OrderDirection.Value), + Set: params.OrderDirection.Set, + }, + Page: internalscalingenginehistory.OptInt(params.Page), + ResultsPerPage: internalscalingenginehistory.OptInt(params.ResultsPerPage), + } + internalResult, err := h.client.V1AppsGUIDScalingHistoriesGet(ctx, internalParams) if err != nil { logger.Error("get", err) + return nil, err + } + jsonResult, err := internalResult.MarshalJSON() + if err != nil { + logger.Error("marshal", err) + return nil, err } - return result, err -} -func (h *ScalingHistoryHandler) BearerAuth(_ context.Context, _ string) (scalinghistory.BearerAuth, error) { - // We are calling the scalingengine server authenticated via mTLS, so no bearer token is necessary. - // Having this function is required by the interface `SecuritySource`in “oas_security_gen”. - return scalinghistory.BearerAuth{Token: "none"}, nil + err = result.UnmarshalJSON(jsonResult) + if err != nil { + logger.Error("unmarshal", err) + return nil, err + } + + return result, err } diff --git a/src/autoscaler/api/schedulerclient/client.go b/src/autoscaler/api/schedulerclient/client.go index 214601296e..86c0632932 100644 --- a/src/autoscaler/api/schedulerclient/client.go +++ b/src/autoscaler/api/schedulerclient/client.go @@ -25,7 +25,7 @@ type Client struct { func New(conf *config.Config, logger lager.Logger) *Client { logger = logger.Session("schedulerclient") - client, err := helpers.CreateHTTPClient(&conf.Scheduler.TLSClientCerts, helpers.DefaultClientConfig(), logger) + client, err := helpers.CreateHTTPSClient(&conf.Scheduler.TLSClientCerts, helpers.DefaultClientConfig(), logger) if err != nil { logger.Error("Failed to create http client for Scheduler", err, lager.Data{"scheduler": conf.Scheduler.TLSClientCerts}) os.Exit(1) diff --git a/src/autoscaler/build-extension-file.sh b/src/autoscaler/build-extension-file.sh index a1adb01176..e85fe1abf6 100755 --- a/src/autoscaler/build-extension-file.sh +++ b/src/autoscaler/build-extension-file.sh @@ -20,7 +20,11 @@ export POSTGRES_ADDRESS="${DEPLOYMENT_NAME}-postgres.tcp.${SYSTEM_DOMAIN}" export POSTGRES_EXTERNAL_PORT="${PR_NUMBER:-5432}" export METRICSFORWARDER_HEALTH_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/autoscaler_metricsforwarder_health_password --quiet)" -export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}" + +export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"${DEPLOYMENT_NAME}metrics"}" +export METRICSFORWARDER_MTLS_HOST="${METRICSFORWARDER_MTLS_HOST:-"${DEPLOYMENT_NAME}-metricsforwarder-mtls"}" +export PUBLICAPISERVER_HOST="${PUBLICAPISERVER_HOST:-"${DEPLOYMENT_NAME}"}" +export SERVICEBROKER_HOST="${SERVICEBROKER_HOST:-"${DEPLOYMENT_NAME}servicebroker"}" export POLICY_DB_PASSWORD="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/database_password --quiet)" export POLICY_DB_SERVER_CA="$(credhub get -n /bosh-autoscaler/${DEPLOYMENT_NAME}/postgres_server --key ca --quiet )" @@ -40,15 +44,24 @@ _schema-version: 3.3.0 modules: - name: metricsforwarder requires: - - name: config + - name: metricsforwarder-config - name: policydb - name: syslog-client parameters: routes: - - route: ${METRICSFORWARDER_APPNAME}.\${default-domain} + - route: ${METRICSFORWARDER_HOST}.\${default-domain} + - route: ${METRICSFORWARDER_MTLS_HOST}.\${default-domain} + + + - name: publicapiserver + parameters: + instances: 0 + routes: + - route: ${PUBLICAPISERVER_HOST}.\${default-domain} + - route: ${SERVICEBROKER_HOST}.\${default-domain} resources: -- name: config +- name: metricsforwarder-config parameters: config: metricsforwarder: @@ -67,4 +80,13 @@ resources: client_cert: "${SYSLOG_CLIENT_CERT//$'\n'/\\n}" client_key: "${SYSLOG_CLIENT_KEY//$'\n'/\\n}" server_ca: "${SYSLOG_CLIENT_CA//$'\n'/\\n}" +- name: publicapiserver-config + parameters: + config: + publicapiserver: + cf: + skip_ssl_validation: true + metrics_forwarder: + metrics_forwarder_url: ${METRICSFORWARDER_HOST}.\${default-domain} + metrics_forwarder_mtls_url: ${METRICSFORWARDER_MTLS_HOST}.\${default-domain} EOF diff --git a/src/autoscaler/eventgenerator/Makefile b/src/autoscaler/eventgenerator/Makefile new file mode 100644 index 0000000000..af77ad173d --- /dev/null +++ b/src/autoscaler/eventgenerator/Makefile @@ -0,0 +1,32 @@ +.PHONY: fetch-config +fetch-config: start-metricsforwarder-vm + # how to define variables in deployment name + mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client + + echo "POSTGRES IP: $(POSTGRES_IP)" + echo "LOG_CACHE IP: $(LOG_CACHE_IP)" + + @echo "Pulling metricforwarder config from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml assets/metricsforwarder.yml + + @echo "Pulling policy db certs from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt assets/certs/policy_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt assets/certs/policy_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key assets/certs/policy_db/. + + @echo "Pulling storeprocedure db certs from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt assets/certs/storedprocedure_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt assets/certs/storedprocedure_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key assets/certs/storedprocedure_db/. + + @echo "Pulling syslog-client certs from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt assets/certs/syslog_client/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt assets/certs/syslog_client/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key assets/certs/syslog_client/. + + @echo "Build metricsforwarder config yaml" + cp assets/metricsforwarder.yml metricsforwarder.yml + + # remove SERVER TLS CONFIG so that it starts a http server + sed -i'' -e 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml + sed -i'' -e 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml diff --git a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go index 0becab0103..6d9229af76 100644 --- a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go +++ b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go @@ -70,7 +70,8 @@ var _ = Describe("MetricPoller", func() { It("logs an error", func() { //TODO this should be a prometheus counter not a log statement check - Eventually(logger.Buffer).Should(Say("retrieveMetric Failed")) + Eventually(logger.Buffer, 2*time.Second).Should(Say("retrieveMetric Failed")) + }) It("does not save any metrics", func() { diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go index dfd948e126..28bbd37062 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go @@ -73,7 +73,10 @@ var _ = SynchronizedBeforeSuite(func() []byte { }) var _ = SynchronizedAfterSuite(func() { - _ = os.Remove(configFile.Name()) + if configFile != nil { + err := os.Remove(configFile.Name()) + Expect(err).NotTo(HaveOccurred()) + } }, func() { gexec.CleanupBuildArtifacts() }) @@ -312,8 +315,10 @@ func initConfig() { ServerConfig: helpers.ServerConfig{ Port: healthport, }, - HealthCheckUsername: "healthcheckuser", - HealthCheckPassword: "healthcheckpassword", + BasicAuth: models.BasicAuth{ + Username: "healthcheckuser", + Password: "healthcheckpassword", + }, }, } configFile = writeConfig(&conf) diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go index b9d1a86a9b..5cd5847861 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go @@ -1,14 +1,16 @@ package main_test import ( - "fmt" "io" "net/http" + "net/url" "os" + "strconv" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -19,11 +21,27 @@ import ( var _ = Describe("Eventgenerator", func() { var ( - runner *EventGeneratorRunner + runner *EventGeneratorRunner + httpClientForEventGenerator *http.Client + httpClientForHealth *http.Client + + serverURL *url.URL + healthURL *url.URL + + err error ) BeforeEach(func() { runner = NewEventGeneratorRunner() + + httpClientForEventGenerator = testhelpers.NewEventGeneratorClient() + httpClientForHealth = &http.Client{} + + serverURL, err = url.Parse("https://127.0.0.1:" + strconv.Itoa(conf.Server.Port)) + healthURL, err = url.Parse("http://127.0.0.1:" + strconv.Itoa(conf.Health.ServerConfig.Port)) + + Expect(err).ToNot(HaveOccurred()) + }) AfterEach(func() { @@ -111,7 +129,7 @@ var _ = Describe("Eventgenerator", func() { }) }) - Context("when an interrupt is sent", func() { + When("an interrupt is sent", func() { BeforeEach(func() { runner.Start() }) @@ -123,113 +141,119 @@ var _ = Describe("Eventgenerator", func() { }) Describe("EventGenerator REST API", func() { - Context("when a request for aggregated metrics history comes", func() { + When("a request for aggregated metrics history comes", func() { BeforeEach(func() { + serverURL.Path = "/v1/apps/an-app-id/aggregated_metric_histories/a-metric-type" runner.Start() }) It("returns with a 200", func() { - rsp, err := httpClient.Get(fmt.Sprintf("https://127.0.0.1:%d/v1/apps/an-app-id/aggregated_metric_histories/a-metric-type", egPort)) + rsp, err := httpClientForEventGenerator.Get(serverURL.String()) Expect(err).NotTo(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) rsp.Body.Close() }) - }) - }) - Describe("when Health server is ready to serve RESTful API", func() { + Describe("EventGenerator Health endpoint", func() { + BeforeEach(func() { - basicAuthConfig := conf - basicAuthConfig.Health.HealthCheckUsername = "" - basicAuthConfig.Health.HealthCheckPassword = "" - runner.configPath = writeConfig(&basicAuthConfig).Name() + serverURL.Path = "/health" + }) - runner.Start() + When("Health server is ready to serve RESTful API", func() { + BeforeEach(func() { + basicAuthConfig := conf + basicAuthConfig.Health.BasicAuth.Username = "" + basicAuthConfig.Health.BasicAuth.Password = "" + runner.configPath = writeConfig(&basicAuthConfig).Name() - }) + runner.Start() - Context("when a request to query health comes", func() { - It("returns with a 200", func() { - rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d/health", healthport)) - Expect(err).NotTo(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - raw, _ := io.ReadAll(rsp.Body) - healthData := string(raw) - Expect(healthData).To(ContainSubstring("autoscaler_eventgenerator_concurrent_http_request")) - Expect(healthData).To(ContainSubstring("autoscaler_eventgenerator_policyDB")) - Expect(healthData).To(ContainSubstring("autoscaler_eventgenerator_appMetricDB")) - Expect(healthData).To(ContainSubstring("go_goroutines")) - Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) - rsp.Body.Close() + }) + When("a request to query health comes", func() { + It("returns with a 200", func() { + rsp, err := httpClientForHealth.Get(healthURL.String()) + Expect(err).NotTo(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + + raw, err := io.ReadAll(rsp.Body) + Expect(err).NotTo(HaveOccurred()) + + healthData := string(raw) + Expect(healthData).To(ContainSubstring("autoscaler_eventgenerator_concurrent_http_request")) + Expect(healthData).To(ContainSubstring("autoscaler_eventgenerator_policyDB")) + Expect(healthData).To(ContainSubstring("autoscaler_eventgenerator_appMetricDB")) + Expect(healthData).To(ContainSubstring("go_goroutines")) + Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) + rsp.Body.Close() + }) }) }) - }) - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - BeforeEach(func() { - runner.Start() - }) - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { + When("Health server is ready to serve RESTful API with basic Auth", func() { + BeforeEach(func() { + runner.Start() + }) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) + When("username and password are incorrect for basic authentication during health check", func() { + It("should return 401", func() { + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) + Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth("wrongusername", "wrongpassword") + req.SetBasicAuth("wrongusername", "wrongpassword") - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) + rsp, err := httpClientForHealth.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) }) - }) - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200", func() { + When("username and password are correct for basic authentication during health check", func() { + It("should return 200", func() { + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) + Expect(err).NotTo(HaveOccurred()) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) + req.SetBasicAuth(conf.Health.BasicAuth.Username, conf.Health.BasicAuth.Password) - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + rsp, err := httpClientForHealth.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + }) }) }) - }) - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - BeforeEach(func() { - runner.Start() - }) - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { + When("Health server is ready to serve RESTful API with basic Auth", func() { + BeforeEach(func() { + runner.Start() + }) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) + When("username and password are incorrect for basic authentication during health check", func() { + It("should return 401", func() { + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) + Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth("wrongusername", "wrongpassword") + req.SetBasicAuth("wrongusername", "wrongpassword") - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) + rsp, err := httpClientForHealth.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) }) - }) - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200", func() { + When("username and password are correct for basic authentication during health check", func() { + It("should return 200", func() { + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) + Expect(err).NotTo(HaveOccurred()) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) + req.SetBasicAuth(conf.Health.BasicAuth.Username, conf.Health.BasicAuth.Password) - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + rsp, err := httpClientForHealth.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + }) }) }) }) diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go index d16ec0196b..47c41d7f5e 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go @@ -2,7 +2,6 @@ package main import ( "io" - "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db/sqldb" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/aggregator" @@ -13,6 +12,7 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "github.com/prometheus/client_golang/prometheus" circuit "github.com/rubyist/circuitbreaker" "flag" @@ -21,7 +21,6 @@ import ( "code.cloudfoundry.org/clock" "code.cloudfoundry.org/lager/v3" - "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" "github.com/tedsuo/ifrit/grouper" "github.com/tedsuo/ifrit/sigmon" @@ -103,19 +102,22 @@ func main() { eventGenerator := ifrit.RunFunc(runFunc(appManager, evaluators, evaluationManager, metricPollers, anAggregator)) - httpServer, err := server.NewServer(logger.Session("http_server"), conf, appManager.QueryAppMetrics, httpStatusCollector) + httpServer := server.NewServer(logger.Session("http_server"), conf, appMetricDB, policyDb, appManager.QueryAppMetrics, httpStatusCollector) + + vmServer, err := httpServer.GetMtlsServer() if err != nil { logger.Error("failed to create http server", err) os.Exit(1) } - healthServer, err := healthendpoint.NewServerWithBasicAuth(conf.Health, []healthendpoint.Checker{}, logger.Session("health-server"), promRegistry, time.Now) + + healthServer, err := httpServer.GetHealthServer() if err != nil { logger.Error("failed to create health server", err) os.Exit(1) } members := grouper.Members{ {"eventGenerator", eventGenerator}, - {"http_server", httpServer}, + {"https_server", vmServer}, {"health_server", healthServer}, } monitor := ifrit.Invoke(sigmon.New(grouper.NewOrdered(os.Interrupt, members))) @@ -162,7 +164,8 @@ func loadConfig(path string) (*config.Config, error) { } configFileBytes, err := io.ReadAll(configFile) - _ = configFile.Close() + defer func() { _ = configFile.Close() }() + if err != nil { return nil, fmt.Errorf("failed to read data from config file %q: %w", path, err) } @@ -182,7 +185,7 @@ func loadConfig(path string) (*config.Config, error) { func createEvaluators(logger lager.Logger, conf *config.Config, triggersChan chan []*models.Trigger, queryMetrics aggregator.QueryAppMetricsFunc, getBreaker func(string) *circuit.Breaker, setCoolDownExpired func(string, int64)) ([]*generator.Evaluator, error) { count := conf.Evaluator.EvaluatorCount - aClient, err := helpers.CreateHTTPClient(&conf.ScalingEngine.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scaling_client")) + seClient, err := helpers.CreateHTTPSClient(&conf.ScalingEngine.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scaling_client")) if err != nil { logger.Error("failed to create http client for ScalingEngine", err, lager.Data{"scalingengineTLS": conf.ScalingEngine.TLSClientCerts}) os.Exit(1) @@ -190,7 +193,7 @@ func createEvaluators(logger lager.Logger, conf *config.Config, triggersChan cha evaluators := make([]*generator.Evaluator, count) for i := 0; i < count; i++ { - evaluators[i] = generator.NewEvaluator(logger, aClient, conf.ScalingEngine.ScalingEngineURL, triggersChan, + evaluators[i] = generator.NewEvaluator(logger, seClient, conf.ScalingEngine.ScalingEngineURL, triggersChan, conf.DefaultBreachDurationSecs, queryMetrics, getBreaker, setCoolDownExpired) } diff --git a/src/autoscaler/eventgenerator/config/config_test.go b/src/autoscaler/eventgenerator/config/config_test.go index 05f3108054..192143cf4c 100644 --- a/src/autoscaler/eventgenerator/config/config_test.go +++ b/src/autoscaler/eventgenerator/config/config_test.go @@ -42,7 +42,8 @@ server: node_addrs: [address1, address2] node_index: 1 health: - port: 9999 + server_config: + port: 9999 db: policy_db: url: postgres://postgres:password@localhost/autoscaler?sslmode=disable @@ -1060,7 +1061,8 @@ metricCollector: defaultStatWindowSecs: 300 defaultBreachDurationSecs: 300 health: - port: NOT-INTEGER-VALUE + server_config: + port: NOT-INTEGER-VALUE `) }) diff --git a/src/autoscaler/eventgenerator/server/server.go b/src/autoscaler/eventgenerator/server/server.go index 195af0e2ac..f47f6c4f99 100644 --- a/src/autoscaler/eventgenerator/server/server.go +++ b/src/autoscaler/eventgenerator/server/server.go @@ -1,8 +1,11 @@ package server import ( + "fmt" "net/http" + "time" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/aggregator" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" @@ -13,6 +16,7 @@ import ( "code.cloudfoundry.org/lager/v3" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" ) @@ -22,19 +26,85 @@ func (vh VarsFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) vh(w, r, vars) } - -func NewServer(logger lager.Logger, conf *config.Config, queryAppMetric aggregator.QueryAppMetricsFunc, httpStatusCollector healthendpoint.HTTPStatusCollector) (ifrit.Runner, error) { - eh := NewEventGenHandler(logger, queryAppMetric) +func createEventGeneratorRouter(logger lager.Logger, queryAppMetric aggregator.QueryAppMetricsFunc, httpStatusCollector healthendpoint.HTTPStatusCollector, serverConfig config.ServerConfig) (*mux.Router, error) { httpStatusCollectMiddleware := healthendpoint.NewHTTPStatusCollectMiddleware(httpStatusCollector) + eh := NewEventGenHandler(logger, queryAppMetric) r := routes.EventGeneratorRoutes() r.Use(otelmux.Middleware("eventgenerator")) r.Use(httpStatusCollectMiddleware.Collect) r.Get(routes.GetAggregatedMetricHistoriesRouteName).Handler(VarsFunc(eh.GetAggregatedMetricHistories)) + return r, nil +} - httpServerConfig := helpers.ServerConfig{ - Port: conf.Server.Port, +type Server struct { + logger lager.Logger + conf *config.Config + appMetricDB db.AppMetricDB + policyDb db.PolicyDB + queryAppMetric aggregator.QueryAppMetricsFunc + httpStatusCollector healthendpoint.HTTPStatusCollector +} + +func (s *Server) GetMtlsServer() (ifrit.Runner, error) { + eventGeneratorRouter, err := createEventGeneratorRouter(s.logger, s.queryAppMetric, s.httpStatusCollector, s.conf.Server) + if err != nil { + return nil, fmt.Errorf("failed to create event generator router: %w", err) + } + + return helpers.NewHTTPServer(s.logger, serverConfigFrom(s.conf), eventGeneratorRouter) +} + +func NewServer(logger lager.Logger, conf *config.Config, appMetricDB db.AppMetricDB, policyDb db.PolicyDB, queryAppMetric aggregator.QueryAppMetricsFunc, httpStatusCollector healthendpoint.HTTPStatusCollector) *Server { + return &Server{ + logger: logger, + conf: conf, + appMetricDB: appMetricDB, + policyDb: policyDb, + queryAppMetric: queryAppMetric, + httpStatusCollector: httpStatusCollector, + } +} + +func serverConfigFrom(conf *config.Config) helpers.ServerConfig { + return helpers.ServerConfig{ TLS: conf.Server.TLS, + Port: conf.Server.Port, + } +} + +func (s *Server) GetHealthServer() (ifrit.Runner, error) { + healthRouter, err := createHealthRouter(s.appMetricDB, s.policyDb, s.logger, s.conf, s.httpStatusCollector) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) + } + return helpers.NewHTTPServer(s.logger, s.conf.Health.ServerConfig, healthRouter) +} + +func createHealthRouter(appMetricDB db.AppMetricDB, policyDb db.PolicyDB, logger lager.Logger, conf *config.Config, httpStatusCollector healthendpoint.HTTPStatusCollector) (*mux.Router, error) { + checkers := []healthendpoint.Checker{} + gatherer := CreatePrometheusRegistry(appMetricDB, policyDb, httpStatusCollector, logger) + healthRouter, err := healthendpoint.NewHealthRouter(conf.Health, checkers, logger.Session("health-server"), gatherer, time.Now) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) } - return helpers.NewHTTPServer(logger, httpServerConfig, r) + return healthRouter, nil } + +func CreatePrometheusRegistry(appMetricDB db.AppMetricDB, policyDb db.PolicyDB, httpStatusCollector healthendpoint.HTTPStatusCollector, logger lager.Logger) *prometheus.Registry { + promRegistry := prometheus.NewRegistry() + healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ + healthendpoint.NewDatabaseStatusCollector("autoscaler", "eventgenerator", "appMetricDB", appMetricDB), + healthendpoint.NewDatabaseStatusCollector("autoscaler", "eventgenerator", "policyDB", policyDb), + httpStatusCollector, + }, true, logger.Session("eventgenerator-prometheus")) + return promRegistry +} + +// func setupMainRouter(egRouter, healthRouter *mux.Router) *mux.Router { +// mainRouter := mux.NewRouter() +// mainRouter.PathPrefix("/v1").Handler(egRouter) +// //mainRouter.PathPrefix("/health").Handler(healthRouter) +// //mainRouter.PathPrefix("/").Handler(healthRouter) +// return mainRouter +// } diff --git a/src/autoscaler/eventgenerator/server/server_suite_test.go b/src/autoscaler/eventgenerator/server/server_suite_test.go index 0868c4d8c4..957f3f9ff4 100644 --- a/src/autoscaler/eventgenerator/server/server_suite_test.go +++ b/src/autoscaler/eventgenerator/server/server_suite_test.go @@ -1,57 +1,13 @@ package server_test import ( - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/server" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "net/url" - "strconv" "testing" - "code.cloudfoundry.org/lager/v3" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/tedsuo/ifrit" - "github.com/tedsuo/ifrit/ginkgomon_v2" -) - -var ( - serverProcess ifrit.Process - serverUrl *url.URL ) func TestServer(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Server Suite") } - -var _ = BeforeSuite(func() { - port := 1111 + GinkgoParallelProcess() - conf := &config.Config{ - Server: config.ServerConfig{ - ServerConfig: helpers.ServerConfig{ - Port: port, - }, - }, - } - queryAppMetrics := func(appID string, metricType string, start int64, end int64, orderType db.OrderType) ([]*models.AppMetric, error) { - return nil, nil - } - - httpStatusCollector := &fakes.FakeHTTPStatusCollector{} - httpServer, err := server.NewServer(lager.NewLogger("test"), conf, queryAppMetrics, httpStatusCollector) - Expect(err).NotTo(HaveOccurred()) - - serverUrl, err = url.Parse("http://127.0.0.1:" + strconv.Itoa(port)) - Expect(err).ToNot(HaveOccurred()) - - serverProcess = ginkgomon_v2.Invoke(httpServer) -}) - -var _ = AfterSuite(func() { - ginkgomon_v2.Interrupt(serverProcess) -}) diff --git a/src/autoscaler/eventgenerator/server/server_test.go b/src/autoscaler/eventgenerator/server/server_test.go index 1727cb8dd1..391f45de7f 100644 --- a/src/autoscaler/eventgenerator/server/server_test.go +++ b/src/autoscaler/eventgenerator/server/server_test.go @@ -2,22 +2,73 @@ package server_test import ( "net/http" + "net/url" + "strconv" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/aggregator" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/server" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/lager/v3" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/tedsuo/ifrit" + "github.com/tedsuo/ifrit/ginkgomon_v2" ) -const TestPathAggregatedMetricHistories = "/v1/apps/an-app-id/aggregated_metric_histories/a-metric-type" - var _ = Describe("Server", func() { var ( - rsp *http.Response - err error + rsp *http.Response + err error + serverProcess ifrit.Process + serverUrl *url.URL + policyDB *fakes.FakePolicyDB + httpStatusCollector *fakes.FakeHTTPStatusCollector + + appMetricDB *fakes.FakeAppMetricDB + conf *config.Config + queryAppMetrics aggregator.QueryAppMetricsFunc ) - Context("when retrieving aggregared metrics history", func() { + BeforeEach(func() { + port := 1111 + GinkgoParallelProcess() + conf = &config.Config{ + Server: config.ServerConfig{ + ServerConfig: helpers.ServerConfig{ + Port: port, + }, + }, + } + + serverUrl, err = url.Parse("http://127.0.0.1:" + strconv.Itoa(port)) + Expect(err).ToNot(HaveOccurred()) + + queryAppMetrics = func(appID string, metricType string, start int64, end int64, orderType db.OrderType) ([]*models.AppMetric, error) { + return nil, nil + } + + httpStatusCollector = &fakes.FakeHTTPStatusCollector{} + policyDB = &fakes.FakePolicyDB{} + appMetricDB = &fakes.FakeAppMetricDB{} + + }) + + AfterEach(func() { + ginkgomon_v2.Interrupt(serverProcess) + }) + + JustBeforeEach(func() { + httpServer, err := server.NewServer(lager.NewLogger("test"), conf, appMetricDB, policyDB, queryAppMetrics, httpStatusCollector).GetMtlsServer() + Expect(err).NotTo(HaveOccurred()) + serverProcess = ginkgomon_v2.Invoke(httpServer) + }) + + Describe("request on /v1/apps/an-app-id/aggregated_metric_histories/a-metric-type", func() { BeforeEach(func() { - serverUrl.Path = TestPathAggregatedMetricHistories + serverUrl.Path = "/v1/apps/an-app-id/aggregated_metric_histories/a-metric-type" }) JustBeforeEach(func() { @@ -29,9 +80,20 @@ var _ = Describe("Server", func() { Expect(rsp.StatusCode).To(Equal(http.StatusOK)) rsp.Body.Close() }) + When("using wrong method to retrieve aggregared metrics history", func() { + JustBeforeEach(func() { + rsp, err = http.Post(serverUrl.String(), "garbage", nil) + }) + + It("should return 405", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusMethodNotAllowed)) + rsp.Body.Close() + }) + }) }) - Context("when requesting the wrong path", func() { + When("requesting the wrong path", func() { BeforeEach(func() { serverUrl.Path = "/not-exist-path" }) @@ -45,22 +107,6 @@ var _ = Describe("Server", func() { Expect(rsp.StatusCode).To(Equal(http.StatusNotFound)) rsp.Body.Close() }) - }) - - Context("when using wrong method to retrieve aggregared metrics history", func() { - BeforeEach(func() { - serverUrl.Path = TestPathAggregatedMetricHistories - }) - - JustBeforeEach(func() { - rsp, err = http.Post(serverUrl.String(), "garbage", nil) - }) - It("should return 405", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusMethodNotAllowed)) - rsp.Body.Close() - }) }) - }) diff --git a/src/autoscaler/healthendpoint/health_readiness_test.go b/src/autoscaler/healthendpoint/health_readiness_test.go index a4aa3a7710..d118d859f3 100644 --- a/src/autoscaler/healthendpoint/health_readiness_test.go +++ b/src/autoscaler/healthendpoint/health_readiness_test.go @@ -48,10 +48,10 @@ var _ = Describe("Health Readiness", func() { logger = lager.NewLogger("healthendpoint-test") logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG)) - config.HealthCheckUsername = "test-user-name" - config.HealthCheckPassword = "test-user-password" - config.HealthCheckPasswordHash = "" - config.HealthCheckUsernameHash = "" + config.BasicAuth.Username = "test-user-name" + config.BasicAuth.Password = "test-user-password" + config.BasicAuth.PasswordHash = "" + config.BasicAuth.UsernameHash = "" config.ReadinessCheckEnabled = true checkers = []healthendpoint.Checker{} tmsttr := time.Now() @@ -67,10 +67,10 @@ var _ = Describe("Health Readiness", func() { Context("Authentication parameter checks", func() { When("username and password are defined", func() { BeforeEach(func() { - config.HealthCheckUsername = "username" - config.HealthCheckPassword = "password" - config.HealthCheckUsernameHash = "" - config.HealthCheckPasswordHash = "" + config.BasicAuth.Username = "username" + config.BasicAuth.Password = "password" + config.BasicAuth.UsernameHash = "" + config.BasicAuth.PasswordHash = "" }) When("Prometheus Health endpoint is called", func() { It("should require basic auth", func() { @@ -85,10 +85,10 @@ var _ = Describe("Health Readiness", func() { }) When("username_hash and password_hash are defined", func() { BeforeEach(func() { - config.HealthCheckUsername = "" - config.HealthCheckPassword = "" - config.HealthCheckUsernameHash = "username_hash" - config.HealthCheckPasswordHash = "username_hash" + config.BasicAuth.Username = "" + config.BasicAuth.Password = "" + config.BasicAuth.UsernameHash = "username_hash" + config.BasicAuth.PasswordHash = "username_hash" }) When("Prometheus Health endpoint is called without basic auth", func() { It("should require basic auth", func() { @@ -109,8 +109,8 @@ var _ = Describe("Health Readiness", func() { Context("without basic auth configured", func() { BeforeEach(func() { - config.HealthCheckUsername = "" - config.HealthCheckPassword = "" + config.BasicAuth.Username = "" + config.BasicAuth.Password = "" }) When("Prometheus Health endpoint is called", func() { It("should respond OK", func() { @@ -326,8 +326,8 @@ var _ = Describe("Health Readiness", func() { Context("pprof endpoint", func() { When("basic auth is not configured", func() { BeforeEach(func() { - config.HealthCheckUsername = "" - config.HealthCheckPassword = "" + config.BasicAuth.Username = "" + config.BasicAuth.Password = "" }) It("should not be available", func() { apitest.New(). diff --git a/src/autoscaler/healthendpoint/server.go b/src/autoscaler/healthendpoint/server.go index 4e4ca9272e..5c0d5c5211 100644 --- a/src/autoscaler/healthendpoint/server.go +++ b/src/autoscaler/healthendpoint/server.go @@ -25,7 +25,6 @@ package healthendpoint // - scheduler import ( - "net/http" "net/http/pprof" "time" @@ -36,50 +35,17 @@ import ( "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/tedsuo/ifrit" - "golang.org/x/crypto/bcrypt" ) // basic authentication credentials struct -type basicAuthenticationMiddleware struct { - usernameHash []byte - passwordHash []byte -} - -// middleware basic authentication middleware functionality for healthcheck -func (bam *basicAuthenticationMiddleware) middleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - username, password, authOK := r.BasicAuth() - - if !authOK || bcrypt.CompareHashAndPassword(bam.usernameHash, []byte(username)) != nil || bcrypt.CompareHashAndPassword(bam.passwordHash, []byte(password)) != nil { - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) - return - } - next.ServeHTTP(w, r) - }) -} - -// NewServerWithBasicAuth open the healthcheck port with basic authentication. -// Make sure that username and password is not empty -func NewServerWithBasicAuth(conf helpers.HealthConfig, healthCheckers []Checker, logger lager.Logger, gatherer prometheus.Gatherer, time func() time.Time) (ifrit.Runner, error) { - healthRouter, err := NewHealthRouter(conf, healthCheckers, logger, gatherer, time) - if err != nil { - return nil, err - } - httpServerConfig := helpers.ServerConfig{ - Port: conf.Port, - TLS: conf.TLS, - } - return helpers.NewHTTPServer(logger, httpServerConfig, healthRouter) -} func NewHealthRouter(conf helpers.HealthConfig, healthCheckers []Checker, logger lager.Logger, gatherer prometheus.Gatherer, time func() time.Time) (*mux.Router, error) { var healthRouter *mux.Router var err error - username := conf.HealthCheckUsername - password := conf.HealthCheckPassword - usernameHash := conf.HealthCheckUsernameHash - passwordHash := conf.HealthCheckPasswordHash + username := conf.BasicAuth.Username + password := conf.BasicAuth.Password + usernameHash := conf.BasicAuth.UsernameHash + passwordHash := conf.BasicAuth.PasswordHash if username == "" && password == "" && usernameHash == "" && passwordHash == "" { //when username and password are not set then don't use basic authentication healthRouter = mux.NewRouter() @@ -97,7 +63,7 @@ func NewHealthRouter(conf helpers.HealthConfig, healthCheckers []Checker, logger } func healthBasicAuthRouter(conf helpers.HealthConfig, healthCheckers []Checker, logger lager.Logger, gatherer prometheus.Gatherer, time func() time.Time) (*mux.Router, error) { - basicAuthentication, err := createBasicAuthMiddleware(logger, conf.HealthCheckUsernameHash, conf.HealthCheckUsername, conf.HealthCheckPasswordHash, conf.HealthCheckPassword) + ba, err := helpers.CreateBasicAuthMiddleware(logger, conf.BasicAuth) if err != nil { return nil, err } @@ -111,10 +77,9 @@ func healthBasicAuthRouter(conf helpers.HealthConfig, healthCheckers []Checker, } //authenticated paths health := router.Path("/health").Subrouter() - health.Use(basicAuthentication.middleware) - + health.Use(ba.BasicAuthenticationMiddleware) pprofRouter := router.PathPrefix("/debug/pprof").Subrouter() - pprofRouter.Use(basicAuthentication.middleware) + pprofRouter.Use(ba.BasicAuthenticationMiddleware) pprofRouter.HandleFunc("/cmdline", pprof.Cmdline) pprofRouter.HandleFunc("/profile", pprof.Profile) @@ -123,65 +88,8 @@ func healthBasicAuthRouter(conf helpers.HealthConfig, healthCheckers []Checker, pprofRouter.PathPrefix("").HandlerFunc(pprof.Index) everything := router.PathPrefix("").Subrouter() - everything.Use(basicAuthentication.middleware) + everything.Use(ba.BasicAuthenticationMiddleware) everything.PathPrefix("").Handler(promHandler) return router, nil } - -func createBasicAuthMiddleware(logger lager.Logger, usernameHash string, username string, passwordHash string, password string) (*basicAuthenticationMiddleware, error) { - usernameHashByte, err := getUserHashBytes(logger, usernameHash, username) - if err != nil { - return nil, err - } - - passwordHashByte, err := getPasswordHashBytes(logger, passwordHash, password) - if err != nil { - return nil, err - } - - basicAuthentication := &basicAuthenticationMiddleware{ - usernameHash: usernameHashByte, - passwordHash: passwordHashByte, - } - return basicAuthentication, nil -} - -func getPasswordHashBytes(logger lager.Logger, passwordHash string, password string) ([]byte, error) { - var passwordHashByte []byte - var err error - if passwordHash == "" { - if len(password) > 72 { - logger.Error("warning-configured-password-too-long-using-only-first-72-characters", bcrypt.ErrPasswordTooLong, lager.Data{"password-length": len(password)}) - password = password[:72] - } - passwordHashByte, err = bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost) // use MinCost as the config already provided it as cleartext - if err != nil { - logger.Error("failed-new-server-password", err) - return nil, err - } - } else { - passwordHashByte = []byte(passwordHash) - } - return passwordHashByte, nil -} - -func getUserHashBytes(logger lager.Logger, usernameHash string, username string) ([]byte, error) { - var usernameHashByte []byte - var err error - if usernameHash == "" { - if len(username) > 72 { - logger.Error("warning-configured-username-too-long-using-only-first-72-characters", bcrypt.ErrPasswordTooLong, lager.Data{"username-length": len(username)}) - username = username[:72] - } - // when username and password are set for health check - usernameHashByte, err = bcrypt.GenerateFromPassword([]byte(username), bcrypt.MinCost) // use MinCost as the config already provided it as cleartext - if err != nil { - logger.Error("failed-new-server-username", err) - return nil, err - } - } else { - usernameHashByte = []byte(usernameHash) - } - return usernameHashByte, err -} diff --git a/src/autoscaler/helpers/auth/auth_suite_test.go b/src/autoscaler/helpers/auth/auth_suite_test.go new file mode 100644 index 0000000000..cc266fc4ed --- /dev/null +++ b/src/autoscaler/helpers/auth/auth_suite_test.go @@ -0,0 +1,13 @@ +package auth_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAuth(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Auth Suite") +} diff --git a/src/autoscaler/helpers/auth/xfcc_auth.go b/src/autoscaler/helpers/auth/xfcc_auth.go new file mode 100644 index 0000000000..7f9857278a --- /dev/null +++ b/src/autoscaler/helpers/auth/xfcc_auth.go @@ -0,0 +1,116 @@ +package auth + +import ( + "crypto/x509" + "encoding/base64" + "errors" + "fmt" + "net/http" + "regexp" + "strings" + + "code.cloudfoundry.org/lager/v3" +) + +var ErrorWrongSpace = errors.New("space guid is wrong") +var ErrorWrongOrg = errors.New("org guid is wrong") +var ErrXFCCHeaderNotFound = errors.New("xfcc header not found") + +type XFCCAuthMiddleware struct { + logger lager.Logger + spaceGuid string + orgGuid string +} + +func (m *XFCCAuthMiddleware) checkAuth(r *http.Request) error { + xfccHeader := r.Header.Get("X-Forwarded-Client-Cert") + if xfccHeader == "" { + return ErrXFCCHeaderNotFound + } + + data, err := base64.StdEncoding.DecodeString(removeQuotes(xfccHeader)) + if err != nil { + return fmt.Errorf("base64 parsing failed: %w", err) + } + + cert, err := x509.ParseCertificate(data) + if err != nil { + return fmt.Errorf("failed to parse certificate: %w", err) + } + + if getSpaceGuid(cert) != m.spaceGuid { + return ErrorWrongSpace + } + + if getOrgGuid(cert) != m.orgGuid { + return ErrorWrongOrg + } + + return nil +} + +func (m *XFCCAuthMiddleware) XFCCAuthenticationMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + err := m.checkAuth(r) + + if err != nil { + m.logger.Error("xfcc-auth-error", err) + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) +} + +func NewXfccAuthMiddleware(logger lager.Logger, orgGuid, spaceGuid string) *XFCCAuthMiddleware { + return &XFCCAuthMiddleware{ + logger: logger, + orgGuid: orgGuid, + spaceGuid: spaceGuid, + } +} + +func getSpaceGuid(cert *x509.Certificate) string { + var certSpaceGuid string + for _, ou := range cert.Subject.OrganizationalUnit { + if strings.Contains(ou, "space:") { + kv := mapFrom(ou) + certSpaceGuid = kv["space"] + break + } + } + return certSpaceGuid +} + +func mapFrom(input string) map[string]string { + result := make(map[string]string) + + r := regexp.MustCompile(`(\w+):(\w+-\w+)`) + matches := r.FindAllStringSubmatch(input, -1) + + for _, match := range matches { + result[match[1]] = match[2] + } + return result +} + +func getOrgGuid(cert *x509.Certificate) string { + var certOrgGuid string + for _, ou := range cert.Subject.OrganizationalUnit { + // capture from string k:v with regex + if strings.Contains(ou, "org:") { + kv := mapFrom(ou) + certOrgGuid = kv["org"] + break + } + } + return certOrgGuid +} + +func removeQuotes(xfccHeader string) string { + if xfccHeader[0] == '"' { + xfccHeader = xfccHeader[1 : len(xfccHeader)-1] + } + return xfccHeader +} diff --git a/src/autoscaler/helpers/auth/xfcc_auth_test.go b/src/autoscaler/helpers/auth/xfcc_auth_test.go new file mode 100644 index 0000000000..6a5a54ac87 --- /dev/null +++ b/src/autoscaler/helpers/auth/xfcc_auth_test.go @@ -0,0 +1,153 @@ +package auth_test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/auth" + + "code.cloudfoundry.org/lager/v3/lagertest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" +) + +var handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +}) + +var _ = Describe("XfccAuthMiddleware", func() { + var ( + server *httptest.Server + resp *http.Response + + buffer *gbytes.Buffer + + err error + xfccClientCert []byte + + orgGuid string + spaceGuid string + ) + + AfterEach(func() { + server.Close() + }) + + JustBeforeEach(func() { + logger := lagertest.NewTestLogger("xfcc-auth-test") + buffer = logger.Buffer() + xm := auth.NewXfccAuthMiddleware(logger, orgGuid, spaceGuid) + + server = httptest.NewServer(xm.XFCCAuthenticationMiddleware(handler)) + + req, err := http.NewRequest("GET", server.URL+"/some-protected-endpoint", nil) + + if len(xfccClientCert) > 0 { + block, _ := pem.Decode(xfccClientCert) + Expect(err).NotTo(HaveOccurred()) + Expect(block).ShouldNot(BeNil()) + + req.Header.Add("X-Forwarded-Client-Cert", base64.StdEncoding.EncodeToString(block.Bytes)) + } + Expect(err).NotTo(HaveOccurred()) + + resp, err = http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + }) + + BeforeEach(func() { + orgGuid = "org-guid" + spaceGuid = "space-guid" + }) + + When("xfcc header is not set", func() { + BeforeEach(func() { + xfccClientCert = []byte{} + }) + + It("should return 401", func() { + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Eventually(buffer).Should(gbytes.Say(auth.ErrXFCCHeaderNotFound.Error())) + }) + }) + + When("xfcc cert matches org and space guids", func() { + BeforeEach(func() { + xfccClientCert, err = generateClientCert(orgGuid, spaceGuid) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should return 200", func() { + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) + }) + + When("xfcc cert does not match org guid", func() { + BeforeEach(func() { + xfccClientCert, err = generateClientCert("wrong-org-guid", spaceGuid) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should return 401", func() { + Eventually(buffer).Should(gbytes.Say(auth.ErrorWrongOrg.Error())) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + }) + + When("xfcc cert does not match space guid", func() { + BeforeEach(func() { + xfccClientCert, err = generateClientCert(orgGuid, "wrong-space-guid") + Expect(err).NotTo(HaveOccurred()) + }) + + It("should return 401", func() { + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Eventually(buffer).Should(gbytes.Say(auth.ErrorWrongSpace.Error())) + }) + }) +}) + +// generateClientCert generates a client certificate with the specified spaceGUID and orgGUID +// included in the organizational unit string. +func generateClientCert(orgGUID, spaceGUID string) ([]byte, error) { + // Generate a random serial number for the certificate + // + serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, err + } + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + + // Create a new X.509 certificate template + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"My Organization"}, + OrganizationalUnit: []string{fmt.Sprintf("space:%s org:%s", spaceGUID, orgGUID)}, + }, + } + // Generate the certificate + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + if err != nil { + return nil, err + } + + // Encode the certificate to PEM format + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + + return certPEM, nil +} diff --git a/src/autoscaler/helpers/basic_auth.go b/src/autoscaler/helpers/basic_auth.go new file mode 100644 index 0000000000..18fe3f77af --- /dev/null +++ b/src/autoscaler/helpers/basic_auth.go @@ -0,0 +1,95 @@ +package helpers + +import ( + "net/http" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/lager/v3" + "golang.org/x/crypto/bcrypt" +) + +type BasicAuthenticationMiddleware struct { + usernameHash []byte + passwordHash []byte + logger lager.Logger +} + +// middleware basic authentication middleware functionality for healthcheck +func (bam *BasicAuthenticationMiddleware) BasicAuthenticationMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + username, password, authOK := r.BasicAuth() + + bam.logger.Info("basic-authentication-middleware", lager.Data{"usernameHash": bam.usernameHash, "passwordHash": bam.passwordHash}) + if bam.usernameHash == nil && bam.passwordHash == nil { + next.ServeHTTP(w, r) + return + } + + if !authOK || bcrypt.CompareHashAndPassword(bam.usernameHash, []byte(username)) != nil || bcrypt.CompareHashAndPassword(bam.passwordHash, []byte(password)) != nil { + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + next.ServeHTTP(w, r) + }) +} + +func CreateBasicAuthMiddleware(logger lager.Logger, ba models.BasicAuth) (*BasicAuthenticationMiddleware, error) { + var basicAuthentication *BasicAuthenticationMiddleware + usernameHash, username, passwordHash, password := ba.UsernameHash, ba.Username, ba.PasswordHash, ba.Password + + usernameHashByte, err := getUserHashBytes(logger, usernameHash, username) + if err != nil { + return nil, err + } + + passwordHashByte, err := getPasswordHashBytes(logger, passwordHash, password) + if err != nil { + return nil, err + } + + basicAuthentication = &BasicAuthenticationMiddleware{ + usernameHash: usernameHashByte, + passwordHash: passwordHashByte, + logger: logger, + } + return basicAuthentication, nil +} + +func getUserHashBytes(logger lager.Logger, usernameHash string, username string) ([]byte, error) { + var usernameHashByte []byte + var err error + if usernameHash == "" { + if len(username) > 72 { + logger.Error("warning-configured-username-too-long-using-only-first-72-characters", bcrypt.ErrPasswordTooLong, lager.Data{"username-length": len(username)}) + username = username[:72] + } + // when username and password are set for health check + usernameHashByte, err = bcrypt.GenerateFromPassword([]byte(username), bcrypt.MinCost) // use MinCost as the config already provided it as cleartext + if err != nil { + logger.Error("failed-new-server-username", err) + return nil, err + } + } else { + usernameHashByte = []byte(usernameHash) + } + return usernameHashByte, err +} + +func getPasswordHashBytes(logger lager.Logger, passwordHash string, password string) ([]byte, error) { + var passwordHashByte []byte + var err error + if passwordHash == "" { + if len(password) > 72 { + logger.Error("warning-configured-password-too-long-using-only-first-72-characters", bcrypt.ErrPasswordTooLong, lager.Data{"password-length": len(password)}) + password = password[:72] + } + passwordHashByte, err = bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost) // use MinCost as the config already provided it as cleartext + if err != nil { + logger.Error("failed-new-server-password", err) + return nil, err + } + } else { + passwordHashByte = []byte(passwordHash) + } + return passwordHashByte, nil +} diff --git a/src/autoscaler/helpers/basic_auth_test.go b/src/autoscaler/helpers/basic_auth_test.go new file mode 100644 index 0000000000..00c57f31f7 --- /dev/null +++ b/src/autoscaler/helpers/basic_auth_test.go @@ -0,0 +1,82 @@ +package helpers_test + +import ( + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/lager/v3" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "net/http" + "net/http/httptest" +) + +var handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +}) + +var _ = Describe("BasicAuthenticationMiddleware", func() { + var ( + server *httptest.Server + ba models.BasicAuth + resp *http.Response + username string + password string + logger lager.Logger + ) + + BeforeEach(func() { + logger = lager.NewLogger("helper-test") + }) + + AfterEach(func() { + server.Close() + }) + + JustBeforeEach(func() { + bam, err := helpers.CreateBasicAuthMiddleware(logger, ba) + Expect(err).NotTo(HaveOccurred()) + + server = httptest.NewServer(bam.BasicAuthenticationMiddleware(handler)) + + req, err := http.NewRequest("GET", server.URL+"/some-protected-endpoint", nil) + req.SetBasicAuth(username, password) + Expect(err).NotTo(HaveOccurred()) + + resp, err = http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + + defer resp.Body.Close() + }) + + When("basic auth is enabled", func() { + BeforeEach(func() { + ba = models.BasicAuth{ + Username: "username", + Password: "password", + } + }) + + When("credentials are correct", func() { + BeforeEach(func() { + username = ba.Username + password = ba.Password + }) + + It("should return 200", func() { + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) + }) + + When("credentials are incorrect", func() { + BeforeEach(func() { + username = "wrong-username" + password = "wrong-password" + }) + + It("should return 401", func() { + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + }) + }) +}) diff --git a/src/autoscaler/helpers/health.go b/src/autoscaler/helpers/health.go index 6631359ba7..96bea4a603 100644 --- a/src/autoscaler/helpers/health.go +++ b/src/autoscaler/helpers/health.go @@ -3,46 +3,44 @@ package helpers import ( "fmt" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" "golang.org/x/crypto/bcrypt" ) type HealthConfig struct { - ServerConfig `yaml:",inline"` - HealthCheckUsername string `yaml:"username"` - HealthCheckUsernameHash string `yaml:"username_hash"` - HealthCheckPassword string `yaml:"password"` - HealthCheckPasswordHash string `yaml:"password_hash"` - ReadinessCheckEnabled bool `yaml:"readiness_enabled"` + ServerConfig ServerConfig `yaml:"server_config"` + BasicAuth models.BasicAuth `yaml:"basic_auth"` + ReadinessCheckEnabled bool `yaml:"readiness_enabled"` } var ErrConfiguration = fmt.Errorf("configuration error") func (c *HealthConfig) Validate() error { - if c.HealthCheckUsername != "" && c.HealthCheckUsernameHash != "" { + if c.BasicAuth.Username != "" && c.BasicAuth.UsernameHash != "" { return fmt.Errorf("%w: both healthcheck username and healthcheck username_hash are set, please provide only one of them", ErrConfiguration) } - if c.HealthCheckPassword != "" && c.HealthCheckPasswordHash != "" { + if c.BasicAuth.Password != "" && c.BasicAuth.PasswordHash != "" { return fmt.Errorf("%w: both healthcheck password and healthcheck password_hash are provided, please provide only one of them", ErrConfiguration) } - if c.HealthCheckUsernameHash != "" { - if _, err := bcrypt.Cost([]byte(c.HealthCheckUsernameHash)); err != nil { + if c.BasicAuth.UsernameHash != "" { + if _, err := bcrypt.Cost([]byte(c.BasicAuth.UsernameHash)); err != nil { return fmt.Errorf("%w: healthcheck username_hash is not a valid bcrypt hash", ErrConfiguration) } } - if c.HealthCheckPasswordHash != "" { - if _, err := bcrypt.Cost([]byte(c.HealthCheckPasswordHash)); err != nil { + if c.BasicAuth.PasswordHash != "" { + if _, err := bcrypt.Cost([]byte(c.BasicAuth.PasswordHash)); err != nil { return fmt.Errorf("%w: healthcheck password_hash is not a valid bcrypt hash", ErrConfiguration) } } - if c.HealthCheckUsername == "" && c.HealthCheckPassword != "" { + if c.BasicAuth.Username == "" && c.BasicAuth.Password != "" { return fmt.Errorf("%w: healthcheck username is empty", ErrConfiguration) } - if c.HealthCheckUsername != "" && c.HealthCheckPassword == "" { + if c.BasicAuth.Username != "" && c.BasicAuth.Password == "" { return fmt.Errorf("%w: healthcheck password is empty", ErrConfiguration) } diff --git a/src/autoscaler/helpers/health_test.go b/src/autoscaler/helpers/health_test.go index 5a3a411b88..1df5b4acc4 100644 --- a/src/autoscaler/helpers/health_test.go +++ b/src/autoscaler/helpers/health_test.go @@ -4,6 +4,7 @@ import ( "errors" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" . "github.com/onsi/ginkgo/v2" @@ -25,9 +26,9 @@ var _ = Describe("Health Config", func() { When("Readiness is not supplied", func() { BeforeEach(func() { healthConfigBytes = []byte(` -port: 9999 -username: test-username -password: password +basic_auth: + username: test-username + password: password readiness_enabled: false `) }) @@ -38,11 +39,10 @@ readiness_enabled: false Expect(err).ToNot(HaveOccurred()) Expect(healthConfig).To(Equal(helpers.HealthConfig{ - ServerConfig: helpers.ServerConfig{ - Port: 9999, + BasicAuth: models.BasicAuth{ + Username: "test-username", + Password: "password", }, - HealthCheckUsername: "test-username", - HealthCheckPassword: "password", ReadinessCheckEnabled: false, })) }) @@ -51,8 +51,9 @@ readiness_enabled: false BeforeEach(func() { healthConfigBytes = []byte(` port: 9999 -username: test-username -password: password +basic_auth: + username: test-username + password: password readiness_enabled: true `) }) @@ -63,11 +64,10 @@ readiness_enabled: true Expect(err).ToNot(HaveOccurred()) Expect(healthConfig).To(Equal(helpers.HealthConfig{ - ServerConfig: helpers.ServerConfig{ - Port: 9999, + BasicAuth: models.BasicAuth{ + Username: "test-username", + Password: "password", }, - HealthCheckUsername: "test-username", - HealthCheckPassword: "password", ReadinessCheckEnabled: true, })) }) @@ -76,10 +76,10 @@ readiness_enabled: true When("both password password_hash are supplied", func() { BeforeEach(func() { healthConfigBytes = []byte(` -port: 9999 -username: test-username -password: password -password_hash: password_hash +basic_auth: + username: test-username + password: password + password_hash: password_hash `) }) It("should fail validation", func() { diff --git a/src/autoscaler/helpers/httpclient.go b/src/autoscaler/helpers/httpclient.go index 5fcd0b93ad..49076b390d 100644 --- a/src/autoscaler/helpers/httpclient.go +++ b/src/autoscaler/helpers/httpclient.go @@ -1,6 +1,7 @@ package helpers import ( + "encoding/base64" "fmt" "net/http" "time" @@ -13,13 +14,52 @@ import ( "code.cloudfoundry.org/cfhttp/v2" ) +type TransportWithBasicAuth struct { + Username string + Password string + Base http.RoundTripper +} + +func (t *TransportWithBasicAuth) base() http.RoundTripper { + if t.Base != nil { + return t.Base + } + return http.DefaultTransport +} + +func (t *TransportWithBasicAuth) RoundTrip(req *http.Request) (*http.Response, error) { + credentials := t.Username + ":" + t.Password + basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials)) + fmt.Println("banana TransportWithBasicAuth:credentials", credentials) + fmt.Println("banana TransportWithBasicAuth:", basicAuth) + req.Header.Add("Authorization", basicAuth) + return t.base().RoundTrip(req) +} + func DefaultClientConfig() cf.ClientConfig { return cf.ClientConfig{ MaxIdleConnsPerHost: 200, IdleConnectionTimeoutMs: 5 * 1000, } } -func CreateHTTPClient(tlsCerts *models.TLSCerts, config cf.ClientConfig, logger lager.Logger) (*http.Client, error) { + +func CreateHTTPClient(ba *models.BasicAuth, config cf.ClientConfig, logger lager.Logger) (*http.Client, error) { + client := cfhttp.NewClient( + cfhttp.WithDialTimeout(30*time.Second), + cfhttp.WithIdleConnTimeout(time.Duration(config.IdleConnectionTimeoutMs)*time.Millisecond), + cfhttp.WithMaxIdleConnsPerHost(config.MaxIdleConnsPerHost), + ) + + client = cf.RetryClient(config, client, logger) + client.Transport = &TransportWithBasicAuth{ + Username: ba.Username, + Password: ba.Password, + } + + return client, nil +} + +func CreateHTTPSClient(tlsCerts *models.TLSCerts, config cf.ClientConfig, logger lager.Logger) (*http.Client, error) { tlsConfig, err := tlsCerts.CreateClientConfig() if err != nil { return nil, fmt.Errorf("failed to create tls config: %w", err) diff --git a/src/autoscaler/integration/components_test.go b/src/autoscaler/integration/components_test.go index 3d911ca479..b63220fcbb 100644 --- a/src/autoscaler/integration/components_test.go +++ b/src/autoscaler/integration/components_test.go @@ -182,7 +182,7 @@ func (components *Components) PrepareGolangApiServerConfig(dbURI string, publicA Logging: helpers.LoggingConfig{ Level: LOGLEVEL, }, - PublicApiServer: helpers.ServerConfig{ + Server: helpers.ServerConfig{ Port: publicApiPort, TLS: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "api.key"), @@ -352,7 +352,7 @@ func (components *Components) PrepareEventGeneratorConfig(dbUri string, port int EvaluatorCount: 1, TriggerArrayChannelSize: 1, }, - DB: egConfig.DBConfig{ + Db: egConfig.DBConfig{ PolicyDB: db.DatabaseConfig{ URL: dbUri, }, diff --git a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_suite_test.go b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_suite_test.go index fb67b8ce82..0aac35a199 100644 --- a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_suite_test.go +++ b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_suite_test.go @@ -136,13 +136,13 @@ var _ = SynchronizedBeforeSuite(func() []byte { cfg.RateLimit.ValidDuration = 1 * time.Second cfg.Logging.Level = "debug" - cfg.Health.HealthCheckUsername = "metricsforwarderhealthcheckuser" - cfg.Health.HealthCheckPassword = "metricsforwarderhealthcheckpassword" + cfg.Health.BasicAuth.Username = "metricsforwarderhealthcheckuser" + cfg.Health.BasicAuth.Password = "metricsforwarderhealthcheckpassword" cfg.Health.ReadinessCheckEnabled = true cfg.Server.Port = 10000 + GinkgoParallelProcess() healthport = 8000 + GinkgoParallelProcess() - cfg.Health.Port = healthport + cfg.Health.ServerConfig.Port = healthport cfg.CacheCleanupInterval = 10 * time.Minute cfg.PolicyPollerInterval = 40 * time.Second cfg.Db = make(map[string]db.DatabaseConfig) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 6536d0b95e..35b5c8e9c0 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -139,7 +139,7 @@ func loadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) } func loadMetricsforwarderConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { - data, err := vcapReader.GetServiceCredentialContent("config", "metricsforwarder") + data, err := vcapReader.GetServiceCredentialContent("metricsforwarder-config", "metricsforwarder") if err != nil { return fmt.Errorf("%w: %v", ErrMetricsforwarderConfigNotFound, err) } diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index f61b8d9d82..b87d97937d 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -9,23 +9,12 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -func bytesToFile(b []byte) string { - if len(b) == 0 { - return "" - } - - file, err := os.CreateTemp("", "") - Expect(err).NotTo(HaveOccurred()) - _, err = file.Write(b) - Expect(err).NotTo(HaveOccurred()) - return file.Name() -} - var _ = Describe("Config", func() { var ( conf *Config @@ -191,7 +180,7 @@ var _ = Describe("Config", func() { When("config is read from file", func() { JustBeforeEach(func() { - configFile = bytesToFile(configBytes) + configFile = testhelpers.BytesToFile(configBytes) conf, err = LoadConfig(configFile, mockVCAPConfigurationReader) }) @@ -242,7 +231,8 @@ db: max_idle_connections: 5 connection_max_lifetime: 60s health: - port: 9999 + server_config: + port: 9999 cred_helper_impl: default `) }) @@ -250,7 +240,6 @@ cred_helper_impl: default It("returns the config", func() { Expect(conf.Server.Port).To(Equal(8081)) Expect(conf.Logging.Level).To(Equal("debug")) - Expect(conf.Health.Port).To(Equal(9999)) Expect(conf.LoggregatorConfig.MetronAddress).To(Equal("127.0.0.1:3457")) Expect(conf.Db[db.PolicyDb]).To(Equal( db.DatabaseConfig{ @@ -278,7 +267,8 @@ db: max_idle_connections: 5 connection_max_lifetime: 60s health: - port: 8081 + server_config: + port: 8081 `) }) @@ -289,7 +279,6 @@ health: Expect(conf.LoggregatorConfig.MetronAddress).To(Equal(DefaultMetronAddress)) Expect(conf.CacheTTL).To(Equal(DefaultCacheTTL)) Expect(conf.CacheCleanupInterval).To(Equal(DefaultCacheCleanupInterval)) - Expect(conf.Health.Port).To(Equal(8081)) }) }) @@ -302,7 +291,6 @@ health: conf = &Config{} conf.Server.Port = 8081 conf.Logging.Level = "debug" - conf.Health.Port = 8081 conf.LoggregatorConfig.MetronAddress = "127.0.0.1:3458" conf.LoggregatorConfig.TLS.CACertFile = "../testcerts/ca.crt" conf.LoggregatorConfig.TLS.CertFile = "../testcerts/client.crt" diff --git a/src/autoscaler/metricsforwarder/exampleconfig/config.yml b/src/autoscaler/metricsforwarder/exampleconfig/config.yml index 5549af3251..6ae2ee4b6f 100644 --- a/src/autoscaler/metricsforwarder/exampleconfig/config.yml +++ b/src/autoscaler/metricsforwarder/exampleconfig/config.yml @@ -15,7 +15,8 @@ db: max_idle_connections: 5 connection_max_lifetime: 60s health: - port: 8081 + server_config: + port: 8081 rate_limit: max_amount: 10 valid_duration: 1s diff --git a/src/autoscaler/metricsforwarder/server/custom_metrics_handlers_test.go b/src/autoscaler/metricsforwarder/server/custom_metrics_handlers_test.go index 9eea0d12ef..7454721ff9 100644 --- a/src/autoscaler/metricsforwarder/server/custom_metrics_handlers_test.go +++ b/src/autoscaler/metricsforwarder/server/custom_metrics_handlers_test.go @@ -3,6 +3,7 @@ package server_test import ( "bytes" "encoding/json" + "fmt" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" @@ -15,6 +16,7 @@ import ( "net/http" "net/http/httptest" + "net/url" "github.com/patrickmn/go-cache" ) @@ -32,7 +34,6 @@ var _ = Describe("MetricHandler", func() { metricsforwarder *fakes.FakeMetricForwarder resp *httptest.ResponseRecorder - req *http.Request err error body []byte @@ -41,6 +42,8 @@ var _ = Describe("MetricHandler", func() { found bool scalingPolicy *models.ScalingPolicy + + serverURL *url.URL ) BeforeEach(func() { @@ -53,11 +56,17 @@ var _ = Describe("MetricHandler", func() { resp = httptest.NewRecorder() handler = NewCustomMetricsHandler(logger, metricsforwarder, policyDB, allowedMetricCache) allowedMetricCache.Flush() + + serverURL, err = url.Parse(fmt.Sprintf("http://127.0.0.1:%d", conf.Server.Port)) + Expect(err).NotTo(HaveOccurred()) }) Describe("PublishMetrics", func() { JustBeforeEach(func() { - req = CreateRequest(body) + serverURL.Path = "/v1/apps/an-app-id/metrics" + req, err := http.NewRequest(http.MethodPost, serverURL.String(), bytes.NewReader(body)) + Expect(err).ToNot(HaveOccurred()) + req.Header.Add("Content-Type", "application/json") Expect(err).ToNot(HaveOccurred()) vars["appid"] = "an-app-id" handler.VerifyCredentialsAndPublishMetrics(resp, req, vars) @@ -71,7 +80,7 @@ var _ = Describe("MetricHandler", func() { }, nil) body = []byte(`{ "instance_index":0, - "test" : + "test" : "metrics":[ { "name":"custom_metric1", @@ -268,10 +277,3 @@ var _ = Describe("MetricHandler", func() { }) }) - -func CreateRequest(body []byte) *http.Request { - req, err := http.NewRequest(http.MethodPost, serverUrl+"/v1/apps/an-app-id/metrics", bytes.NewReader(body)) - Expect(err).ToNot(HaveOccurred()) - req.Header.Add("Content-Type", "application/json") - return req -} diff --git a/src/autoscaler/metricsforwarder/server/server_suite_test.go b/src/autoscaler/metricsforwarder/server/server_suite_test.go index 23dbdbcf31..7272358256 100644 --- a/src/autoscaler/metricsforwarder/server/server_suite_test.go +++ b/src/autoscaler/metricsforwarder/server/server_suite_test.go @@ -1,7 +1,6 @@ package server_test import ( - "fmt" "os" "path/filepath" "time" @@ -26,7 +25,6 @@ import ( var ( conf *config.Config serverProcess ifrit.Process - serverUrl string policyDB *fakes.FakePolicyDB rateLimiter *fakes.FakeLimiter fakeCredentials *fakes.FakeCredentials @@ -72,8 +70,10 @@ var _ = SynchronizedBeforeSuite(func() []byte { healthConfig := helpers.HealthConfig{ ReadinessCheckEnabled: true, - HealthCheckUsername: "metricsforwarderhealthcheckuser", - HealthCheckPassword: "metricsforwarderhealthcheckpassword", + BasicAuth: models.BasicAuth{ + Username: "metricsforwarderhealthcheckuser", + Password: "metricsforwarderhealthcheckpassword", + }, } conf = &config.Config{ Server: serverConfig, @@ -94,7 +94,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { httpServer, err := NewServer(logger, conf, policyDB, fakeCredentials, allowedMetricCache, httpStatusCollector, rateLimiter) Expect(err).NotTo(HaveOccurred()) - serverUrl = fmt.Sprintf("http://127.0.0.1:%d", conf.Server.Port) + serverProcess = ginkgomon_v2.Invoke(httpServer) }) diff --git a/src/autoscaler/metricsforwarder/server/server_test.go b/src/autoscaler/metricsforwarder/server/server_test.go index 8c3d121249..09feffaca8 100644 --- a/src/autoscaler/metricsforwarder/server/server_test.go +++ b/src/autoscaler/metricsforwarder/server/server_test.go @@ -2,11 +2,12 @@ package server_test import ( "bytes" - "encoding/base64" "encoding/json" "errors" + "fmt" "io" "net/http" + "net/url" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -15,35 +16,17 @@ import ( . "github.com/onsi/gomega" ) -// Helper function to create a basic auth string -func basicAuth(username, password string) string { - auth := username + ":" + password - return base64.StdEncoding.EncodeToString([]byte(auth)) -} - -// Helper function to create a new request -func newRequest(method, url string, body []byte) (*http.Request, error) { - req, err := http.NewRequest(method, url, bytes.NewReader(body)) +// Helper function to set up a new client and request +func setupRequest(method string, url *url.URL, body []byte) (*http.Request, error) { + req, err := http.NewRequest(method, url.String(), bytes.NewReader(body)) if err != nil { return nil, err } + req.Header.Add("Content-Type", "application/json") return req, nil } -// Helper function to set up a new client and request -func setupRequest(method, url, authHeader string, body []byte) (*http.Client, *http.Request, error) { - client := &http.Client{} - req, err := newRequest(method, url, body) - if err != nil { - return nil, nil, err - } - if authHeader != "" { - req.Header.Add("Authorization", authHeader) - } - return client, req, nil -} - var _ = Describe("CustomMetrics Server", func() { var ( resp *http.Response @@ -52,12 +35,21 @@ var _ = Describe("CustomMetrics Server", func() { err error scalingPolicy *models.ScalingPolicy client *http.Client - authHeader string + + serverURL *url.URL + healthURL *url.URL ) BeforeEach(func() { client = &http.Client{} fakeCredentials.ValidateReturns(true, nil) + + serverURL, err = url.Parse(fmt.Sprintf("http://127.0.0.1:%d", conf.Server.Port)) + Expect(err).NotTo(HaveOccurred()) + + // health url runs on the same port as metricsforwarder, maybe we need to roll back to use original port + healthURL, err = url.Parse(fmt.Sprintf("http://127.0.0.1:%d", conf.Server.Port)) + Expect(err).NotTo(HaveOccurred()) }) When("POST /v1/apps/some-app-id/metrics", func() { @@ -79,8 +71,9 @@ var _ = Describe("CustomMetrics Server", func() { body, err = json.Marshal(models.MetricsConsumer{InstanceIndex: 0, CustomMetrics: customMetrics}) Expect(err).NotTo(HaveOccurred()) - authHeader = "Basic " + basicAuth("username", "Password") - client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + serverURL.Path = "/v1/apps/an-app-id/metrics" + req, err = setupRequest("POST", serverURL, body) + req.SetBasicAuth("username", "password") Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) @@ -98,7 +91,8 @@ var _ = Describe("CustomMetrics Server", func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) - client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", "", body) + serverURL.Path = "/v1/apps/an-app-id/metrics" + req, err = setupRequest("POST", serverURL, body) Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) @@ -115,8 +109,8 @@ var _ = Describe("CustomMetrics Server", func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) - authHeader = basicAuth("username", "password") - client, req, err = setupRequest("POST", serverUrl+"/v1/apps/san-app-id/metrics", authHeader, body) + serverURL.Path = "/v1/apps/an-app-id/metrics" + req, err = setupRequest("POST", serverURL, body) Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) @@ -135,8 +129,9 @@ var _ = Describe("CustomMetrics Server", func() { fakeCredentials.ValidateReturns(false, errors.New("wrong credentials")) - authHeader = "Basic " + basicAuth("invalidUsername", "invalidPassword") - client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + serverURL.Path = "/v1/apps/an-app-id/metrics" + req, err = setupRequest("POST", serverURL, body) + req.SetBasicAuth("invalidUsername", "invalidPassword") Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) @@ -153,8 +148,9 @@ var _ = Describe("CustomMetrics Server", func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) - authHeader = "Basic " + basicAuth("username", "password") - client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + serverURL.Path = "/v1/apps/an-app-id/metrics" + req, err = setupRequest("POST", serverURL, body) + req.SetBasicAuth("username", "password") Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) @@ -186,8 +182,9 @@ var _ = Describe("CustomMetrics Server", func() { body, err = json.Marshal(models.MetricsConsumer{InstanceIndex: 0, CustomMetrics: customMetrics}) Expect(err).NotTo(HaveOccurred()) - authHeader = "Basic " + basicAuth("username", "password") - client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + serverURL.Path = "/v1/apps/an-app-id/metrics" + req, err = setupRequest("POST", serverURL, body) + req.SetBasicAuth("username", "password") Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) @@ -204,10 +201,16 @@ var _ = Describe("CustomMetrics Server", func() { }) When("the Health server is ready to serve RESTful API with basic Auth", func() { + var client *http.Client + + BeforeEach(func() { + healthURL.Path = "/health" + client = &http.Client{} + }) + When("username and password are incorrect for basic authentication during health check", func() { It("should return 401", func() { - client := &http.Client{} - req, err = http.NewRequest("GET", serverUrl+"/health", nil) + req, err = http.NewRequest("GET", healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) req.SetBasicAuth("wrongusername", "wrongpassword") rsp, err := client.Do(req) @@ -217,15 +220,11 @@ var _ = Describe("CustomMetrics Server", func() { }) When("username and password are correct for basic authentication during health check", func() { - BeforeEach(func() { - client = &http.Client{} - }) - When("a request to query health comes", func() { It("returns with a 200", func() { - req, err = http.NewRequest("GET", serverUrl, nil) + req, err = http.NewRequest("GET", healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) + req.SetBasicAuth(conf.Health.BasicAuth.Username, conf.Health.BasicAuth.Password) rsp, err := client.Do(req) Expect(err).NotTo(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) @@ -240,16 +239,18 @@ var _ = Describe("CustomMetrics Server", func() { }) It("should return 200 for /health", func() { - req, err = http.NewRequest("GET", serverUrl+"/health", nil) + healthURL.Path = "/health" + req, err = http.NewRequest("GET", healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) + req.SetBasicAuth(conf.Health.BasicAuth.Username, conf.Health.BasicAuth.Password) rsp, err := client.Do(req) Expect(err).ToNot(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) }) It("should return 200 for /health/readiness", func() { - req, err = http.NewRequest("GET", serverUrl+"/health/readiness", nil) + healthURL.Path = "/health/readiness" + req, err = http.NewRequest("GET", healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) rsp, err := client.Do(req) Expect(err).ToNot(HaveOccurred()) diff --git a/src/autoscaler/models/security.go b/src/autoscaler/models/security.go index cf94d8d5d5..62ca0e9e36 100644 --- a/src/autoscaler/models/security.go +++ b/src/autoscaler/models/security.go @@ -6,6 +6,13 @@ import ( "code.cloudfoundry.org/tlsconfig" ) +type BasicAuth struct { + Username string `yaml:"username"` + UsernameHash string `yaml:"username_hash"` + Password string `yaml:"password"` + PasswordHash string `yaml:"password_hash"` +} + type TLSCerts struct { KeyFile string `yaml:"key_file" json:"keyFile"` CertFile string `yaml:"cert_file" json:"certFile"` diff --git a/src/autoscaler/mta.tpl.yaml b/src/autoscaler/mta.tpl.yaml index 7f61c7dac9..4079cc8418 100644 --- a/src/autoscaler/mta.tpl.yaml +++ b/src/autoscaler/mta.tpl.yaml @@ -13,7 +13,7 @@ modules: GO_INSTALL_PACKAGE_SPEC: code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/cmd/metricsforwarder DT_RELEASE_BUILD_VERSION: ${mta-version} requires: - - name: config + - name: metricsforwarder-config - name: policydb - name: syslog-client - name: app-autoscaler-application-logs @@ -27,15 +27,40 @@ modules: build-parameters: builder: custom commands: - - make vendor + - make clean vendor + - name: publicapiserver + type: go + path: . + properties: + GO_INSTALL_PACKAGE_SPEC: code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/cmd/api + requires: + - name: publicapiserver-config + - name: policydb + - name: app-autoscaler-application-logs + parameters: + memory: 1G + disk-quota: 1G + instances: 2 + stack: cflinuxfs4 + routes: + build-parameters: + builder: custom + commands: + - make clean vendor resources: -- name: config +- name: metricsforwarder-config type: org.cloudfoundry.user-provided-service parameters: service-tags: - - config + - metricsforwarder-config path: metricsforwarder/default_config.json +- name: publicapiserver-config + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - publicapiserver-config + path: api/default_config.json - name: policydb type: org.cloudfoundry.user-provided-service parameters: diff --git a/src/autoscaler/operator/cmd/operator/main.go b/src/autoscaler/operator/cmd/operator/main.go index 370fef6553..513fbcef95 100644 --- a/src/autoscaler/operator/cmd/operator/main.go +++ b/src/autoscaler/operator/cmd/operator/main.go @@ -82,20 +82,12 @@ func main() { policyDb := sqldb.CreatePolicyDb(conf.AppSyncer.DB, logger) defer func() { _ = policyDb.Close() }() - promRegistry := prometheus.NewRegistry() - healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ - healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "policyDB", policyDb), - - healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "appMetricsDB", appMetricsDB), - healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "scalingEngineDB", scalingEngineDB), - }, true, logger.Session("operator-prometheus")) - - scalingEngineHttpclient, err := helpers.CreateHTTPClient(&conf.ScalingEngine.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scaling_client")) + scalingEngineHttpclient, err := helpers.CreateHTTPSClient(&conf.ScalingEngine.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scaling_client")) if err != nil { logger.Error("failed to create http client for scalingengine", err, lager.Data{"scalingengineTLS": conf.ScalingEngine.TLSClientCerts}) os.Exit(1) } - schedulerHttpclient, err := helpers.CreateHTTPClient(&conf.Scheduler.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scheduler_client")) + schedulerHttpclient, err := helpers.CreateHTTPSClient(&conf.Scheduler.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("scheduler_client")) if err != nil { logger.Error("failed to create http client for scheduler", err, lager.Data{"schedulerTLS": conf.Scheduler.TLSClientCerts}) os.Exit(1) @@ -143,7 +135,14 @@ func main() { }) members = append(grouper.Members{{"db-lock-maintainer", dbLockMaintainer}}, members...) - healthServer, err := healthendpoint.NewServerWithBasicAuth(conf.Health, []healthendpoint.Checker{}, logger.Session("health-server"), promRegistry, time.Now) + gatherer := createPrometheusRegistry(policyDb, appMetricsDB, scalingEngineDB, logger) + healthRouter, err := healthendpoint.NewHealthRouter(conf.Health, []healthendpoint.Checker{}, logger, gatherer, time.Now) + if err != nil { + logger.Error("failed to create health router", err) + os.Exit(1) + } + + healthServer, err := helpers.NewHTTPServer(logger, conf.Health.ServerConfig, healthRouter) if err != nil { logger.Error("failed to create health server", err) os.Exit(1) @@ -162,3 +161,13 @@ func main() { logger.Info("exited") } + +func createPrometheusRegistry(policyDB db.PolicyDB, appMetricsDB db.AppMetricDB, scalingEngineDB db.ScalingEngineDB, logger lager.Logger) *prometheus.Registry { + promRegistry := prometheus.NewRegistry() + healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ + healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "policyDB", policyDB), + healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "appMetricsDB", appMetricsDB), + healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "scalingEngineDB", scalingEngineDB), + }, true, logger.Session("operator-prometheus")) + return promRegistry +} diff --git a/src/autoscaler/operator/cmd/operator/operator_suite_test.go b/src/autoscaler/operator/cmd/operator/operator_suite_test.go index 61aa719124..9e7c63f859 100644 --- a/src/autoscaler/operator/cmd/operator/operator_suite_test.go +++ b/src/autoscaler/operator/cmd/operator/operator_suite_test.go @@ -74,7 +74,7 @@ func initConfig() { Secret: "secret", } healthport = 8000 + GinkgoParallelProcess() - cfg.Health.Port = healthport + cfg.Health.ServerConfig.Port = healthport cfg.Logging.Level = "debug" dbUrl := testhelpers.GetDbUrl() @@ -124,8 +124,8 @@ func initConfig() { cfg.AppSyncer.SyncInterval = 60 * time.Second cfg.HttpClientTimeout = 10 * time.Second - cfg.Health.HealthCheckUsername = "operatorhealthcheckuser" - cfg.Health.HealthCheckPassword = "operatorhealthcheckuser" + cfg.Health.BasicAuth.Username = "operatorhealthcheckuser" + cfg.Health.BasicAuth.Password = "operatorhealthcheckuser" } func writeConfig(c *config.Config) *os.File { diff --git a/src/autoscaler/operator/cmd/operator/operator_test.go b/src/autoscaler/operator/cmd/operator/operator_test.go index adc2e9f1aa..1db13b2099 100644 --- a/src/autoscaler/operator/cmd/operator/operator_test.go +++ b/src/autoscaler/operator/cmd/operator/operator_test.go @@ -133,9 +133,9 @@ var _ = Describe("Operator", Serial, func() { Eventually(runner.Session.Buffer, 5*time.Second).Should(Say("operator.started")) secondRunner = NewOperatorRunner() secondRunner.startCheck = "" - cfg.Health.HealthCheckUsername = "" - cfg.Health.HealthCheckPassword = "" - cfg.Health.Port = 9000 + GinkgoParallelProcess() + cfg.Health.BasicAuth.Username = "" + cfg.Health.BasicAuth.Password = "" + cfg.Health.ServerConfig.Port = 9000 + GinkgoParallelProcess() secondRunner.configPath = writeConfig(&cfg).Name() secondRunner.Start() @@ -150,7 +150,7 @@ var _ = Describe("Operator", Serial, func() { Consistently(secondRunner.Session.Buffer, 5*time.Second).ShouldNot(Say("operator.successfully-acquired-lock")) By("checking the health endpoint of the standing-by instance") - rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d/health", cfg.Health.Port)) + rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d/health", cfg.Health.ServerConfig.Port)) Expect(err).NotTo(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) @@ -166,9 +166,9 @@ var _ = Describe("Operator", Serial, func() { secondRunner = NewOperatorRunner() secondRunner.startCheck = "" - cfg.Health.HealthCheckUsername = "" - cfg.Health.HealthCheckPassword = "" - cfg.Health.Port = 9000 + GinkgoParallelProcess() + cfg.Health.BasicAuth.Username = "" + cfg.Health.BasicAuth.Password = "" + cfg.Health.ServerConfig.Port = 9000 + GinkgoParallelProcess() secondRunner.configPath = writeConfig(&cfg).Name() secondRunner.Start() }) @@ -210,7 +210,7 @@ var _ = Describe("Operator", Serial, func() { runner.Start() Eventually(runner.Session.Buffer, 10*time.Second).Should(Say("operator.started")) secondRunner = NewOperatorRunner() - cfg.Health.Port = 9000 + GinkgoParallelProcess() + cfg.Health.ServerConfig.Port = 9000 + GinkgoParallelProcess() secondRunner.configPath = writeConfig(&cfg).Name() secondRunner.startCheck = "" secondRunner.Start() @@ -315,8 +315,8 @@ var _ = Describe("Operator", Serial, func() { Describe("when Health server is ready to serve RESTful API", func() { BeforeEach(func() { basicAuthConfig := cfg - basicAuthConfig.Health.HealthCheckUsername = "" - basicAuthConfig.Health.HealthCheckPassword = "" + basicAuthConfig.Health.BasicAuth.Username = "" + basicAuthConfig.Health.BasicAuth.Password = "" runner.configPath = writeConfig(&basicAuthConfig).Name() runner.Start() @@ -375,7 +375,7 @@ var _ = Describe("Operator", Serial, func() { req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth(cfg.Health.HealthCheckUsername, cfg.Health.HealthCheckPassword) + req.SetBasicAuth(cfg.Health.BasicAuth.Username, cfg.Health.BasicAuth.Password) rsp, err := healthHttpClient.Do(req) Expect(err).ToNot(HaveOccurred()) @@ -415,7 +415,7 @@ var _ = Describe("Operator", Serial, func() { req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth(cfg.Health.HealthCheckUsername, cfg.Health.HealthCheckPassword) + req.SetBasicAuth(cfg.Health.BasicAuth.Username, cfg.Health.BasicAuth.Password) rsp, err := healthHttpClient.Do(req) Expect(err).ToNot(HaveOccurred()) diff --git a/src/autoscaler/operator/config/config_test.go b/src/autoscaler/operator/config/config_test.go index 936e3d19c0..80210fee9d 100644 --- a/src/autoscaler/operator/config/config_test.go +++ b/src/autoscaler/operator/config/config_test.go @@ -50,7 +50,7 @@ var _ = Describe("Config", func() { Expect(conf.CF.ClientID).To(Equal("client-id")) Expect(conf.CF.Secret).To(Equal("client-secret")) Expect(conf.CF.SkipSSLValidation).To(Equal(false)) - Expect(conf.Health.Port).To(Equal(9999)) + Expect(conf.Health.ServerConfig.Port).To(Equal(9999)) Expect(conf.Logging.Level).To(Equal("debug")) Expect(conf.AppMetricsDB.DB).To(Equal(db.DatabaseConfig{ @@ -95,7 +95,7 @@ var _ = Describe("Config", func() { Expect(err).NotTo(HaveOccurred()) Expect(conf.Logging.Level).To(Equal(config.DefaultLoggingLevel)) - Expect(conf.Health.Port).To(Equal(8081)) + Expect(conf.Health.ServerConfig.Port).To(Equal(8081)) Expect(conf.AppMetricsDB.DB).To(Equal(db.DatabaseConfig{ URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", MaxOpenConnections: 0, @@ -195,7 +195,7 @@ scheduler: conf.AppSyncer.DB.URL = "postgres://pqgotest:password@exampl.com/pqgotest" conf.DBLock.DB.URL = "postgres://pqgotest:password@exampl.com/pqgotest" conf.HttpClientTimeout = 10 * time.Second - conf.Health.Port = 8081 + conf.Health.ServerConfig.Port = 8081 }) diff --git a/src/autoscaler/operator/config/testdata/valid.yml b/src/autoscaler/operator/config/testdata/valid.yml index 08d141c610..51418f6e8a 100644 --- a/src/autoscaler/operator/config/testdata/valid.yml +++ b/src/autoscaler/operator/config/testdata/valid.yml @@ -5,7 +5,8 @@ cf: secret: client-secret skip_ssl_validation: false health: - port: 9999 + server_config: + port: 9999 logging: level: "debug" app_metrics_db: diff --git a/src/autoscaler/scalingengine/apis/generate.go b/src/autoscaler/scalingengine/apis/generate.go new file mode 100644 index 0000000000..532c7b1a6d --- /dev/null +++ b/src/autoscaler/scalingengine/apis/generate.go @@ -0,0 +1,3 @@ +package apis + +//go:generate go run github.com/ogen-go/ogen/cmd/ogen --config ogen-config.yaml --package scalinghistory --target scalinghistory --clean ../../../../api/internal-scaling-history-api.openapi.yaml diff --git a/src/autoscaler/scalingengine/apis/ogen-config.yaml b/src/autoscaler/scalingengine/apis/ogen-config.yaml new file mode 100644 index 0000000000..5b85450eaa --- /dev/null +++ b/src/autoscaler/scalingengine/apis/ogen-config.yaml @@ -0,0 +1,2 @@ +parser: + allow_remote: true diff --git a/src/autoscaler/scalingengine/client/security.go b/src/autoscaler/scalingengine/client/security.go new file mode 100644 index 0000000000..244f27088f --- /dev/null +++ b/src/autoscaler/scalingengine/client/security.go @@ -0,0 +1,4 @@ +package client + +type SecuritySource struct { +} diff --git a/src/autoscaler/scalingengine/cmd/scalingengine/main.go b/src/autoscaler/scalingengine/cmd/scalingengine/main.go index 86a66994c7..91679fbc6d 100644 --- a/src/autoscaler/scalingengine/cmd/scalingengine/main.go +++ b/src/autoscaler/scalingengine/cmd/scalingengine/main.go @@ -4,11 +4,9 @@ import ( "flag" "fmt" "os" - "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db/sqldb" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/config" @@ -16,7 +14,6 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/server" "code.cloudfoundry.org/clock" "code.cloudfoundry.org/lager/v3" - "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" "github.com/tedsuo/ifrit/grouper" "github.com/tedsuo/ifrit/sigmon" @@ -79,25 +76,17 @@ func main() { } defer func() { _ = schedulerDB.Close() }() - httpStatusCollector := healthendpoint.NewHTTPStatusCollector("autoscaler", "scalingengine") - promRegistry := prometheus.NewRegistry() - healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ - healthendpoint.NewDatabaseStatusCollector("autoscaler", "scalingengine", "policyDB", policyDb), - healthendpoint.NewDatabaseStatusCollector("autoscaler", "scalingengine", "scalingengineDB", scalingEngineDB), - healthendpoint.NewDatabaseStatusCollector("autoscaler", "scalingengine", "schedulerDB", schedulerDB), - httpStatusCollector, - }, true, logger.Session("scalingengine-prometheus")) - scalingEngine := scalingengine.NewScalingEngine(logger, cfClient, policyDb, scalingEngineDB, eClock, conf.DefaultCoolDownSecs, conf.LockSize) synchronizer := schedule.NewActiveScheduleSychronizer(logger, schedulerDB, scalingEngineDB, scalingEngine) - httpServer, err := server.NewServer(logger.Session("http-server"), conf, scalingEngineDB, scalingEngine, synchronizer, httpStatusCollector) + server := server.NewServer(logger.Session("http-server"), conf, policyDb, scalingEngineDB, schedulerDB, scalingEngine, synchronizer) + httpServer, err := server.GetMtlsServer() if err != nil { logger.Error("failed to create http server", err) os.Exit(1) } - healthServer, err := healthendpoint.NewServerWithBasicAuth(conf.Health, []healthendpoint.Checker{}, logger.Session("health-server"), promRegistry, time.Now) + healthServer, err := server.GetHealthServer() if err != nil { logger.Error("failed to create health server", err) os.Exit(1) diff --git a/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_suite_test.go b/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_suite_test.go index 1e87800ebe..f7d77c37e1 100644 --- a/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_suite_test.go +++ b/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_suite_test.go @@ -78,7 +78,7 @@ var _ = SynchronizedBeforeSuite( conf.Server.TLS.KeyFile = filepath.Join(testCertDir, "scalingengine.key") conf.Server.TLS.CertFile = filepath.Join(testCertDir, "scalingengine.crt") conf.Server.TLS.CACertFile = filepath.Join(testCertDir, "autoscaler-ca.crt") - conf.Health.Port = healthport + conf.Health.ServerConfig.Port = healthport conf.Logging.Level = "debug" dbUrl := GetDbUrl() @@ -105,8 +105,8 @@ var _ = SynchronizedBeforeSuite( conf.LockSize = 32 conf.HttpClientTimeout = 10 * time.Second - conf.Health.HealthCheckUsername = "scalingenginehealthcheckuser" - conf.Health.HealthCheckPassword = "scalingenginehealthcheckpassword" + conf.Health.BasicAuth.Username = "scalingenginehealthcheckuser" + conf.Health.BasicAuth.Password = "scalingenginehealthcheckpassword" configFile = writeConfig(&conf) @@ -137,7 +137,7 @@ var _ = SynchronizedBeforeSuite( _, err = testDB.Exec(testDB.Rebind("INSERT INTO policy_json(app_id, policy_json, guid) values(?, ?, ?)"), appId, policy, "1234") FailOnError("insert failed", err) - httpClient = NewEventGeneratorClient() + httpClient = NewScalingEngineClient() healthHttpClient = &http.Client{} }) diff --git a/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go b/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go index 9c19c16ecc..cc94c100b9 100644 --- a/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go +++ b/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go @@ -2,6 +2,7 @@ package main_test import ( "io" + "strconv" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" @@ -17,17 +18,24 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "os" ) var _ = Describe("Main", func() { - var ( runner *ScalingEngineRunner + err error + + healthURL *url.URL + serverURL *url.URL ) BeforeEach(func() { runner = NewScalingEngineRunner() + serverURL, err = url.Parse("https://127.0.0.1:" + strconv.Itoa(conf.Server.Port)) + healthURL, err = url.Parse("http://127.0.0.1:" + strconv.Itoa(conf.Health.ServerConfig.Port)) + Expect(err).ToNot(HaveOccurred()) }) JustBeforeEach(func() { @@ -39,7 +47,6 @@ var _ = Describe("Main", func() { }) Describe("with a correct config", func() { - Context("when starting 1 scaling engine instance", func() { It("scaling engine should start", func() { Eventually(runner.Session.Buffer, 2*time.Second).Should(gbytes.Say(runner.startCheck)) @@ -49,10 +56,6 @@ var _ = Describe("Main", func() { It("http server starts directly", func() { Eventually(runner.Session.Buffer, 2*time.Second).Should(gbytes.Say("scalingengine.http-server.new-http-server")) }) - - It("health server starts directly", func() { - Eventually(runner.Session.Buffer, 2*time.Second).Should(gbytes.Say("scalingengine.health-server.new-http-server")) - }) }) Context("when starting multiple scaling engine instances", func() { @@ -65,7 +68,7 @@ var _ = Describe("Main", func() { secondConf := conf secondConf.Server.Port += 500 - secondConf.Health.Port += 500 + secondConf.Health.ServerConfig.Port += 500 secondRunner.configPath = writeConfig(&secondConf).Name() secondRunner.Start() }) @@ -160,9 +163,10 @@ var _ = Describe("Main", func() { body, err := json.Marshal(models.Trigger{Adjustment: "+1"}) Expect(err).NotTo(HaveOccurred()) - rsp, err := httpClient.Post(fmt.Sprintf("https://127.0.0.1:%d/v1/apps/%s/scale", port, appId), - "application/json", bytes.NewReader(body)) + serverURL.Path = fmt.Sprintf("/v1/apps/%s/scale", appId) + rsp, err := httpClient.Post(serverURL.String(), "application/json", bytes.NewReader(body)) Expect(err).NotTo(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) rsp.Body.Close() }) @@ -170,9 +174,9 @@ var _ = Describe("Main", func() { Context("when a request to retrieve scaling history comes", func() { It("returns with a 200", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d/v1/apps/%s/scaling_histories", port, appId), nil) + serverURL.Path = fmt.Sprintf("/v1/apps/%s/scaling_histories", appId) + req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil) Expect(err).NotTo(HaveOccurred()) - req.Header.Set("Authorization", "Bearer none") rsp, err := httpClient.Do(req) Expect(err).NotTo(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) @@ -182,10 +186,11 @@ var _ = Describe("Main", func() { It("handles the start and end of a schedule", func() { By("start of a schedule") - url := fmt.Sprintf("https://127.0.0.1:%d/v1/apps/%s/active_schedules/111111", port, appId) + serverURL.Path = fmt.Sprintf("/v1/apps/%s/active_schedules/111111", appId) + bodyReader := bytes.NewReader([]byte(`{"instance_min_count":1, "instance_max_count":5, "initial_min_instance_count":3}`)) - req, err := http.NewRequest(http.MethodPut, url, bodyReader) + req, err := http.NewRequest(http.MethodPut, serverURL.String(), bodyReader) Expect(err).NotTo(HaveOccurred()) rsp, err := httpClient.Do(req) @@ -194,7 +199,7 @@ var _ = Describe("Main", func() { rsp.Body.Close() By("end of a schedule") - req, err = http.NewRequest(http.MethodDelete, url, nil) + req, err = http.NewRequest(http.MethodDelete, serverURL.String(), nil) Expect(err).NotTo(HaveOccurred()) rsp, err = httpClient.Do(req) @@ -205,11 +210,10 @@ var _ = Describe("Main", func() { }) Describe("when Health server is ready to serve RESTful API", func() { - BeforeEach(func() { basicAuthConfig := conf - basicAuthConfig.Health.HealthCheckUsername = "" - basicAuthConfig.Health.HealthCheckPassword = "" + basicAuthConfig.Health.BasicAuth.Username = "" + basicAuthConfig.Health.BasicAuth.Password = "" runner.configPath = writeConfig(&basicAuthConfig).Name() }) @@ -219,7 +223,7 @@ var _ = Describe("Main", func() { Context("when a request to query health comes", func() { It("returns with a 200", func() { - rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d", healthport)) + rsp, err := httpClient.Get(healthURL.String()) Expect(err).NotTo(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) raw, _ := io.ReadAll(rsp.Body) @@ -237,19 +241,22 @@ var _ = Describe("Main", func() { }) Describe("when Health server is ready to serve RESTful API with basic Auth", func() { + BeforeEach(func() { + healthURL.Path = "/health" + }) + JustBeforeEach(func() { Eventually(runner.Session.Buffer, 2).Should(gbytes.Say("scalingengine.started")) }) Context("when username and password are incorrect for basic authentication during health check", func() { It("should return 401", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) req.SetBasicAuth("wrongusername", "wrongpassword") - rsp, err := healthHttpClient.Do(req) + rsp, err := httpClient.Do(req) Expect(err).ToNot(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) }) @@ -258,12 +265,12 @@ var _ = Describe("Main", func() { Context("when username and password are correct for basic authentication during health check", func() { It("should return 200", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) + req.SetBasicAuth(conf.Health.BasicAuth.Username, conf.Health.BasicAuth.Password) - rsp, err := healthHttpClient.Do(req) + rsp, err := httpClient.Do(req) Expect(err).ToNot(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) }) @@ -271,6 +278,10 @@ var _ = Describe("Main", func() { }) Describe("when Health server is ready to serve RESTful API with basic Auth", func() { + BeforeEach(func() { + healthURL.Path = "/health" + }) + JustBeforeEach(func() { Eventually(runner.Session.Buffer, 2).Should(gbytes.Say("scalingengine.started")) }) @@ -278,12 +289,12 @@ var _ = Describe("Main", func() { Context("when username and password are incorrect for basic authentication during health check", func() { It("should return 401", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) req.SetBasicAuth("wrongusername", "wrongpassword") - rsp, err := healthHttpClient.Do(req) + rsp, err := httpClient.Do(req) Expect(err).ToNot(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) }) @@ -292,12 +303,12 @@ var _ = Describe("Main", func() { Context("when username and password are correct for basic authentication during health check", func() { It("should return 200", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) + req, err := http.NewRequest(http.MethodGet, healthURL.String(), nil) Expect(err).NotTo(HaveOccurred()) - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) + req.SetBasicAuth(conf.Health.BasicAuth.Username, conf.Health.BasicAuth.Password) - rsp, err := healthHttpClient.Do(req) + rsp, err := httpClient.Do(req) Expect(err).ToNot(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) }) diff --git a/src/autoscaler/scalingengine/config/config_test.go b/src/autoscaler/scalingengine/config/config_test.go index eab991c81e..50aaa06eae 100644 --- a/src/autoscaler/scalingengine/config/config_test.go +++ b/src/autoscaler/scalingengine/config/config_test.go @@ -55,7 +55,7 @@ var _ = Describe("Config", func() { Expect(conf.Server.TLS.CertFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/server.crt")) Expect(conf.Server.TLS.CACertFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/ca.crt")) - Expect(conf.Health.Port).To(Equal(9999)) + Expect(conf.Health.ServerConfig.Port).To(Equal(9999)) Expect(conf.Logging.Level).To(Equal("debug")) Expect(conf.DB.PolicyDB).To(Equal( @@ -98,7 +98,7 @@ var _ = Describe("Config", func() { Expect(conf.CF.SkipSSLValidation).To(Equal(false)) Expect(conf.Server.Port).To(Equal(8080)) - Expect(conf.Health.Port).To(Equal(8081)) + Expect(conf.Health.ServerConfig.Port).To(Equal(8081)) Expect(conf.Logging.Level).To(Equal("info")) Expect(conf.DB.PolicyDB).To(Equal( db.DatabaseConfig{ @@ -144,7 +144,8 @@ server: BeforeEach(func() { configBytes = []byte(` health: - port: port + server_config: + port: port `) }) diff --git a/src/autoscaler/scalingengine/config/testdata/valid.yml b/src/autoscaler/scalingengine/config/testdata/valid.yml index 2aef197852..487d50b22c 100644 --- a/src/autoscaler/scalingengine/config/testdata/valid.yml +++ b/src/autoscaler/scalingengine/config/testdata/valid.yml @@ -10,7 +10,8 @@ server: cert_file: /var/vcap/jobs/autoscaler/config/certs/server.crt ca_file: /var/vcap/jobs/autoscaler/config/certs/ca.crt health: - port: 9999 + server_config: + port: 9999 logging: level: DeBug @@ -32,4 +33,4 @@ db: connection_max_lifetime: 60s defaultCoolDownSecs: 300 lockSize: 32 -http_client_timeout: 10s \ No newline at end of file +http_client_timeout: 10s diff --git a/src/autoscaler/scalingengine/server/scaling_history_handler.go b/src/autoscaler/scalingengine/server/scaling_history_handler.go index cdbf114df8..80f91a4655 100644 --- a/src/autoscaler/scalingengine/server/scaling_history_handler.go +++ b/src/autoscaler/scalingengine/server/scaling_history_handler.go @@ -12,17 +12,15 @@ import ( "github.com/ogen-go/ogen/ogenerrors" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/apis/scalinghistory" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/routes" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/apis/scalinghistory" "code.cloudfoundry.org/lager/v3" "net/http" ) -var ( - _ = scalinghistory.SecurityHandler(&ScalingHistoryHandler{}) -) +type SecuritySource struct{} type ScalingHistoryHandler struct { logger lager.Logger @@ -45,7 +43,7 @@ func (h *ScalingHistoryHandler) NewError(_ context.Context, err error) *scalingh result.SetStatusCode(http.StatusUnauthorized) result.SetResponse(scalinghistory.ErrorResponse{ Code: scalinghistory.NewOptString(http.StatusText(http.StatusUnauthorized)), - Message: scalinghistory.NewOptString("missing bearer authentication"), + Message: scalinghistory.NewOptString("missing authentication"), }) } else { result.SetStatusCode(http.StatusInternalServerError) @@ -57,11 +55,6 @@ func (h *ScalingHistoryHandler) NewError(_ context.Context, err error) *scalingh return result } -func (h *ScalingHistoryHandler) HandleBearerAuth(ctx context.Context, operationName string, t scalinghistory.BearerAuth) (context.Context, error) { - // This handler is a no-op, as this handler shall only be available internally via mTLS - return ctx, nil -} - func (h *ScalingHistoryHandler) V1AppsGUIDScalingHistoriesGet(ctx context.Context, params scalinghistory.V1AppsGUIDScalingHistoriesGetParams) (*scalinghistory.History, error) { appId := params.GUID // actually not necessary if a default is provided in the schema, however this is not exposed yet: diff --git a/src/autoscaler/scalingengine/server/scaling_history_handler_test.go b/src/autoscaler/scalingengine/server/scaling_history_handler_test.go index 7b6fb582b8..9e1666f9df 100644 --- a/src/autoscaler/scalingengine/server/scaling_history_handler_test.go +++ b/src/autoscaler/scalingengine/server/scaling_history_handler_test.go @@ -8,7 +8,7 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" "code.cloudfoundry.org/lager/v3/lagertest" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/apis/scalinghistory" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/apis/scalinghistory" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/server" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/src/autoscaler/scalingengine/server/server.go b/src/autoscaler/scalingengine/server/server.go index b9133179d1..d14f2171db 100644 --- a/src/autoscaler/scalingengine/server/server.go +++ b/src/autoscaler/scalingengine/server/server.go @@ -1,17 +1,20 @@ package server import ( + "time" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/apis/scalinghistory" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/routes" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/apis/scalinghistory" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/schedule" "code.cloudfoundry.org/lager/v3" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" @@ -26,15 +29,89 @@ func (vh VarsFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { vh(w, r, vars) } -func NewServer(logger lager.Logger, conf *config.Config, scalingEngineDB db.ScalingEngineDB, scalingEngine scalingengine.ScalingEngine, synchronizer schedule.ActiveScheduleSychronizer, httpStatusCollector healthendpoint.HTTPStatusCollector) (ifrit.Runner, error) { - handler := NewScalingHandler(logger, scalingEngineDB, scalingEngine) - syncHandler := NewSyncHandler(logger, synchronizer) +type Server struct { + logger lager.Logger + conf *config.Config + policyDB db.PolicyDB + scalingEngineDB db.ScalingEngineDB + schedulerDB db.SchedulerDB + scalingEngine scalingengine.ScalingEngine + synchronizer schedule.ActiveScheduleSychronizer +} + +func NewServer(logger lager.Logger, conf *config.Config, policyDB db.PolicyDB, scalingEngineDB db.ScalingEngineDB, schedulerDB db.SchedulerDB, scalingEngine scalingengine.ScalingEngine, synchronizer schedule.ActiveScheduleSychronizer) *Server { + return &Server{ + logger: logger, + conf: conf, + policyDB: policyDB, + scalingEngineDB: scalingEngineDB, + schedulerDB: schedulerDB, + scalingEngine: scalingEngine, + synchronizer: synchronizer, + } +} + +func (s *Server) GetHealthServer() (ifrit.Runner, error) { + httpStatusCollector := healthendpoint.NewHTTPStatusCollector("autoscaler", "scalingengine") + healthRouter, err := createHealthRouter(s.logger, s.conf, s.policyDB, s.scalingEngineDB, s.schedulerDB, httpStatusCollector) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) + } + + return helpers.NewHTTPServer(s.logger, s.conf.Health.ServerConfig, healthRouter) +} + +func (s *Server) GetMtlsServer() (ifrit.Runner, error) { + httpStatusCollector := healthendpoint.NewHTTPStatusCollector("autoscaler", "scalingengine") + scalingEngineRouter, err := createScalingEngineRouter(s.logger, s.scalingEngineDB, s.scalingEngine, s.synchronizer, httpStatusCollector, s.conf.Server) + if err != nil { + return nil, fmt.Errorf("failed to create scaling engine router: %w", err) + } + + // mainRouter := setupMainRouter(scalingEngineRouter, healthRouter) + + return helpers.NewHTTPServer(s.logger, s.conf.Server, scalingEngineRouter) +} + +func createPrometheusRegistry(policyDB db.PolicyDB, scalingEngineDB db.ScalingEngineDB, schedulerDB db.SchedulerDB, httpStatusCollector healthendpoint.HTTPStatusCollector, logger lager.Logger) *prometheus.Registry { + promRegistry := prometheus.NewRegistry() + //validate that db are not nil + + if policyDB == nil || scalingEngineDB == nil || schedulerDB == nil { + logger.Error("failed-to-create-prometheus-registry", fmt.Errorf("db is nil: have policyDB: %t, have scalingEngineDB: %t, have schedulerDB: %t", policyDB != nil, scalingEngineDB != nil, schedulerDB != nil)) + return promRegistry + } + + healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ + healthendpoint.NewDatabaseStatusCollector("autoscaler", "scalingengine", "policyDB", policyDB), + healthendpoint.NewDatabaseStatusCollector("autoscaler", "scalingengine", "scalingengineDB", scalingEngineDB), + healthendpoint.NewDatabaseStatusCollector("autoscaler", "scalingengine", "schedulerDB", schedulerDB), + httpStatusCollector, + }, true, logger.Session("scalingengine-prometheus")) + return promRegistry +} + +func createHealthRouter(logger lager.Logger, conf *config.Config, policyDB db.PolicyDB, scalingEngineDB db.ScalingEngineDB, schedulerDB db.SchedulerDB, httpStatusCollector healthendpoint.HTTPStatusCollector) (*mux.Router, error) { + checkers := []healthendpoint.Checker{} + gatherer := createPrometheusRegistry(policyDB, scalingEngineDB, schedulerDB, httpStatusCollector, logger) + healthRouter, err := healthendpoint.NewHealthRouter(conf.Health, checkers, logger.Session("health-server"), gatherer, time.Now) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) + } + return healthRouter, nil +} + +func createScalingEngineRouter(logger lager.Logger, scalingEngineDB db.ScalingEngineDB, scalingEngine scalingengine.ScalingEngine, synchronizer schedule.ActiveScheduleSychronizer, httpStatusCollector healthendpoint.HTTPStatusCollector, serverConfig helpers.ServerConfig) (*mux.Router, error) { httpStatusCollectMiddleware := healthendpoint.NewHTTPStatusCollectMiddleware(httpStatusCollector) + + se := NewScalingHandler(logger, scalingEngineDB, scalingEngine) + syncHandler := NewSyncHandler(logger, synchronizer) + r := routes.ScalingEngineRoutes() r.Use(otelmux.Middleware("scalingengine")) r.Use(httpStatusCollectMiddleware.Collect) - r.Get(routes.ScaleRouteName).Handler(VarsFunc(handler.Scale)) + r.Get(routes.ScaleRouteName).Handler(VarsFunc(se.Scale)) scalingHistoryHandler, err := newScalingHistoryHandler(logger, scalingEngineDB) if err != nil { @@ -42,21 +119,28 @@ func NewServer(logger lager.Logger, conf *config.Config, scalingEngineDB db.Scal } r.Get(routes.GetScalingHistoriesRouteName).Handler(scalingHistoryHandler) - r.Get(routes.SetActiveScheduleRouteName).Handler(VarsFunc(handler.StartActiveSchedule)) - r.Get(routes.DeleteActiveScheduleRouteName).Handler(VarsFunc(handler.RemoveActiveSchedule)) - r.Get(routes.GetActiveSchedulesRouteName).Handler(VarsFunc(handler.GetActiveSchedule)) + r.Get(routes.SetActiveScheduleRouteName).Handler(VarsFunc(se.StartActiveSchedule)) + r.Get(routes.DeleteActiveScheduleRouteName).Handler(VarsFunc(se.RemoveActiveSchedule)) + r.Get(routes.GetActiveSchedulesRouteName).Handler(VarsFunc(se.GetActiveSchedule)) r.Get(routes.SyncActiveSchedulesRouteName).Handler(VarsFunc(syncHandler.Sync)) - - return helpers.NewHTTPServer(logger, conf.Server, r) + return r, nil } +// func setupMainRouter(r *mux.Router, healthRouter *mux.Router) *mux.Router { +// mainRouter := mux.NewRouter() +// mainRouter.PathPrefix("/v1").Handler(r) +// mainRouter.PathPrefix("/health").Handler(healthRouter) +// mainRouter.PathPrefix("/").Handler(healthRouter) +// return mainRouter +// } + func newScalingHistoryHandler(logger lager.Logger, scalingEngineDB db.ScalingEngineDB) (http.Handler, error) { scalingHistoryHandler, err := NewScalingHistoryHandler(logger, scalingEngineDB) if err != nil { return nil, fmt.Errorf("error creating scaling history handler: %w", err) } - server, err := scalinghistory.NewServer(scalingHistoryHandler, scalingHistoryHandler) + server, err := scalinghistory.NewServer(scalingHistoryHandler) if err != nil { return nil, fmt.Errorf("error creating ogen scaling history server: %w", err) } diff --git a/src/autoscaler/scalingengine/server/server_test.go b/src/autoscaler/scalingengine/server/server_test.go index d3fbff3192..f36e331baf 100644 --- a/src/autoscaler/scalingengine/server/server_test.go +++ b/src/autoscaler/scalingengine/server/server_test.go @@ -1,6 +1,8 @@ package server_test import ( + "strconv" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -15,47 +17,20 @@ import ( "bytes" "encoding/json" - "fmt" "io" "net/http" + "net/url" ) -var ( - server ifrit.Process - serverUrl string - scalingEngineDB *fakes.FakeScalingEngineDB - sychronizer *fakes.FakeActiveScheduleSychronizer - httpStatusCollector *fakes.FakeHTTPStatusCollector -) - -var _ = SynchronizedBeforeSuite(func() []byte { - return nil -}, func(_ []byte) { - port := 2222 + GinkgoParallelProcess() - conf := &config.Config{ - Server: helpers.ServerConfig{ - Port: port, - }, - } - scalingEngineDB = &fakes.FakeScalingEngineDB{} - scalingEngine := &fakes.FakeScalingEngine{} - sychronizer = &fakes.FakeActiveScheduleSychronizer{} - httpStatusCollector = &fakes.FakeHTTPStatusCollector{} - - httpServer, err := NewServer(lager.NewLogger("test"), conf, scalingEngineDB, scalingEngine, sychronizer, httpStatusCollector) - Expect(err).NotTo(HaveOccurred()) - server = ginkgomon_v2.Invoke(httpServer) - serverUrl = fmt.Sprintf("http://127.0.0.1:%d", conf.Server.Port) -}) - -var _ = SynchronizedAfterSuite(func() { - ginkgomon_v2.Interrupt(server) -}, func() { -}) - var _ = Describe("Server", func() { var ( - urlPath string + serverUrl *url.URL + server ifrit.Process + scalingEngineDB *fakes.FakeScalingEngineDB + sychronizer *fakes.FakeActiveScheduleSychronizer + + conf *config.Config + rsp *http.Response req *http.Request body []byte @@ -66,22 +41,48 @@ var _ = Describe("Server", func() { ) BeforeEach(func() { + port := 2222 + GinkgoParallelProcess() + conf = &config.Config{ + Server: helpers.ServerConfig{ + Port: port, + }, + } + scalingEngineDB = &fakes.FakeScalingEngineDB{} + scalingEngine := &fakes.FakeScalingEngine{} + policyDb := &fakes.FakePolicyDB{} + schedulerDB := &fakes.FakeSchedulerDB{} + sychronizer = &fakes.FakeActiveScheduleSychronizer{} + + httpServer, err := NewServer(lager.NewLogger("test"), conf, policyDb, scalingEngineDB, schedulerDB, scalingEngine, sychronizer).GetMtlsServer() + Expect(err).NotTo(HaveOccurred()) + server = ginkgomon_v2.Invoke(httpServer) + serverUrl, err = url.Parse("http://127.0.0.1:" + strconv.Itoa(port)) + Expect(err).ToNot(HaveOccurred()) + }) + AfterEach(func() { + ginkgomon_v2.Interrupt(server) + }) + JustBeforeEach(func() { + req, err = http.NewRequest(method, serverUrl.String(), bodyReader) + Expect(err).NotTo(HaveOccurred()) + rsp, err = http.DefaultClient.Do(req) }) - Context("when triggering scaling action", func() { + When("triggering scaling action", func() { BeforeEach(func() { body, err = json.Marshal(models.Trigger{Adjustment: "+1"}) Expect(err).NotTo(HaveOccurred()) + bodyReader = bytes.NewReader(body) uPath, err := route.Get(routes.ScaleRouteName).URLPath("appid", "test-app-id") Expect(err).NotTo(HaveOccurred()) - urlPath = uPath.Path + serverUrl.Path = uPath.Path }) - Context("when requesting correctly", func() { - JustBeforeEach(func() { - rsp, err = http.Post(serverUrl+urlPath, "application/json", bytes.NewReader(body)) + When("requesting correctly", func() { + BeforeEach(func() { + method = http.MethodPost }) It("should return 200", func() { @@ -90,76 +91,44 @@ var _ = Describe("Server", func() { rsp.Body.Close() }) }) - - Context("when requesting the wrong path", func() { - JustBeforeEach(func() { - rsp, err = http.Post(serverUrl+"/not-exist-path", "application/json", bytes.NewReader(body)) - }) - - It("should return 404", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusNotFound)) - rsp.Body.Close() - }) - }) - }) - Context("when getting scaling histories", func() { + When("getting scaling histories", func() { BeforeEach(func() { uPath, err := route.Get(routes.GetScalingHistoriesRouteName).URLPath("guid", "8ea70e4e-e0bc-4e15-9d32-cd69daaf012a") Expect(err).NotTo(HaveOccurred()) - urlPath = uPath.Path + method = http.MethodGet + serverUrl.Path = uPath.Path }) - Context("when requesting correctly", func() { - JustBeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, serverUrl+urlPath, nil) - req.Header.Set("Authorization", "Bearer ignore") - Expect(err).NotTo(HaveOccurred()) - rsp, err = (&http.Client{}).Do(req) - }) + JustBeforeEach(func() { + req, err = http.NewRequest(method, serverUrl.String(), nil) + Expect(err).NotTo(HaveOccurred()) - It("should return 200", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - rsp.Body.Close() - }) }) - Context("when requesting the wrong path", func() { - JustBeforeEach(func() { - rsp, err = http.Get(serverUrl + "/not-exist-path") - }) - - It("should return 404", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusNotFound)) - rsp.Body.Close() - }) + It("should return 200", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + rsp.Body.Close() }) }) - Context("when requesting active shedule", func() { + When("requesting active shedule", func() { - JustBeforeEach(func() { - req, err = http.NewRequest(method, serverUrl+urlPath, bodyReader) + BeforeEach(func() { + uPath, err := route.Get(routes.SetActiveScheduleRouteName).URLPath("appid", "test-app-id", "scheduleid", "test-schedule-id") Expect(err).NotTo(HaveOccurred()) - rsp, err = http.DefaultClient.Do(req) + serverUrl.Path = uPath.Path + method = http.MethodPut }) - Context("when setting active schedule", func() { + When("setting active schedule", func() { BeforeEach(func() { - uPath, err := route.Get(routes.SetActiveScheduleRouteName).URLPath("appid", "test-app-id", "scheduleid", "test-schedule-id") - Expect(err).NotTo(HaveOccurred()) - urlPath = uPath.Path bodyReader = bytes.NewReader([]byte(`{"instance_min_count":1, "instance_max_count":5, "initial_min_instance_count":3}`)) }) - Context("when requesting correctly", func() { - BeforeEach(func() { - method = http.MethodPut - }) + When("credentials are correct", func() { It("should return 200", func() { Expect(err).ToNot(HaveOccurred()) @@ -170,8 +139,7 @@ var _ = Describe("Server", func() { Context("when requesting the wrong path", func() { BeforeEach(func() { - method = http.MethodPut - urlPath = "/not-exist" + serverUrl.Path = "/not-exist" }) It("should return 404", func() { @@ -182,15 +150,16 @@ var _ = Describe("Server", func() { }) }) - Context("when deleting active schedule", func() { + When("deleting active schedule", func() { BeforeEach(func() { uPath, err := route.Get(routes.DeleteActiveScheduleRouteName).URLPath("appid", "test-app-id", "scheduleid", "test-schedule-id") Expect(err).NotTo(HaveOccurred()) - urlPath = uPath.Path + serverUrl.Path = uPath.Path bodyReader = nil method = http.MethodDelete }) - Context("when requesting correctly", func() { + + When("requesting correctly", func() { It("should return 200", func() { Expect(err).ToNot(HaveOccurred()) Expect(rsp.StatusCode).To(Equal(http.StatusOK)) @@ -200,7 +169,7 @@ var _ = Describe("Server", func() { Context("when requesting the wrong path", func() { BeforeEach(func() { - urlPath = "/not-exist" + serverUrl.Path = "/not-exist" }) It("should return 404", func() { @@ -211,16 +180,16 @@ var _ = Describe("Server", func() { }) }) - Context("when getting active schedule", func() { + When("getting active schedule", func() { BeforeEach(func() { uPath, err := route.Get(routes.GetActiveSchedulesRouteName).URLPath("appid", "test-app-id") Expect(err).NotTo(HaveOccurred()) - urlPath = uPath.Path + serverUrl.Path = uPath.Path bodyReader = nil method = http.MethodGet }) - Context("when requesting correctly", func() { + When("requesting correctly", func() { BeforeEach(func() { activeSchedule := &models.ActiveSchedule{ ScheduleId: "a-schedule-id", @@ -241,20 +210,15 @@ var _ = Describe("Server", func() { }) }) - Context("when requesting sync shedule", func() { - JustBeforeEach(func() { + When("requesting sync shedule", func() { + BeforeEach(func() { uPath, err := route.Get(routes.SyncActiveSchedulesRouteName).URLPath() Expect(err).NotTo(HaveOccurred()) - urlPath = uPath.Path + serverUrl.Path = uPath.Path bodyReader = nil - - req, err = http.NewRequest(method, serverUrl+urlPath, bodyReader) - Expect(err).NotTo(HaveOccurred()) - rsp, err = http.DefaultClient.Do(req) - Expect(err).NotTo(HaveOccurred()) }) - Context("when requesting correctly", func() { + When("requesting correctly", func() { BeforeEach(func() { method = http.MethodPut }) @@ -267,7 +231,7 @@ var _ = Describe("Server", func() { }) }) - Context("when requesting with incorrect http method", func() { + When("requesting with incorrect http method", func() { BeforeEach(func() { method = http.MethodGet }) diff --git a/src/autoscaler/testhelpers/clients.go b/src/autoscaler/testhelpers/clients.go index eecf376d1b..0c7c338e6e 100644 --- a/src/autoscaler/testhelpers/clients.go +++ b/src/autoscaler/testhelpers/clients.go @@ -31,6 +31,10 @@ func NewSchedulerClient() *http.Client { return CreateClientFor("scheduler") } +func NewScalingEngineClient() *http.Client { + return CreateClientFor("scalingengine") +} + func CreateClientFor(name string) *http.Client { certFolder := TestCertFolder() return CreateClient(filepath.Join(certFolder, name+".crt"), diff --git a/src/autoscaler/testhelpers/files.go b/src/autoscaler/testhelpers/files.go index c92cd9a5bb..6bb63aab23 100644 --- a/src/autoscaler/testhelpers/files.go +++ b/src/autoscaler/testhelpers/files.go @@ -12,3 +12,15 @@ func LoadFile(filename string) string { FailOnError("Could not read file", err) return string(file) } + +func BytesToFile(b []byte) string { + if len(b) == 0 { + return "" + } + + file, err := os.CreateTemp("", "") + FailOnError("Could create file", err) + _, err = file.Write(b) + FailOnError("Could write file", err) + return file.Name() +}