From d7b9561f25000d2008cdced4ead615aae133d82e Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora Date: Thu, 27 Jun 2024 08:06:41 +0530 Subject: [PATCH 1/4] Run Autotune on AKS This PR adds initial commit to run Autotune on AKS Signed-off-by: Krishna Harsha Voora --- deploy.sh | 10 +- docs/autotune_install.md | 38 +- manifests/autotune/configmaps/aks-config.yaml | 28 ++ .../aks/kruize-crc-aks.yaml | 375 ++++++++++++++++++ scripts/aks-helpers.sh | 212 ++++++++++ scripts/prometheus_on_aks.sh | 103 +++++ 6 files changed, 760 insertions(+), 6 deletions(-) create mode 100644 manifests/autotune/configmaps/aks-config.yaml create mode 100644 manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml create mode 100755 scripts/aks-helpers.sh create mode 100755 scripts/prometheus_on_aks.sh diff --git a/deploy.sh b/deploy.sh index 1b34abae0..9597ae177 100755 --- a/deploy.sh +++ b/deploy.sh @@ -43,9 +43,10 @@ AUTOTUNE_QUERY_VARIABLES_MANIFEST_TEMPLATE="${AUTOTUNE_DIR}/autotune-query-varia AUTOTUNE_QUERY_VARIABLES_MANIFEST="${AUTOTUNE_DIR}/autotune-query-variables/query-variable.yaml" KRUIZE_CRC_DEPLOY_MANIFEST_OPENSHIFT="${CRC_DIR}/openshift/kruize-crc-openshift.yaml" KRUIZE_CRC_DEPLOY_MANIFEST_MINIKUBE="${CRC_DIR}/minikube/kruize-crc-minikube.yaml" +KRUIZE_CRC_DEPLOY_MANIFEST_AKS="${CRC_DIR}/aks/kruize-crc-aks.yaml" AUTOTUNE_PORT="8080" -AUTOTUNE_DOCKER_REPO="docker.io/kruize/autotune_operator" +AUTOTUNE_DOCKER_REPO="quay.io/kruize/autotune_operator" #Fetch autotune version from the pom.xml file. AUTOTUNE_VERSION="$(grep -A 1 "autotune" "${ROOT_DIR}"/pom.xml | grep version | awk -F '>' '{ split($2, a, "<"); print a[1] }')" @@ -63,6 +64,7 @@ HPO_DOCKER_IMAGE=${HPO_DOCKER_REPO}:${HPO_VERSION} # source all the helpers scripts . "${SCRIPTS_DIR}"/minikube-helpers.sh +. "${SCRIPTS_DIR}"/aks-helpers.sh . "${SCRIPTS_DIR}"/openshift-helpers.sh . "${SCRIPTS_DIR}"/common_utils.sh @@ -96,7 +98,7 @@ trap "ctrlc_handler" 1 2 3 function usage() { echo - echo "Usage: $0 [-a] [-k url] [-c [docker|minikube|openshift]] [-i autotune docker image] [-o hpo docker image] [-n namespace] [-d configmaps-dir ] [--timeout=x, x in seconds, for docker only]" + echo "Usage: $0 [-a] [-k url] [-c [docker|minikube|openshift|aks]] [-i autotune docker image] [-o hpo docker image] [-n namespace] [-d configmaps-dir ] [--timeout=x, x in seconds, for docker only]" echo " -s = start(default), -t = terminate" echo " -s: Deploy autotune [Default]" echo " -t: Terminate autotune deployment" @@ -113,7 +115,7 @@ function usage() { # Check if the cluster_type is one of icp or openshift function check_cluster_type() { case "${cluster_type}" in - docker | minikube | openshift) ;; + docker | minikube | openshift | aks) ;; *) echo "Error: unsupported cluster type: ${cluster_type}" @@ -187,7 +189,7 @@ done # Call the proper setup function based on the cluster_type if [ ${setup} == 1 ]; then if [ ${target} == "crc" ]; then - if [ ${cluster_type} == "minikube" ] || [ ${cluster_type} == "openshift" ]; then + if [ ${cluster_type} == "minikube" ] || [ ${cluster_type} == "openshift" ] || [ ${cluster_type} == "aks" ]; then ${cluster_type}_crc_start else echo "Unsupported cluster!" diff --git a/docs/autotune_install.md b/docs/autotune_install.md index bd7c1164e..fbbc0e61f 100644 --- a/docs/autotune_install.md +++ b/docs/autotune_install.md @@ -34,7 +34,7 @@ Autotune can be deployed to a supported Kubernetes cluster. We currently support Minikube setup with 8 CPUs and 16 GB Memory is recommended for autotune deployment. After setting up minikube, install prometheus from autotune repo with the following command ``` -$ ./scripts/prometheus_on_minikube.sh -as +$ ./scripts/prometheus_on_minikube.sh -as Info: installing prometheus... @@ -69,7 +69,7 @@ $ ./build.sh -i autotune:test Note - You can use the 'dev friendly mode' option to quickly build the autotune docker image using the cached layers. -# Deploy Autotune +# Deploy Autotune on minikube Let us now deploy autotune using the docker image onto the minikube cluster @@ -125,6 +125,40 @@ autotune-58cf47df84-rhqhx 1/1 Running 0 54s Info: Access Autotune at http://192.168.39.12:30113/listKruizeTunables ``` +### Install Prometheus on AKS + +This step assumes you have AKS installed and running,. After setting up AKS, install prometheus from autotune repo with the following command + +``` +$ ./scripts/prometheus_on_aks.sh -as + +Info: Installing Prometheus on AKS +... + +Info: Prometheus Setup on AKS +No resources found in monitoring namespace. +Info: Cloning Prometheus Operator & few AKS Customisations git + +``` +# Deploy Autotune on AKS + +Let us now deploy autotune container image onto the Azure Kubernetes cluster +``` +$ ./deploy.sh -c aks + + Usage: ./deploy.sh [-c [docker|minikube|openshift]] [-i autotune docker image] [-o hpo docker image] [-n namespace] [-d configmaps-dir ] [-s start] [-t terminate] + -s: Deploy autotune [Default] + -t: Terminate autotune deployment + -c: kubernetes cluster type. At present we support only minikube [Default - minikube] + -i: deploy with specific autotune operator docker image name [Default - kruize/autotune_operator:] + -o: deploy with specific hpo docker image name [Default - kruize/hpo:] + -n: Namespace to which autotune is deployed [Default - monitoring for cluster type minikube] + -d: Config maps directory [Default - manifests/configmaps] + + For example, + ./deploy.sh -c aks -i /autotune_operator:test -o /hpo:test +``` + # Demo Please see the [demo](https://github.com/kruize/kruize-demos) repo for more details on how to use Autotune. diff --git a/manifests/autotune/configmaps/aks-config.yaml b/manifests/autotune/configmaps/aks-config.yaml new file mode 100644 index 000000000..615293cbc --- /dev/null +++ b/manifests/autotune/configmaps/aks-config.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: autotune-config +data: + cluster_type: "kubernetes" + k8s_type: "aks" + auth_type: "" + auth_token: "" + monitoring_agent: "prometheus" + monitoring_service: "prometheus-k8s" + monitoring_agent_endpoint: "" + root_logging_level: "error" + logging_level: "info" + kruizeconfigjson: | + { + "datasource": [ + { + "name": "prometheus-1", + "provider": "prometheus", + "serviceName": "prometheus-k8s", + "namespace": "monitoring", + "url": "" + } + ] + } + + diff --git a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml new file mode 100644 index 000000000..532d94e55 --- /dev/null +++ b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml @@ -0,0 +1,375 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + storageClassName: managed + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment + namespace: monitoring + labels: + app: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + #securityContext: #todo + # runAsNonRoot: true + # runAsUser: 1000 + containers: + - name: postgres + image: quay.io/kruizehub/postgres:15.2 + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_PASSWORD + value: admin + - name: POSTGRES_USER + value: admin + - name: POSTGRES_DB + value: kruizeDB + ports: + - containerPort: 5432 + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-storage + persistentVolumeClaim: + claimName: postgres-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-service + namespace: monitoring + labels: + app: postgres +spec: + type: ClusterIP + ports: + - name: postgres-port + port: 5432 + targetPort: 5432 + selector: + app: postgres +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kruizeconfig + namespace: monitoring +data: + dbconfigjson: | + { + "database": { + "adminPassword": "admin", + "adminUsername": "admin", + "hostname": "postgres-service", + "name": "kruizeDB", + "password": "admin", + "port": 5432, + "sslMode": "require", + "username": "admin" + } + } + kruizeconfigjson: | + { + "clustertype": "kubernetes", + "k8stype": "minikube", + "authtype": "", + "monitoringagent": "prometheus", + "monitoringservice": "prometheus-k8s", + "monitoringendpoint": "prometheus-k8s", + "savetodb": "true", + "dbdriver": "jdbc:postgresql://", + "plots": "true", + "local": "false", + "logAllHttpReqAndResp": "true", + "hibernate": { + "dialect": "org.hibernate.dialect.PostgreSQLDialect", + "driver": "org.postgresql.Driver", + "c3p0minsize": 2, + "c3p0maxsize": 5, + "c3p0timeout": 300, + "c3p0maxstatements": 50, + "hbm2ddlauto": "none", + "showsql": "false", + "timezone": "UTC" + }, + "logging" : { + "cloudwatch": { + "accessKeyId": "", + "logGroup": "kruize-logs", + "logStream": "kruize-stream", + "region": "", + "secretAccessKey": "", + "logLevel": "INFO" + } + }, + "datasource": [ + { + "name": "prometheus-1", + "provider": "prometheus", + "serviceName": "prometheus-k8s", + "namespace": "monitoring", + "url": "" + } + ] + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kruize + labels: + app: kruize + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + name: kruize + template: + metadata: + labels: + app: kruize + name: kruize + spec: + containers: + - name: kruize + image: quay.io/kruize/autotune_operator:0.0.22_rm + imagePullPolicy: Always + volumeMounts: + - name: config-volume + mountPath: /etc/config + env: + - name: LOGGING_LEVEL + value: "info" + - name: ROOT_LOGGING_LEVEL + value: "error" + - name: DB_CONFIG_FILE + value: "/etc/config/dbconfigjson" + - name: KRUIZE_CONFIG_FILE + value: "/etc/config/kruizeconfigjson" + - name: JAVA_TOOL_OPTIONS + value: "-XX:MaxRAMPercentage=80" + ports: + - name: kruize-port + containerPort: 8080 + # livenessProbe: + # exec: + # command: + # - sh + # - -c + # - chmod +x /home/autotune/app/target/bin/TestDBConnection && /home/autotune/app/target/bin/TestDBConnection + # initialDelaySeconds: 10 + # periodSeconds: 5 + # timeoutSeconds: 20 + volumes: + - name: config-volume + configMap: + name: kruizeconfig +--- +apiVersion: v1 +kind: Service +metadata: + name: kruize + namespace: monitoring + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + labels: + app: kruize +spec: + type: NodePort + selector: + app: kruize + ports: + - name: kruize-port + port: 8080 + targetPort: 8080 +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: create-partition-cronjob + namespace: monitoring +spec: + schedule: "0 0 25 * *" # Run on 25th of every month at midnight + jobTemplate: + spec: + template: + spec: + containers: + - name: kruizecronjob + image: quay.io/kruize/autotune_operator:0.0.22_rm + imagePullPolicy: Always + volumeMounts: + - name: config-volume + mountPath: /etc/config + command: + - sh + - -c + - | + /home/autotune/app/target/bin/CreatePartition + args: [ "" ] + env: + - name: START_AUTOTUNE + value: "false" + - name: LOGGING_LEVEL + value: "info" + - name: ROOT_LOGGING_LEVEL + value: "error" + - name: DB_CONFIG_FILE + value: "/etc/config/dbconfigjson" + - name: KRUIZE_CONFIG_FILE + value: "/etc/config/kruizeconfigjson" + volumes: + - name: config-volume + configMap: + name: kruizeconfig + restartPolicy: OnFailure +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kruize-service-monitor + namespace: monitoring + labels: + app: kruize +spec: + selector: + matchLabels: + app: kruize + endpoints: + - port: kruize-port + interval: 30s + path: /metrics +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config + namespace: monitoring +data: + nginx.conf: | + events {} + http { + upstream kruize-api { + server kruize:8080; + } + + server { + listen 8080; + server_name localhost; + + root /usr/share/nginx/html; + + location ^~ /api/ { + rewrite ^/api(.*)$ $1 break; + proxy_pass http://kruize-api; + } + + location / { + index index.html; + error_page 404 =200 /index.html; + } + } + } +--- +apiVersion: v1 +kind: Service +metadata: + name: kruize-ui-nginx-service + namespace: monitoring +spec: + type: NodePort + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + app: kruize-ui-nginx +--- +apiVersion: v1 +kind: Pod +metadata: + name: kruize-ui-nginx-pod + namespace: monitoring + labels: + app: kruize-ui-nginx +spec: + containers: + - name: kruize-ui-nginx-container + image: quay.io/kruize/kruize-ui:0.0.3 + imagePullPolicy: Always + env: + - name: KRUIZE_UI_ENV + value: "production" + volumeMounts: + - name: nginx-config-volume + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + volumes: + - name: nginx-config-volume + configMap: + name: nginx-config +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: kruize-delete-partition-cronjob + namespace: monitoring +spec: + schedule: "0 0 25 * *" # Run on 25th of every month at midnight + jobTemplate: + spec: + template: + spec: + containers: + - name: kruizedeletejob + image: quay.io/kruize/autotune_operator:0.0.22_rm + imagePullPolicy: Always + volumeMounts: + - name: config-volume + mountPath: /etc/config + command: + - sh + - -c + - | + /home/autotune/app/target/bin/RetentionPartition + args: [ "" ] + env: + - name: START_AUTOTUNE + value: "false" + - name: LOGGING_LEVEL + value: "info" + - name: ROOT_LOGGING_LEVEL + value: "error" + - name: DB_CONFIG_FILE + value: "/etc/config/dbconfigjson" + - name: KRUIZE_CONFIG_FILE + value: "/etc/config/kruizeconfigjson" + - name: deletepartitionsthreshold + value: "15" + volumes: + - name: config-volume + configMap: + name: kruizeconfig + restartPolicy: OnFailure +--- diff --git a/scripts/aks-helpers.sh b/scripts/aks-helpers.sh new file mode 100755 index 000000000..335df3d68 --- /dev/null +++ b/scripts/aks-helpers.sh @@ -0,0 +1,212 @@ +#!/bin/bash +# +# Copyright (c) 2020, 2021 Red Hat, IBM Corporation and others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################### v AKS v ################################# + +function aks_first() { + #Create a namespace + echo "Create autotune namespace ${autotune_ns}" + kubectl create namespace ${autotune_ns} + + kubectl_cmd="kubectl -n ${autotune_ns}" + echo "Info: One time setup - Create a service account to deploy autotune" + + ${kubectl_cmd} apply -f ${AUTOTUNE_SA_MANIFEST} + check_err "Error: Failed to create service account and RBAC" + + ${kubectl_cmd} apply -f ${AUTOTUNE_OPERATOR_CRD} + check_err "Error: Failed to create autotune CRD" + + ${kubectl_cmd} apply -f ${AUTOTUNE_CONFIG_CRD} + check_err "Error: Failed to create autotuneconfig CRD" + + ${kubectl_cmd} apply -f ${AUTOTUNE_QUERY_VARIABLE_CRD} + check_err "Error: Failed to create autotunequeryvariable CRD" + + ${kubectl_cmd} apply -f ${AUTOTUNE_PERF_PROFILE_CRD} + check_err "Error: Failed to create autotunePerformanceProfile CRD" + + ${kubectl_cmd} apply -f ${AUTOTUNE_ROLE_MANIFEST} + check_err "Error: Failed to create role" + + sed -e "s|{{ AUTOTUNE_NAMESPACE }}|${autotune_ns}|" ${AUTOTUNE_RB_MANIFEST_TEMPLATE} > ${AUTOTUNE_RB_MANIFEST} + ${kubectl_cmd} apply -f ${AUTOTUNE_RB_MANIFEST} + check_err "Error: Failed to create role binding" + + sed -e "s|{{ AUTOTUNE_NAMESPACE }}|${autotune_ns}|" -e "s|{{ CLUSTER_TYPE }}|minikube|" ${AUTOTUNE_QUERY_VARIABLES_MANIFEST_TEMPLATE} > ${AUTOTUNE_QUERY_VARIABLES_MANIFEST} + ${kubectl_cmd} apply -f ${AUTOTUNE_QUERY_VARIABLES_MANIFEST} + check_err "Error: Failed to create query variables" + + ${kubectl_cmd} apply -f ${SERVICE_MONITOR_MANIFEST} + check_err "Error: Failed to create service monitor for Prometheus" +} + +# You can deploy using kubectl +function aks_deploy() { + echo + echo "Creating environment variable in minikube cluster using configMap" + ${kubectl_cmd} apply -f ${AUTOTUNE_CONFIGMAPS}/${cluster_type}-config.yaml + + echo + echo "Deploying AutotuneConfig objects" + ${kubectl_cmd} apply -f ${AUTOTUNE_CONFIGS} + + echo + echo "Deploying Performance Profile objects" + ${kubectl_cmd} apply -f ${AUTOTUNE_PERF_PROFILE_ROS} + + echo "Info: Deploying autotune yaml to minikube cluster" + + # Replace autotune docker image in deployment yaml + sed -e "s|{{ AUTOTUNE_IMAGE }}|${AUTOTUNE_DOCKER_IMAGE}|" ${AUTOTUNE_DEPLOY_MANIFEST_TEMPLATE} > ${AUTOTUNE_DEPLOY_MANIFEST} + #sed -i "s|{{ HPO_IMAGE }}|${HPO_DOCKER_IMAGE}|" ${AUTOTUNE_DEPLOY_MANIFEST} + sed -i'' -e "s|{{ HPO_IMAGE }}|${HPO_DOCKER_IMAGE}|" "${AUTOTUNE_DEPLOY_MANIFEST}" + + ${kubectl_cmd} apply -f ${AUTOTUNE_DEPLOY_MANIFEST} + sleep 2 + check_running autotune ${autotune_ns} + if [ "${err}" != "0" ]; then + # Indicate deploy failed on error + exit 1 + fi + + # Get the Autotune application port in minikube + MINIKUBE_IP=$(minikube ip) + AUTOTUNE_PORT=$(${kubectl_cmd} get svc autotune --no-headers -o=custom-columns=PORT:.spec.ports[*].nodePort) + echo "Info: Access Autotune at http://${MINIKUBE_IP}:${AUTOTUNE_PORT}/listKruizeTunables" + echo +} + +function aks_start() { + echo + echo "### Installing autotune for minikube" + echo + + # If autotune_ns was not set by the user + if [ -z "$autotune_ns" ]; then + autotune_ns="monitoring" + fi + + check_prometheus_installation + aks_first + aks_deploy +} + +function check_prometheus_installation() { + echo + echo "Info: Checking pre requisites for AKS..." + check_kustomize + + kubectl_cmd="kubectl" + prometheus_pod_running=$(${kubectl_cmd} get pods --all-namespaces | grep "prometheus-k8s-1") + if [ "${prometheus_pod_running}" == "" ]; then + echo "Prometheus is not running, use 'scripts/prometheus_on_aks.sh' to install." + exit 1 + fi + echo "Prometheus is installed and running." +} + + +function aks_terminate() { + # If autotune_ns was not set by the user + if [ -z "$autotune_ns" ]; then + autotune_ns="monitoring" + fi + + echo -n "### Removing autotune for AKS" + + kubectl_cmd="kubectl -n ${autotune_ns}" + + echo + echo "Removing Performance Profile" + ${kubectl_cmd} delete -f ${AUTOTUNE_PERF_PROFILE_CRD} 2>/dev/null + + echo + echo "Removing autotune" + ${kubectl_cmd} delete -f ${AUTOTUNE_DEPLOY_MANIFEST} 2>/dev/null + + echo + echo "Removing autotune service account" + ${kubectl_cmd} delete -f ${AUTOTUNE_SA_MANIFEST} 2>/dev/null + + echo + echo "Removing autotune role" + ${kubectl_cmd} delete -f ${AUTOTUNE_ROLE_MANIFEST} 2>/dev/null + + echo + echo "Removing autotune rolebinding" + ${kubectl_cmd} delete -f ${AUTOTUNE_RB_MANIFEST} 2>/dev/null + + echo + echo "Removing autotune serviceMonitor" + ${kubectl_cmd} delete -f ${SERVICE_MONITOR_MANIFEST} 2>/dev/null + + echo + echo "Removing AutotuneConfig objects" + ${kubectl_cmd} delete -f ${AUTOTUNE_CONFIGS} 2>/dev/null + + echo + echo "Removing AutotuneQueryVariable objects" + ${kubectl_cmd} delete -f ${AUTOTUNE_QUERY_VARIABLES} 2>/dev/null + + echo + echo "Removing Autotune configmap" + ${kubectl_cmd} delete -f ${AUTOTUNE_CONFIGMAPS}/${cluster_type}-config.yaml 2>/dev/null + + echo + echo "Removing Autotune CRD" + ${kubectl_cmd} delete -f ${AUTOTUNE_OPERATOR_CRD} 2>/dev/null + + echo + echo "Removing AutotuneConfig CRD" + ${kubectl_cmd} delete -f ${AUTOTUNE_CONFIG_CRD} 2>/dev/null + + echo + echo "Removing AutotuneQueryVariables CRD" + ${kubectl_cmd} delete -f ${AUTOTUNE_QUERY_VARIABLE_CRD} 2>/dev/null + + rm ${AUTOTUNE_DEPLOY_MANIFEST} + rm ${AUTOTUNE_RB_MANIFEST} + rm ${AUTOTUNE_QUERY_VARIABLES_MANIFEST} +} + +# Deploy kruize in remote monitoring mode +function aks_crc_start() { + echo + echo "### Installing kruize for minikube" + echo + # If autotune_ns was not set by the user + if [ -z "$autotune_ns" ]; then + autotune_ns="monitoring" + fi + CRC_MANIFEST_FILE=${KRUIZE_CRC_DEPLOY_MANIFEST_MINIKUBE} + + kruize_crc_start +} + +function aks_crc_terminate() { + # If autotune_ns was not set by the user + if [ -z "$autotune_ns" ]; then + autotune_ns="monitoring" + fi + kubectl_cmd="kubectl -n ${autotune_ns}" + CRC_MANIFEST_FILE=${KRUIZE_CRC_DEPLOY_MANIFEST_MINIKUBE} + + echo -n "### Removing Kruize for minikube" + echo + ${kubectl_cmd} delete -f ${CRC_MANIFEST_FILE} 2>/dev/null +} diff --git a/scripts/prometheus_on_aks.sh b/scripts/prometheus_on_aks.sh new file mode 100755 index 000000000..f0a3ed96c --- /dev/null +++ b/scripts/prometheus_on_aks.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Copyright (c) 2020, 2021 Red Hat, IBM Corporation and others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +current_dir="$(dirname "$0")" +non_interactive=0 +# Call setup by default (and not terminate) +setup=1 + +function install_prometheus(){ + echo + echo "Info: Prometheus Setup on AKS" + + prometheus_ns="monitoring" + kubectl_cmd="kubectl -n ${prometheus_ns}" + prometheus_pod_running=$(${kubectl_cmd} get pods | grep "prometheus-k8s-1") + + if [ "${prometheus_pod_running}" != "" ]; then + echo "Prometheus is already installed and running." + return; + fi + + rm -rf aks_prom_download || true + mkdir aks_prom_download 2>/dev/null + pushd aks_prom_download >/dev/null + echo "Info: Cloning Prometheus Operator & few AKS Customisations git" + git clone https://github.com/mukrishn/arohcp-workspace + git clone https://github.com/prometheus-operator/kube-prometheus + pushd kube-prometheus + kubectl apply --server-side -f manifests/setup + kubectl wait \ + --for condition=Established \ + --all CustomResourceDefinition \ + --namespace=monitoring + kubectl apply -f manifests/ + popd + echo "Info: Applying Custom config Changes" + pushd arohcp-workspace/aks-monitoring-config + kubectl apply -f prometheus-roleBindingSpecificNamespaces.yaml || true + kubectl apply -f prometheus-roleSpecificNamespaces.yaml || true + kubectl apply -f prom-public-svc.yaml || true +} + +function delete_prometheus(){ + echo "Info: Removing Prometheus Setup from AKS" + rm -rf aks_remove || true + mkdir aks_prom_remove 2>/dev/null + pushd aks_prom_remove + echo "Info: Cloning Prometheus Operator" + git clone https://github.com/prometheus-operator/kube-prometheus + pushd kube-prometheus + kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup +} + +# Input from user to install/delete prometheus +function usage() { + echo >&2 "usage: $0 [-a] [-s|t] where -a= non-interactive mode, -s=start, -t=terminate "; + exit 1; +} + +# empty argument validation +if [ $# -eq 0 ]; then + usage +fi + +while getopts "ast" option; +do + case "${option}" in + a) + non_interactive=1 + ;; + s) # For option s to install the prometheus + setup=1 + ;; + t) # For option t terminating and deleting the prometheus + setup=0 + ;; + \?) # For invalid option + usage + ;; + esac +done + +if [ "${setup}" == 1 ]; then + echo "Info: Installing Prometheus on AKS..." + install_prometheus +else + echo "Info: Deleting prometheus from AKS..." + delete_prometheus +fi From 73c6a0031564a517469baae9488fdd8361d999be Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora Date: Thu, 27 Jun 2024 15:25:20 +0530 Subject: [PATCH 2/4] Minor nits Signed-off-by: Krishna Harsha Voora --- scripts/aks-helpers.sh | 18 +++++++++--------- scripts/common_utils.sh | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/aks-helpers.sh b/scripts/aks-helpers.sh index 335df3d68..e04850f3b 100755 --- a/scripts/aks-helpers.sh +++ b/scripts/aks-helpers.sh @@ -47,7 +47,7 @@ function aks_first() { ${kubectl_cmd} apply -f ${AUTOTUNE_RB_MANIFEST} check_err "Error: Failed to create role binding" - sed -e "s|{{ AUTOTUNE_NAMESPACE }}|${autotune_ns}|" -e "s|{{ CLUSTER_TYPE }}|minikube|" ${AUTOTUNE_QUERY_VARIABLES_MANIFEST_TEMPLATE} > ${AUTOTUNE_QUERY_VARIABLES_MANIFEST} + sed -e "s|{{ AUTOTUNE_NAMESPACE }}|${autotune_ns}|" -e "s|{{ CLUSTER_TYPE }}|aks|" ${AUTOTUNE_QUERY_VARIABLES_MANIFEST_TEMPLATE} > ${AUTOTUNE_QUERY_VARIABLES_MANIFEST} ${kubectl_cmd} apply -f ${AUTOTUNE_QUERY_VARIABLES_MANIFEST} check_err "Error: Failed to create query variables" @@ -58,7 +58,7 @@ function aks_first() { # You can deploy using kubectl function aks_deploy() { echo - echo "Creating environment variable in minikube cluster using configMap" + echo "Creating environment variable in AKS cluster using configMap" ${kubectl_cmd} apply -f ${AUTOTUNE_CONFIGMAPS}/${cluster_type}-config.yaml echo @@ -69,7 +69,7 @@ function aks_deploy() { echo "Deploying Performance Profile objects" ${kubectl_cmd} apply -f ${AUTOTUNE_PERF_PROFILE_ROS} - echo "Info: Deploying autotune yaml to minikube cluster" + echo "Info: Deploying autotune yaml to AKS cluster" # Replace autotune docker image in deployment yaml sed -e "s|{{ AUTOTUNE_IMAGE }}|${AUTOTUNE_DOCKER_IMAGE}|" ${AUTOTUNE_DEPLOY_MANIFEST_TEMPLATE} > ${AUTOTUNE_DEPLOY_MANIFEST} @@ -84,7 +84,7 @@ function aks_deploy() { exit 1 fi - # Get the Autotune application port in minikube + # Get the Autotune application port in AKS MINIKUBE_IP=$(minikube ip) AUTOTUNE_PORT=$(${kubectl_cmd} get svc autotune --no-headers -o=custom-columns=PORT:.spec.ports[*].nodePort) echo "Info: Access Autotune at http://${MINIKUBE_IP}:${AUTOTUNE_PORT}/listKruizeTunables" @@ -93,7 +93,7 @@ function aks_deploy() { function aks_start() { echo - echo "### Installing autotune for minikube" + echo "### Installing autotune for AKS" echo # If autotune_ns was not set by the user @@ -187,13 +187,13 @@ function aks_terminate() { # Deploy kruize in remote monitoring mode function aks_crc_start() { echo - echo "### Installing kruize for minikube" + echo "### Installing kruize for AKS" echo # If autotune_ns was not set by the user if [ -z "$autotune_ns" ]; then autotune_ns="monitoring" fi - CRC_MANIFEST_FILE=${KRUIZE_CRC_DEPLOY_MANIFEST_MINIKUBE} + CRC_MANIFEST_FILE=${KRUIZE_CRC_DEPLOY_MANIFEST_AKS} kruize_crc_start } @@ -204,9 +204,9 @@ function aks_crc_terminate() { autotune_ns="monitoring" fi kubectl_cmd="kubectl -n ${autotune_ns}" - CRC_MANIFEST_FILE=${KRUIZE_CRC_DEPLOY_MANIFEST_MINIKUBE} + CRC_MANIFEST_FILE=${KRUIZE_CRC_DEPLOY_MANIFEST_AKS} - echo -n "### Removing Kruize for minikube" + echo -n "### Removing Kruize for AKS" echo ${kubectl_cmd} delete -f ${CRC_MANIFEST_FILE} 2>/dev/null } diff --git a/scripts/common_utils.sh b/scripts/common_utils.sh index f0908301f..43f7b1a63 100755 --- a/scripts/common_utils.sh +++ b/scripts/common_utils.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -x # # Copyright (c) 2020, 2020 Red Hat, IBM Corporation and others. # From ec76b6df16cc310d8d6851ce7c93943f1d473c22 Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora Date: Wed, 3 Jul 2024 00:21:55 +0530 Subject: [PATCH 3/4] Fixes postgres start-up issues Signed-off-by: Krishna Harsha Voora --- .../aks/kruize-crc-aks.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml index 532d94e55..ce32a99c1 100644 --- a/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml +++ b/manifests/crc/default-db-included-installation/aks/kruize-crc-aks.yaml @@ -28,9 +28,6 @@ spec: labels: app: postgres spec: - #securityContext: #todo - # runAsNonRoot: true - # runAsUser: 1000 containers: - name: postgres image: quay.io/kruizehub/postgres:15.2 @@ -42,11 +39,13 @@ spec: value: admin - name: POSTGRES_DB value: kruizeDB + - name: PGDATA + value: /var/lib/postgresql/backup ports: - containerPort: 5432 volumeMounts: - name: postgres-storage - mountPath: /var/lib/postgresql/data + mountPath: /var/lib/postgresql volumes: - name: postgres-storage persistentVolumeClaim: From 27e26f0b573a42f29a53937172258c6275cd57cc Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora Date: Fri, 5 Jul 2024 19:07:16 +0530 Subject: [PATCH 4/4] Minor nits Signed-off-by: Krishna Harsha Voora --- scripts/common_utils.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/common_utils.sh b/scripts/common_utils.sh index 18700a670..76b9306e4 100755 --- a/scripts/common_utils.sh +++ b/scripts/common_utils.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # # Copyright (c) 2020, 2020 Red Hat, IBM Corporation and others. #