Tests refactoring (using docker compose) and upgrade clickhouse-server to 24.8 in examples and test manifests

.github/workflows/check_helm.yaml

@@ -21,7 +21,7 @@ jobs:
         with:
           driver: docker
           container-runtime: containerd
-          kubernetes-version: v1.31.0
+          kubernetes-version: v1.31.2
           cpus: max
           memory: max
 

.github/workflows/run_tests.yaml

@@ -47,7 +47,7 @@ jobs:
         with:
           driver: docker
           container-runtime: containerd
-          kubernetes-version: v1.31.0
+          kubernetes-version: v1.31.2
           cpus: max
           memory: max
 
@@ -66,6 +66,15 @@ jobs:
           minikube image load operator.tar
           minikube image load metrics-exporter.tar
 
+      - name: Deploy prometheus
+        run: |
+          cp ./deploy/prometheus/prometheus-sensitive-data.example.sh ./deploy/prometheus/prometheus-sensitive-data.sh
+          NO_WAIT=1 bash ./deploy/prometheus/create-prometheus.sh
+
+      - name: Deploy minio
+        run: |
+          NO_WAIT=1 bash ./deploy/minio/create-minio.sh
+
       - name: Run Tests
         id: run-tests
         continue-on-error: true
@@ -93,12 +102,22 @@ jobs:
             test_mode="--test-to-end"
           fi
 
-          ~/venv/qa/bin/python3 ./tests/regression.py --only=/regression/e2e.test_operator/${ONLY} $test_mode --trim-results on -o short --native --log ./tests/raw.log
+          for test_file in ./tests/e2e/test_operator*.py; do
+            name=$(basename "$test_file" .py | sed 's/^test_//')
+            run_cmd="~/venv/qa/bin/python3 ./tests/regression.py --only=/regression/e2e?test_${name}/${ONLY} $test_mode --trim-results on -o short --native --log ./tests/raw_${name}.log && "
+            run_cmd+="~/venv/qa/bin/tfs --no-colors transform compact ./tests/raw_${name}.log ./tests/compact_${name}.log.txt && "
+            run_cmd+="~/venv/qa/bin/tfs --no-colors transform nice ./tests/raw_${name}.log ./tests/nice_${name}.log.txt && "
+            run_cmd+="~/venv/qa/bin/tfs --no-colors transform short ./tests/raw_${name}.log ./tests/short_${name}.log.txt && "
+            run_cmd+="bash -xec '~/venv/qa/bin/tfs --no-colors report results -a '${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/' ./tests/raw_${name}.log - --confidential --copyright 'Altinity Inc.' --logo ./tests/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./tests/report_${name}.html'"
+
+            run_tests+=(
+              "${run_cmd}"
+            )
+          done          
+          printf "%s\n" "${run_tests[@]}" | xargs -P 2 -I {} bash -xec '{}'
+
+          ls -la ./tests/*.html
           test_result=$?
-          ~/venv/qa/bin/tfs --no-colors transform compact ./tests/raw.log ./tests/compact.log
-          ~/venv/qa/bin/tfs --no-colors transform nice ./tests/raw.log ./tests/nice.log.txt
-          ~/venv/qa/bin/tfs --no-colors transform short ./tests/raw.log ./tests/short.log.txt
-          ~/venv/qa/bin/tfs --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./tests/raw.log - --confidential --copyright "Altinity Inc." --logo ./tests/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./tests/report.html
 
           echo "test_result=$test_result" >> $GITHUB_OUTPUT
           exit "$test_result"
@@ -116,7 +135,7 @@ jobs:
         with:
           name: testflows-report
           path: |
-            tests/report.html
+            tests/*.html
           if-no-files-found: error
           retention-days: 90
 

.gitignore

@@ -42,6 +42,8 @@ venv
 
 # Tests cached files
 tests/image/cache
+tests/*.log.txt
+tests/*.html
 
 # Skip tmp folder
 /tmp/

Vagrantfile

@@ -121,15 +121,13 @@ Vagrant.configure(2) do |config|
     # docker
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
     add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) test"
-    apt-get install --no-install-recommends -y docker-ce pigz
+    apt-get install --no-install-recommends -y docker-ce docker-ce-cli containerd.io docker-compose-plugin pigz
 
-    # docker compose
     apt-get install -y --no-install-recommends python3-distutils
     curl -sL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
     python3 /tmp/get-pip.py
 
-    pip3 install -U setuptools
-    pip3 install -U docker-compose
+    pip3 install -U -r ./tests/image/requirements.txt
 
     # k9s CLI
     K9S_VERSION=$(curl -sL https://github.com/derailed/k9s/releases/latest -H "Accept: application/json" | jq -r .tag_name)
@@ -170,7 +168,8 @@ Vagrant.configure(2) do |config|
 #    K8S_VERSION=${K8S_VERSION:-1.23.1}
 #    K8S_VERSION=${K8S_VERSION:-1.24.8}
 #    K8S_VERSION=${K8S_VERSION:-1.25.4}
-    K8S_VERSION=${K8S_VERSION:-1.31.1}
+#    K8S_VERSION=${K8S_VERSION:-1.31.1}
+    K8S_VERSION=${K8S_VERSION:-1.31.2}
     export VALIDATE_YAML=true
 
     killall kubectl || true

config/chi/templates.d/001-templates.json.example

@@ -29,7 +29,7 @@
             "containers" : [
               {
                 "name": "clickhouse",
-                "image": "clickhouse/clickhouse-server:23.8",
+                "image": "clickhouse/clickhouse-server:24.8",
                 "ports": [
                   {
                     "name": "http",

deploy/builder/templates-config/chi/templates.d/001-templates.json.example

@@ -29,7 +29,7 @@
             "containers" : [
               {
                 "name": "clickhouse",
-                "image": "clickhouse/clickhouse-server:23.8",
+                "image": "clickhouse/clickhouse-server:24.8",
                 "ports": [
                   {
                     "name": "http",

deploy/clickhouse-keeper/clickhouse-keeper-manually/clickhouse-keeper-1-node-256M-for-test-only.yaml

@@ -378,13 +378,13 @@ spec:
       containers:
         - name: clickhouse-keeper
           imagePullPolicy: Always
-          image: "clickhouse/clickhouse-keeper:latest-alpine"
+          image: "clickhouse/clickhouse-keeper:24.8"
           resources:
             requests:
               memory: "256M"
               cpu: "1"
             limits:
-              memory: "4Gi"
+              memory: "1Gi"
               cpu: "2"
           volumeMounts:
             - name: clickhouse-keeper-settings

deploy/clickhouse-keeper/clickhouse-keeper-manually/clickhouse-keeper-3-nodes-256M-for-test-only.yaml

@@ -378,13 +378,13 @@ spec:
       containers:
         - name: clickhouse-keeper
           imagePullPolicy: Always
-          image: "clickhouse/clickhouse-keeper:latest-alpine"
+          image: "clickhouse/clickhouse-keeper:24.8"
           resources:
             requests:
               memory: "256M"
               cpu: "1"
             limits:
-              memory: "4Gi"
+              memory: "1Gi"
               cpu: "2"
           volumeMounts:
             - name: clickhouse-keeper-settings

deploy/helm/clickhouse-operator/values.yaml

@@ -584,7 +584,7 @@ configs:
                   "containers" : [
                     {
                       "name": "clickhouse",
-                      "image": "clickhouse/clickhouse-server:23.8",
+                      "image": "clickhouse/clickhouse-server:24.8",
                       "ports": [
                         {
                           "name": "http",

deploy/minio/install-minio-operator.sh

@@ -4,7 +4,7 @@ echo "External value for \$MINIO_NAMESPACE=$MINIO_NAMESPACE"
 echo "External value for \$MINIO_OPERATOR_VERSION=$MINIO_OPERATOR_VERSION"
 
 MINIO_NAMESPACE="${MINIO_NAMESPACE:-minio}"
-MINIO_OPERATOR_VERSION="${MINIO_OPERATOR_VERSION:-v4.1.3}"
+MINIO_OPERATOR_VERSION="${MINIO_OPERATOR_VERSION:-v6.0.4}"
 
 echo "Setup minio.io"
 echo "OPTIONS"
@@ -63,7 +63,9 @@ echo "Setup minio.io operator ${MINIO_OPERATOR_VERSION} into ${MINIO_NAMESPACE}
 ## TODO: need to refactor after next minio-operator release
 
 MINIO_KUSTOMIZE_DIR="${MINIO_OPERATOR_DIR}/resources"
+sed -i -e "s/replicas: 2/replicas: 1/" $MINIO_KUSTOMIZE_DIR/base/deployment.yaml
 sed -i -e "s/name: minio-operator/name: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/namespace.yaml
+sed -i -e "s/: restricted/: baseline/" $MINIO_KUSTOMIZE_DIR/base/namespace.yaml
 sed -i -e "s/namespace: default/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/*.yaml
 sed -i -e "s/namespace: minio-operator/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/*.yaml
 sed -i -e "s/namespace: minio-operator/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/kustomization.yaml

deploy/minio/install-minio-tenant.sh

@@ -1,29 +1,14 @@
 export MINIO_BACKUP_BUCKET=${MINIO_BACKUP_BUCKET:-clickhouse-backup}
 
 export MINIO_NAMESPACE="${MINIO_NAMESPACE:-minio}"
-# look to https://github.com/minio/operator/blob/v4.1.3/examples/tenant.yaml
-export MINIO_VERSION="${MINIO_VERSION:-RELEASE.2021-06-17T00-10-46Z}"
-# export MINIO_VERSION="${MINIO_VERSION:-latest}"
-export MINIO_CLIENT_VERSION="${MINIO_CLIENT_VERSION:-latest}"
-export MINIO_CONSOLE_VERSION="${MINIO_CONSOLE_VERSION:-latest}"
+# look to https://github.com/minio/operator/blob/master/examples/kustomization/base/tenant.yaml
+export MINIO_VERSION="${MINIO_VERSION:-RELEASE.2024-10-02T17-50-41Z}"
+export MINIO_CLIENT_VERSION="${MINIO_CLIENT_VERSION:-RELEASE.2024-10-29T15-34-59Z}"
 
 export MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minio-access-key}"
-export MINIO_ACCESS_KEY_B64=$(echo -n "$MINIO_ACCESS_KEY" | base64)
-
 export MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-minio-secret-key}"
-export MINIO_SECRET_KEY_B64=$(echo -n "$MINIO_SECRET_KEY" | base64)
-
-export MINIO_CONSOLE_PBKDF_PASSPHRASE="${MINIO_CONSOLE_PBKDF_PASSPHRASE}:-SECRET"
-export MINIO_CONSOLE_PBKDF_PASSPHRASE_B64=$(echo -n "${MINIO_CONSOLE_PBKDF_PASSPHRASE}" | base64)
-
-export MINIO_CONSOLE_PBKDF_SALT="${MINIO_CONSOLE_PBKDF_SALT}:-SECRET"
-export MINIO_CONSOLE_PBKDF_SALT_B64=$(echo -n "${MINIO_CONSOLE_PBKDF_SALT}" | base64)
-
 export MINIO_CONSOLE_ACCESS_KEY="${MINIO_CONSOLE_ACCESS_KEY:-minio_console}"
-export MINIO_CONSOLE_ACCESS_KEY_B64=$(echo -n "${MINIO_CONSOLE_ACCESS_KEY}" | base64)
-
 export MINIO_CONSOLE_SECRET_KEY="${MINIO_CONSOLE_SECRET_KEY:-minio_console}"
-export MINIO_CONSOLE_SECRET_KEY_B64=$(echo -n "${MINIO_CONSOLE_SECRET_KEY}" | base64)
 
 
 
@@ -36,8 +21,7 @@ export MINIO_CONSOLE_SECRET_KEY_B64=$(echo -n "${MINIO_CONSOLE_SECRET_KEY}" | ba
 function wait_minio_to_start() {
     # Fetch Minio's deployment_name and namespace from params
     local namespace=$1
-    local deployment_name=$2
-    local pod_name=$3
+    local pod_name=$2
 
     echo -n "Waiting Minio pod '${namespace}/${pod_name}' to start"
     # Check minio tenatna have all pods ready
@@ -47,13 +31,6 @@ function wait_minio_to_start() {
     done
     echo "...DONE"
 
-    echo -n "Waiting Minio Console Deployment '${namespace}/${deployment_name}' to start"
-    # Check minio-console deployment have all pods ready
-    while [[ $(kubectl --namespace="${namespace}" get deployments | grep "${deployment_name}" | grep -c "1/1") == "0" ]]; do
-        printf "."
-        sleep 1
-    done
-    echo "...DONE"
 }
 
 
@@ -78,7 +55,7 @@ kubectl apply -n "${MINIO_NAMESPACE}" -f <(
   envsubst < "$CUR_DIR/minio-tenant-template.yaml"
 )
 
-wait_minio_to_start "$MINIO_NAMESPACE" minio-console minio-pool-0-0
+wait_minio_to_start "$MINIO_NAMESPACE" minio-pool-0-0
 
 kubectl apply -n "${MINIO_NAMESPACE}" -f <(
   envsubst < "$CUR_DIR/minio-tenant-create-bucket-template.yaml"

deploy/minio/minio-tenant-create-bucket-template.yaml

@@ -9,11 +9,10 @@ spec:
       restartPolicy: Never
       containers:
         - name: minio-client
-          image: "minio/mc:${MINIO_CLIENT_VERSION}"
+          image: "quay.io/minio/mc:${MINIO_CLIENT_VERSION}"
           imagePullPolicy: IfNotPresent
           command:
             - /bin/bash
             - -xc
-            - mc alias set miniok8s https://minio-hl.minio:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} --insecure && mc mb miniok8s/${MINIO_BACKUP_BUCKET} --ignore-existing --insecure
-
+            - mc alias set miniok8s http://minio-hl.${MINIO_NAMESPACE}:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} && mc mb miniok8s/${MINIO_BACKUP_BUCKET} --ignore-existing --insecure
   backoffLimit: 10