diff --git a/.github/workflows/builders.yaml b/.github/workflows/builders.yaml new file mode 100644 index 0000000..2bae5a1 --- /dev/null +++ b/.github/workflows/builders.yaml @@ -0,0 +1,20 @@ +name: Builders +on: + workflow_call: +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: build and install + run: | + pip install . \ No newline at end of file diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml new file mode 100644 index 0000000..47b0d16 --- /dev/null +++ b/.github/workflows/ci-tests.yaml @@ -0,0 +1,23 @@ +name: CI tests +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + +jobs: + lint: + uses: ./.github/workflows/pylint.yaml + + build: + uses: ./.github/workflows/builders.yaml + + tests: + needs: build + uses: ./.github/workflows/tests.yaml + secrets: inherit + diff --git a/.github/workflows/build-push.yaml b/.github/workflows/image-push.yaml similarity index 89% rename from .github/workflows/build-push.yaml rename to .github/workflows/image-push.yaml index 2670210..f6873ae 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/image-push.yaml @@ -1,9 +1,13 @@ -name: Build and Push Image -on: [ push ] +name: Image Push +on: + push: + branches: + - master + - main jobs: build: - name: Build and push image + name: Image Push runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yaml similarity index 91% rename from .github/workflows/pylint.yml rename to .github/workflows/pylint.yaml index 25699ee..075ad9c 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yaml @@ -1,25 +1,26 @@ name: Pylint - -on: [push,pull_request] - +on: + workflow_call: jobs: - build: + lint: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.11"] steps: - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip pip install pylint pip install -r requirements.txt - pip install . + - name: Analysing the code with pylint run: | pylint -d C0103 -d R0912 $(git ls-files '*/*.py' '*.py') \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..9191983 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,45 @@ +name: Execute tests +on: + workflow_call: +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + + - name: Install bats + uses: bats-core/bats-action@1.5.4 + with: + bats-version: 1.10.0 + support-install: false + assert-install: false + detik-install: false + file-install: false + + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + pip install -r requirements.txt + pip install . + + - name: Execute Tests + run: | + bats -F pretty -p test.bats + env: + TERM: linux + QE_ES_SERVER: ${{ secrets.QE_ES_SERVER }} \ No newline at end of file diff --git a/examples/label-small-scale-cluster-density.yaml b/examples/label-small-scale-cluster-density.yaml index 924e0b1..13b69fd 100644 --- a/examples/label-small-scale-cluster-density.yaml +++ b/examples/label-small-scale-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS masterNodesType: m6a.xlarge diff --git a/examples/payload-scale-415.yaml b/examples/payload-scale-415.yaml index 475a98b..7d65d3e 100644 --- a/examples/payload-scale-415.yaml +++ b/examples/payload-scale-415.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed @@ -52,8 +52,8 @@ tests : value: duration agg_type: avg - name : aws-payload-scale-node-density - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/payload-scale-416.yaml b/examples/payload-scale-416.yaml index 6232235..5adb728 100644 --- a/examples/payload-scale-416.yaml +++ b/examples/payload-scale-416.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed @@ -52,8 +52,8 @@ tests : value: duration agg_type: avg - name : aws-payload-scale-node-density - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/readout-control-plane-cdv2.yaml b/examples/readout-control-plane-cdv2.yaml index 1fd0614..8d52bee 100644 --- a/examples/readout-control-plane-cdv2.yaml +++ b/examples/readout-control-plane-cdv2.yaml @@ -1,7 +1,7 @@ tests : - name : cluster-density-v2-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -37,8 +37,8 @@ tests : agg_type: avg - name : cluster-density-v2-120nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -74,8 +74,8 @@ tests : agg_type: avg - name : cluster-density-v2-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -111,8 +111,8 @@ tests : agg_type: avg - name : cdv2-kube-apiserver-etcd-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -160,8 +160,8 @@ tests : agg_type: avg - name : cd-v2-controller-manager-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -204,8 +204,8 @@ tests : agg_type: avg - name : cd-v2-api-request-latency-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -249,8 +249,8 @@ tests : agg_type: avg - name : cd-v2-etcd-latency-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS diff --git a/examples/readout-control-plane-node-density.yaml b/examples/readout-control-plane-node-density.yaml index c46c347..e7a73b3 100644 --- a/examples/readout-control-plane-node-density.yaml +++ b/examples/readout-control-plane-node-density.yaml @@ -1,7 +1,7 @@ tests : - name : node-density-heavy-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -49,8 +49,8 @@ tests : agg_type: avg - name : node-density-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -81,8 +81,8 @@ tests : agg_type: avg - name : node-density-120nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -113,8 +113,8 @@ tests : agg_type: avg - name : node-density-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -145,8 +145,8 @@ tests : agg_type: avg - name : node-density-cni-24nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -177,8 +177,8 @@ tests : agg_type: avg - name : node-density-cni-120nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS @@ -209,8 +209,8 @@ tests : agg_type: avg - name : node-density-cni-249nodes - index: ospst-perf-scale-ci* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: ocpVersion: 4.15 platform: AWS diff --git a/examples/small-scale-cluster-density.yaml b/examples/small-scale-cluster-density.yaml index c2b13b3..2cb7d4a 100644 --- a/examples/small-scale-cluster-density.yaml +++ b/examples/small-scale-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS masterNodesType: m6a.xlarge diff --git a/examples/small-scale-node-density-cni.yaml b/examples/small-scale-node-density-cni.yaml index 64b99f4..e45119d 100644 --- a/examples/small-scale-node-density-cni.yaml +++ b/examples/small-scale-node-density-cni.yaml @@ -1,7 +1,7 @@ tests : - name : aws-small-scale-node-density-cni - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS masterNodesType: m6a.xlarge diff --git a/examples/trt-external-payload-cluster-density.yaml b/examples/trt-external-payload-cluster-density.yaml index e672294..192a7f0 100644 --- a/examples/trt-external-payload-cluster-density.yaml +++ b/examples/trt-external-payload-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : payload-cluster-density-v2 - index: perf_scale_ci-* - benchmarkIndex: ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/examples/trt-payload-cluster-density.yaml b/examples/trt-payload-cluster-density.yaml index 713d3f5..192a7f0 100644 --- a/examples/trt-payload-cluster-density.yaml +++ b/examples/trt-payload-cluster-density.yaml @@ -1,7 +1,7 @@ tests : - name : payload-cluster-density-v2 - index: ospst-perf-scale-ci-* - benchmarkIndex: ospst-ripsaw-kube-burner* + index: {{ es_metadata_index }} + benchmarkIndex: {{ es_benchmark_index }} metadata: platform: AWS clusterType: self-managed diff --git a/test.bats b/test.bats new file mode 100644 index 0000000..8a1df04 --- /dev/null +++ b/test.bats @@ -0,0 +1,71 @@ +#!/usr/bin/env bats +# vi: ft=bash +# shellcheck disable=SC2086,SC2030,SC2031,SC2164 + + +check_file_list() { + for f in "${@}"; do + if [[ ! -f ${f} ]]; then + echo "File ${f} not found" + echo "Content of $(dirname ${f}):" + ls -l "$(dirname ${f})" + return 1 + fi + if [[ $(jq .[0].metricName ${f}) == "" ]]; then + echo "Incorrect format in ${f}" + cat "${f}" + return 1 + fi + done + return 0 +} + +run_cmd(){ + echo "$@" + ${@} +} + +setup() { + export ES_SERVER="$QE_ES_SERVER" + export es_metadata_index="perf_scale_ci*" + export es_benchmark_index="ripsaw-kube-burner*" +} + +@test "orion cmd label small scale cluster density with hunter-analyze" { + run_cmd orion cmd --config "examples/label-small-scale-cluster-density.yaml" --lookback 5d --hunter-analyze +} + +@test "orion cmd payload scale 4.15" { + run_cmd orion cmd --config "examples/payload-scale-415.yaml" --lookback 5d +} + +@test "orion cmd payload scale 4.16 without lookback period" { + run_cmd orion cmd --config "examples/payload-scale-416.yaml" +} + +@test "orion cmd readout control plane cdv2 with text output" { + run_cmd orion cmd --config "examples/readout-control-plane-cdv2.yaml" --lookback 5d --hunter-analyze --output-format text --save-output-path=output.txt + check_file_list output_cluster-density-v2-24nodes.txt +} + +@test "orion cmd readout control plane node-density with json output" { + run_cmd orion cmd --config "examples/readout-control-plane-node-density.yaml" --lookback 5d --hunter-analyze --output-format json --save-output-path=output.json + check_file_list output_node-density-heavy-24nodes.json +} + +@test "orion cmd readout netperf tcp with junit output" { + run_cmd orion cmd --config "examples/readout-netperf-tcp.yaml" --lookback 5d --output-format junit --hunter-analyze --save-output-path=output.xml + check_file_list output_k8s-netperf-tcp.xml +} + +@test "orion cmd small scale cluster density with anomaly detection" { + run_cmd orion cmd --config "examples/small-scale-cluster-density.yaml" --lookback 5d --anomaly-detection +} + +@test "orion cmd small scale node density cni with anomaly detection with a window" { + run_cmd orion cmd --config "examples/small-scale-node-density-cni.yaml" --anomaly-detection --anomaly-window 3 +} + +@test "orion cmd trt external payload cluster density with anomaly detection with minimum percentage" { + run_cmd orion cmd --config "examples/trt-external-payload-cluster-density.yaml" --anomaly-detection --anomaly-window 3 --min-anomaly-percent 5 +}