.github/ISSUE_TEMPLATE/benchmark-spec-request.md

@@ -0,0 +1,40 @@
+---
+name: Benchmark Specification Request
+about: File a benchmark SPEC request for improvement or addition
+title: "[SPEC]: "
+labels: ["spec", "triage"]
+assignees: 'filipecosta90'
+---
+
+**Improvement/Addition type**
+
+What changes to the current SPEC does this issue focuses upon?
+
+- [] Add a benchmark variant
+- [] Improve a benchmark variant
+- [] Raise awareness for a new Redis benchmark tool
+- [] Suggest a different Redis Build Variant
+- [] Other (fill details in section bellow)
+
+**Detail the requested use-case**
+
+Include reference to the benchmarks you've seen and want to see part of the spec.
+
+**Screenshots/CLI snippets of what's expected to happen**
+If applicable, add screenshots to help explain your problem. For example a chart.
+
+**Detail the used benchmark tool**
+
+What benchamrk tool to use:
+- [] memtier_benchmark
+- [] redis-benchmark
+- [] redis-benchmark-go
+- [] Other (name it):
+
+Sample benchmark tool command to implement the requested use-case:
+```
+<benchmark tool> -h <host> ...
+```
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,29 @@
+---
+name: Bug report
+about: Create a report to help us improve the benchmark SPEC implementation ( via redis-benchamrks-specification python package )
+title: "[BUG]: "
+labels: ["bug", "triage"]
+assignees: 'filipecosta90'
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. ...
+2. ....
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots/CLI snippets**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (please complete the following information):**
+ - OS: [e.g. ubuntu 20.04]
+ - PyPi version package [e.g. 0.5.9]
+ - Platfrom [e.g. x86, Jetson, ARM]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/release-drafter-config.yml

@@ -29,6 +29,11 @@ categories:
   - title: 'Tooling changes: Large scale coordinator'
     labels:
       - 'tool-large-coordinator'
+
+  - title: 'Tooling changes: Setup agnostic cli benchmark runner (solely client)'
+    labels:
+      - 'tool-client-runner'
+
   - title: 'Maintenance'
     label: 'chore'
 change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
@@ -49,6 +54,7 @@ template: |
   - [*Spec tool implementation: Benchmark artifacts builder*](https://github.com/filipecosta90/redis-benchmarks-specification/tree/main/redis_benchmarks_specification/__builder__/): **Experimental** components are released and available for beta testing.
   - [*Spec tool implementation: Self-contained coordinator*](https://github.com/filipecosta90/redis-benchmarks-specification/tree/main/redis_benchmarks_specification/__self_contained_coordinator__/): **Experimental** components are released and available for beta testing.
   - [*Spec tool implementation: Large scale coordinator*](https://github.com/filipecosta90/redis-benchmarks-specification/tree/main/): **Draft** components are under design, and have not been added to the specification.
+  - [*Spec tool implementation: Setup agnostic cli benchmark runner (solely client)](https://github.com/filipecosta90/redis-benchmarks-specification/tree/main/redis_benchmarks_specification/__runner__/): contains the client utility that triggers the actual benchmarks against an endpoint provided. This tool is setup agnostic and expects the DB to be properly spinned beforehand.
 
   ### Release changes:
 

.github/workflows/publish-pypi.yml

@@ -4,40 +4,35 @@ on:
     types: [ published ]
 
 jobs:
-  pytest:
+  pypi:
     name: Publish to PyPi
     runs-on: ubuntu-latest
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
-      - uses: actions/checkout@master
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
+    - uses: actions/checkout@master
 
-      - name: Install Poetry
-        uses: dschep/install-poetry-action@v1.3
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.10'
 
-      - name: Cache Poetry virtualenv
-        uses: actions/cache@v1
-        id: cache
-        with:
-          path: ~/.virtualenvs
-          key: poetry-${{ hashFiles('**/poetry.lock') }}
-          restore-keys: |
-            poetry-${{ hashFiles('**/poetry.lock') }}
+    - uses: docker/setup-docker-action@v4
 
-      - name: Set Poetry config
-        run: |
-          poetry config virtualenvs.in-project false
-          poetry config virtualenvs.path ~/.virtualenvs
+    - name: Install Poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
 
-      - name: Install Dependencies
-        run: poetry install
-        if: steps.cache.outputs.cache-hit != 'true'
+    - name: Install Dev requirements
+      run: |
+        pip install -U setuptools wheel
+        pip install -r dev_requirements.txt
 
-      - name: Publish to PyPI
-        if: github.event_name == 'release'
-        run: |
-          poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }} --build
+    - name: Install Dependencies
+      run: |
+        poetry install
+
+    - name: Publish to PyPI
+      if: github.event_name == 'release'
+      run: |
+        poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }} --build

.github/workflows/tox.yml

@@ -5,17 +5,20 @@ on:
     branches:
       - master
       - main
+
 jobs:
   pytest:
     strategy:
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
-        os: [ ubuntu-latest ]
+        python-version: [ '3.10', '3.11', '3.12' ]
+        os: [ "ubuntu-latest", "ubuntu-24.04" ]
+      fail-fast: false
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
-      USING_COVERAGE: "3.9"
+      USING_COVERAGE: "3.10"
+      USING_COVERAGE_OS: "ubuntu-latest"
 
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
     name: os ${{ matrix.os }} python ${{ matrix.python-version }} Linting, testing, and compliance
     steps:
     - uses: actions/checkout@master
@@ -25,21 +28,22 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - uses: docker-practice/actions-setup-docker@master
-
     - name: Install Poetry
-      uses: dschep/install-poetry-action@v1.3
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
 
-    - name: Install Tox
+    - name: Install Dev requirements
       run: |
-        pip3 install black coverage flake8 tox tox-docker tox-poetry
+        pip install -U setuptools wheel
+        pip install -r dev_requirements.txt
 
     - name: Run tox
       run: |
         tox
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2
       if: contains(env.USING_COVERAGE, matrix.python-version)
       with:
+        token: ${{secrets.CODECOV_TOKEN}}
         fail_ci_if_error: true

.github/workflows/validate-spec-fields.yml

@@ -0,0 +1,39 @@
+name: Validate SPEC fields
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  pytest:
+    name: Validate SPEC fields
+    runs-on: ubuntu-22.04
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.10'
+
+    - uses: docker-practice/actions-setup-docker@master
+
+    - name: Install Poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+
+    - name: Install Dev requirements
+      run: |
+        pip install -U setuptools wheel
+        pip install -r dev_requirements.txt
+
+    - name: Install Dependencies
+      run: |
+        poetry install
+
+    - name: Validate SPEC fields
+      run: |
+        poetry run redis-benchmarks-spec-cli --tool stats --fail-on-required-diff

.gitignore

@@ -1,3 +1,11 @@
+# work files
+*.csv
+utils/tests/test_data/dump.rdb
+
+# mac
+
+.DS_Store
+
 # Node rules:
 ## Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
 .grunt
@@ -222,3 +230,6 @@ fabric.properties
 
 # Json result files
 *.json
+
+# perf related
+*.out