Fixed wrong package dir on spec builder (#10)

* [add] Improved package description and badges

* [fix] Fixed wrong package dir on spec builder

Author: filipecosta90

.github/workflows/poetry-ci-test-lint.yml

@@ -0,0 +1,45 @@
+name: Run Tests using tox
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  pytest:
+    strategy:
+      matrix:
+        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+        os: [ ubuntu-latest ]
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      USING_COVERAGE: "3.9"
+
+    runs-on: ${{ matrix.os }}
+    name: os ${{ matrix.os }} python ${{ matrix.python-version }} Linting, testing, and compliance
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - uses: docker-practice/actions-setup-docker@master
+
+    - name: Install Poetry
+      uses: dschep/install-poetry-action@v1.3
+
+    - name: Install Tox
+      run: |
+        pip3 install black coverage flake8 tox tox-docker tox-poetry
+
+    - name: Run tox
+      run: |
+        tox
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      if: contains(env.USING_COVERAGE, matrix.python-version)
+      with:
+        fail_ci_if_error: true

.github/workflows/tox.yml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
 version = "0.1.0"
-description = ""
+description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>"]
 
 [tool.poetry.dependencies]
@@ -14,16 +14,19 @@ marshmallow = "^3.12.2"
 argparse = "^1.4.0"
 Flask-HTTPAuth = "^4.4.0"
 PyYAML = "^5.4.1"
-docker = "^5.0.0"
+docker = "^4.4.4"
 redisbench-admin = "^0.4.8"
 psutil = "^5.8.0"
-black = "20.8b1"
+tox-docker = "^3.0.0"
 
 [tool.poetry.dev-dependencies]
+black = "20.8b1"
 pytest = "^5.2"
 flake8 = "^3.9.2"
 pytest-cov = "^2.12.1"
 codecov = "^2.1.12"
+tox-poetry-installer = {extras = ["poetry"], version = "^0.8.1"}
+tox-docker = {extras = ["poetry"], version = "^3.0.0"}
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

poetry.lock

@@ -52,28 +52,31 @@ def verify_password(username, password):
     return result
 
 
-def commit_schema_to_stream(json_str: str):
+def commit_schema_to_stream(json_str: str, conn: redis.StrictRedis):
     """ uses to the provided JSON dict of fields and pushes that info to the corresponding stream  """
     fields = loads(json_str)
     reply_fields = loads(json_str)
     result = False
     error_msg = None
-    github_url = "https://github.com/redis/redis/archive/{}.zip".format(
-        fields["git_hash"]
-    )
-    try:
-        response = urlopen(github_url, timeout=5)
-        content = response.read()
-        fields["zip_archive"] = bytes(content)
-        fields["zip_archive_len"] = len(bytes(content))
-        reply_fields["archived_zip"] = True
-        result = True
-    except URLError as e:
-        error_msg = "Catched URLError while fetching {} content. Error {}".format(
-            github_url, e.__str__()
+    if "git_hash" not in fields:
+        error_msg = "Missing required 'git_hash' field"
+    else:
+        github_url = "https://github.com/redis/redis/archive/{}.zip".format(
+            fields["git_hash"]
         )
-        logging.error(error_msg)
-        result = False
+        try:
+            response = urlopen(github_url, timeout=5)
+            content = response.read()
+            fields["zip_archive"] = bytes(content)
+            fields["zip_archive_len"] = len(bytes(content))
+            reply_fields["archived_zip"] = True
+            result = True
+        except URLError as e:
+            error_msg = "Catched URLError while fetching {} content. Error {}".format(
+                github_url, e.__str__()
+            )
+            logging.error(error_msg)
+            result = False
 
     if result is True:
         id = conn.xadd(STREAM_KEYNAME_GH_EVENTS_COMMIT, fields)
@@ -84,6 +87,7 @@ def commit_schema_to_stream(json_str: str):
 
 class CommitSchema(Schema):
     git_branch = fields.String(required=False)
+    git_tag = fields.String(required=False)
     git_hash = fields.String(required=True)
 
 
@@ -103,7 +107,9 @@ def base():
     # Convert request body back to JSON str
     data_now_json_str = dumps(result)
 
-    result, response_data, err_message = commit_schema_to_stream(data_now_json_str)
+    result, response_data, err_message = commit_schema_to_stream(
+        data_now_json_str, conn
+    )
     if result is False:
         return jsonify(err_message), 400
 

pyproject.toml

@@ -3,6 +3,8 @@
 import logging
 import tempfile
 import shutil
+
+import docker
 import redis
 import os
 from zipfile import ZipFile, ZipInfo
@@ -20,7 +22,6 @@
     STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
     STREAM_KEYNAME_NEW_BUILD_EVENTS,
 )
-from redis_benchmarks_specification.__common__.package import PACKAGE_DIR
 
 
 class ZipFileWithPermissions(ZipFile):
@@ -73,10 +74,9 @@ def main():
             level=LOG_LEVEL,
             datefmt=LOG_DATEFMT,
         )
-    logging.info("Using package dir {} for inner file paths".format(PACKAGE_DIR))
-    builders_folder = os.path.abspath(
-        PACKAGE_DIR + "/" + args.setups_folder + "/builders"
-    )
+
+    builders_folder = os.path.abspath(args.setups_folder + "/builders")
+    logging.info("Using package dir {} for inner file paths".format(builders_folder))
     different_build_specs = os.listdir(builders_folder)
     logging.info(
         "Using the following build specs folder {}, containing {} different specs.".format(
@@ -106,35 +106,18 @@ def main():
         logging.error("Error message {}".format(e.__str__()))
         exit(1)
 
-    logging.info("checking build spec requirements")
-    already_checked_images = []
-    for build_spec in different_build_specs:
-        build_config, id = get_build_config(builders_folder + "/" + build_spec)
-        if build_config["kind"] == "docker":
-            build_image = build_config["build_image"]
-            if build_image not in already_checked_images:
-                logging.info(
-                    "Build {} requirement: checking build image {} is available.".format(
-                        id, build_image
-                    )
-                )
-                import docker
+    build_spec_image_prefetch(builders_folder, different_build_specs)
 
-                client = docker.from_env()
-                image = client.images.pull(build_image)
-                logging.info(
-                    "Build {} requirement: build image {} is available with id: {}.".format(
-                        id, build_image, image.id
-                    )
-                )
-                already_checked_images.append(build_image)
-            else:
-                logging.info(
-                    "Build {} requirement: build image {} availability was already checked.".format(
-                        id, build_image
-                    )
-                )
+    builder_consumer_group_create(conn)
 
+    previous_id = args.consumer_start_id
+    while True:
+        previous_id = builder_process_stream(
+            builders_folder, conn, different_build_specs, previous_id
+        )
+
+
+def builder_consumer_group_create(conn):
     try:
         conn.xgroup_create(
             STREAM_KEYNAME_GH_EVENTS_COMMIT,
@@ -152,21 +135,20 @@ def main():
                 STREAM_GH_EVENTS_COMMIT_BUILDERS_CG
             )
         )
-    previous_id = None
-    while True:
-        if previous_id is None:
-            previous_id = args.consumer_start_id
-        logging.info("Entering blocking read waiting for work.")
-        newTestInfo = conn.xreadgroup(
-            STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
-            "{}-proc#{}".format(STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, "1"),
-            {STREAM_KEYNAME_GH_EVENTS_COMMIT: previous_id},
-            count=1,
-            block=0,
-        )
-        if len(newTestInfo[0]) < 2 or len(newTestInfo[0][1]) < 1:
-            previous_id = ">"
-            continue
+
+
+def builder_process_stream(builders_folder, conn, different_build_specs, previous_id):
+    logging.info("Entering blocking read waiting for work.")
+    newTestInfo = conn.xreadgroup(
+        STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
+        "{}-proc#{}".format(STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, "1"),
+        {STREAM_KEYNAME_GH_EVENTS_COMMIT: previous_id},
+        count=1,
+        block=0,
+    )
+    if len(newTestInfo[0]) < 2 or len(newTestInfo[0][1]) < 1:
+        previous_id = ">"
+    else:
         streamId, testDetails = newTestInfo[0][1][0]
         logging.info("Received work . Stream id {}.".format(streamId))
         # commit = None
@@ -276,7 +258,37 @@ def main():
                         )
                     )
                 shutil.rmtree(temporary_dir, ignore_errors=True)
-
         else:
             logging.error("Missing commit information within received message.")
-            continue
+    return previous_id
+
+
+def build_spec_image_prefetch(builders_folder, different_build_specs):
+    logging.info("checking build spec requirements")
+    already_checked_images = []
+    for build_spec in different_build_specs:
+        build_config, id = get_build_config(builders_folder + "/" + build_spec)
+        if build_config["kind"] == "docker":
+            build_image = build_config["build_image"]
+            if build_image not in already_checked_images:
+                logging.info(
+                    "Build {} requirement: checking build image {} is available.".format(
+                        id, build_image
+                    )
+                )
+                import docker
+
+                client = docker.from_env()
+                image = client.images.pull(build_image)
+                logging.info(
+                    "Build {} requirement: build image {} is available with id: {}.".format(
+                        id, build_image, image.id
+                    )
+                )
+                already_checked_images.append(build_image)
+            else:
+                logging.info(
+                    "Build {} requirement: build image {} availability was already checked.".format(
+                        id, build_image
+                    )
+                )

redis_benchmarks_specification/__api__/api.py

@@ -37,3 +37,17 @@ clientconfig:
     requests:
       cpus: "2"
       memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: "$.StartTime"
+    metrics:
+      - "$.Tests.Overall.rps"
+      - "$.Tests.Overall.avg_latency_ms"
+      - "$.Tests.Overall.p50_latency_ms"
+      - "$.Tests.Overall.p95_latency_ms"
+      - "$.Tests.Overall.p99_latency_ms"
+      - "$.Tests.Overall.max_latency_ms"
+      - "$.Tests.Overall.min_latency_ms"

redis_benchmarks_specification/__builder__/builder.py

@@ -0,0 +1,23 @@
+[testenv:integration-tests]
+deps =
+    pytest
+    pytest-cov
+    codecov
+    black
+    flake8
+
+
+commands =
+ black --check redis_benchmarks_specification
+ flake8 redis_benchmarks_specification
+ coverage erase
+ coverage run --include=redis_benchmarks_specification/* -m pytest -ra
+ coverage report -m
+
+docker =
+    rts_datasink
+
+[docker:rts_datasink]
+image = redislabs/redistimeseries:1.4.7
+ports =
+    6379:6379/tcp

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -0,0 +1,36 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs
+#  All rights reserved.
+#
+from redis_benchmarks_specification.__api__.api import commit_schema_to_stream
+import redis
+
+from redis_benchmarks_specification.__common__.env import (
+    STREAM_KEYNAME_GH_EVENTS_COMMIT,
+)
+
+
+def test_commit_schema_to_stream():
+    result, reply_fields, error_msg = commit_schema_to_stream(
+        '{"git_hashss":"0cf2df84d4b27af4bffd2bf3543838f09e10f874"}', None
+    )
+    assert result == False
+    assert error_msg is not None
+    try:
+        conn = redis.StrictRedis()
+        conn.ping()
+        conn.flushall()
+        result, reply_fields, error_msg = commit_schema_to_stream(
+            '{"git_hash":"0cf2df84d4b27af4bffd2bf3543838f09e10f874"}', conn
+        )
+        assert result == True
+        assert error_msg == None
+        reply = conn.xread({STREAM_KEYNAME_GH_EVENTS_COMMIT: 0}, 1)
+        assert (
+            reply[0][1][0][1][b"git_hash"]
+            == b"0cf2df84d4b27af4bffd2bf3543838f09e10f874"
+        )
+
+    except redis.exceptions.ConnectionError:
+        pass

redis_benchmarks_specification/test-suites/redis-benchmark-full-suite-1Mkeys-100B.yml

@@ -0,0 +1,36 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs
+#  All rights reserved.
+#
+from redis_benchmarks_specification.__api__.api import commit_schema_to_stream
+import redis
+
+from redis_benchmarks_specification.__common__.env import (
+    STREAM_KEYNAME_GH_EVENTS_COMMIT,
+)
+
+
+def test_commit_schema_to_stream():
+    result, reply_fields, error_msg = commit_schema_to_stream(
+        '{"git_hashss":"0cf2df84d4b27af4bffd2bf3543838f09e10f874"}', None
+    )
+    assert result == False
+    assert error_msg is not None
+    try:
+        conn = redis.StrictRedis()
+        conn.ping()
+        conn.flushall()
+        result, reply_fields, error_msg = commit_schema_to_stream(
+            '{"git_hash":"0cf2df84d4b27af4bffd2bf3543838f09e10f874"}', conn
+        )
+        assert result == True
+        assert error_msg == None
+        reply = conn.xread({STREAM_KEYNAME_GH_EVENTS_COMMIT: 0}, 1)
+        assert (
+            reply[0][1][0][1][b"git_hash"]
+            == b"0cf2df84d4b27af4bffd2bf3543838f09e10f874"
+        )
+
+    except redis.exceptions.ConnectionError:
+        pass