Merge pull request #2907 from DataDog/cbeauchesne/python-2799

[python] Use healthcheck to get info #2799
DataDog · Aug 26, 2024 · 0a114f4 · 0a114f4
2 parents 4046f64 + a523e38
commit 0a114f4
Show file tree

Hide file tree

Showing 15 changed files with 93 additions and 33 deletions.
diff --git a/utils/_context/_scenarios/parametric.py b/utils/_context/_scenarios/parametric.py
@@ -146,7 +146,7 @@ def configure(self, config):
             self._clean_networks()
 
         # https://github.com/DataDog/system-tests/issues/2799
-        if library in ("nodejs",):
+        if library in ("nodejs", "python"):
             output = _get_client().containers.run(
                 self.apm_test_server_definition.container_tag,
                 remove=True,
@@ -338,7 +338,8 @@ def python_library_factory() -> APMLibraryTestServer:
 WORKDIR /app
 RUN pyenv global 3.9.16
 RUN python3.9 -m pip install fastapi==0.89.1 uvicorn==0.20.0
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/parametric/system_tests_library_version.sh system_tests_library_version.sh
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 ENV DD_PATCH_MODULES="fastapi:false"
 """,

diff --git a/utils/_context/containers.py b/utils/_context/containers.py
@@ -658,7 +658,7 @@ def configure(self, replay):
         )
 
         # https://github.com/DataDog/system-tests/issues/2799
-        if self.library in ("nodejs",):
+        if self.library in ("nodejs", "python"):
             self.healthcheck = {
                 "test": f"curl --fail --silent --show-error localhost:{self.port}/healthcheck",
                 "retries": 60,
@@ -685,18 +685,14 @@ def configure(self, replay):
         else:
             self.appsec_rules_file = (self.image.env | self.environment).get("DD_APPSEC_RULES", None)
 
-        if self.weblog_variant == "python3.12":
-            if self.library < "[email protected]":  # profiling causes a seg fault on 2.0.0
-                self.environment["DD_PROFILING_ENABLED"] = "false"
-
     def post_start(self):
         from utils import weblog
 
         logger.debug(f"Docker host is {weblog.domain}")
 
-        # new way of getting info from the weblog. Only working for nodejs right now
+        # new way of getting info from the weblog. Only working for nodejs and python right now
         # https://github.com/DataDog/system-tests/issues/2799
-        if self.library == "nodejs":
+        if self.library in ("nodejs", "python"):
             with open(self.healthcheck_log_file, mode="r", encoding="utf-8") as f:
                 data = json.load(f)
                 lib = data["library"]
@@ -749,6 +745,10 @@ def __init__(self, host_log_folder) -> None:
             image_name="postgres:alpine",
             name="postgres",
             host_log_folder=host_log_folder,
+            # healthcheck={
+            #     "test": "pg_isready -q -U postgres -d system_tests_dbname",
+            #     "retries": 30,
+            # },
             user="postgres",
             environment={"POSTGRES_PASSWORD": "password", "PGPORT": "5433"},
             volumes={

diff --git a/utils/build/docker/python/django-poc.Dockerfile b/utils/build/docker/python/django-poc.Dockerfile
@@ -3,7 +3,7 @@ FROM datadog/system-tests:django-poc.base-v2
 WORKDIR /app
 
 
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 
 COPY utils/build/docker/python/django /app

diff --git a/utils/build/docker/python/django/app/urls.py b/utils/build/docker/python/django/app/urls.py
@@ -25,6 +25,7 @@
     weak_hash_secure_algorithm,
 )
 
+import ddtrace
 from ddtrace import Pin, tracer, patch_all
 from ddtrace.appsec import trace_utils as appsec_trace_utils
 
@@ -63,6 +64,29 @@ def sample_rate(request, i):
 _TRACK_CUSTOM_APPSEC_EVENT_NAME = "system_tests_appsec_event"
 
 
+@csrf_exempt
+def healthcheck(request):
+    with open(ddtrace.appsec.__path__[0] + "/rules.json", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if "metadata" not in data:
+        appsec_event_rules_version = "1.2.5"
+    else:
+        appsec_event_rules_version = data["metadata"]["rules_version"]
+
+    result = {
+        "status": "ok",
+        "library": {
+            "language": "python",
+            "version": ddtrace.__version__,
+            "libddwaf_version": ddtrace.appsec._ddwaf.ddwaf_get_version().decode(),
+            "appsec_event_rules_version": appsec_event_rules_version,
+        },
+    }
+
+    return HttpResponse(json.dumps(result), content_type="application/json")
+
+
 @csrf_exempt
 def waf(request, *args, **kwargs):
     if "tag_value" in kwargs:
@@ -662,6 +686,7 @@ def create_extra_service(request):
 urlpatterns = [
     path("", hello_world),
     path("sample_rate_route/<int:i>", sample_rate),
+    path("healthcheck", healthcheck),
     path("waf", waf),
     path("waf/", waf),
     path("waf/<url>", waf),

diff --git a/utils/build/docker/python/fastapi.Dockerfile b/utils/build/docker/python/fastapi.Dockerfile
@@ -2,7 +2,7 @@ FROM datadog/system-tests:fastapi.base-v1
 
 WORKDIR /app
 
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 RUN python3.10 -m pip install --upgrade pip
 RUN python3.10 -m pip install PyYAML uvicorn requests psycopg2-binary python-multipart

diff --git a/utils/build/docker/python/fastapi/main.py b/utils/build/docker/python/fastapi/main.py
@@ -27,6 +27,7 @@
 import urllib3
 import xmltodict
 
+import ddtrace
 from ddtrace import Pin
 from ddtrace import patch_all
 from ddtrace import tracer
@@ -64,6 +65,27 @@ async def root():
     return "Hello, World!"
 
 
+@app.get("/healthcheck")
+async def healthcheck():
+    with open(ddtrace.appsec.__path__[0] + "/rules.json", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if "metadata" not in data:
+        appsec_event_rules_version = "1.2.5"
+    else:
+        appsec_event_rules_version = data["metadata"]["rules_version"]
+
+    return {
+        "status": "ok",
+        "library": {
+            "language": "python",
+            "version": ddtrace.__version__,
+            "libddwaf_version": ddtrace.appsec._ddwaf.ddwaf_get_version().decode(),
+            "appsec_event_rules_version": appsec_event_rules_version,
+        },
+    }
+
+
 @app.get("/sample_rate_route/{i}", response_class=PlainTextResponse)
 async def sample_rate(i):
     return "OK"

diff --git a/utils/build/docker/python/flask-poc.Dockerfile b/utils/build/docker/python/flask-poc.Dockerfile
@@ -2,7 +2,7 @@ FROM datadog/system-tests:flask-poc.base-v6
 
 WORKDIR /app
 
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 
 COPY utils/build/docker/python/flask /app

diff --git a/utils/build/docker/python/flask/app.py b/utils/build/docker/python/flask/app.py
@@ -167,6 +167,28 @@ def hello_world():
     return "Hello, World!\\n"
 
 
+@app.route("/healthcheck")
+def healthcheck():
+    path = ddtrace.appsec.__path__[0] + "/rules.json"
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+
+    if "metadata" not in data:
+        appsec_event_rules_version = "1.2.5"
+    else:
+        appsec_event_rules_version = data["metadata"]["rules_version"]
+
+    return {
+        "status": "ok",
+        "library": {
+            "language": "python",
+            "version": ddtrace.__version__,
+            "libddwaf_version": ddtrace.appsec._ddwaf.ddwaf_get_version().decode(),
+            "appsec_event_rules_version": appsec_event_rules_version,
+        },
+    }
+
+
 @app.route("/sample_rate_route/<i>")
 def sample_rate(i):
     return "OK"

diff --git a/utils/build/docker/python/get_appsec_rules_version.py b/utils/build/docker/python/get_appsec_rules_version.py
diff --git a/utils/build/docker/python/install_ddtrace.sh b/utils/build/docker/python/install_ddtrace.sh
@@ -24,9 +24,7 @@ fi
 
 cd -
 
-python -c "import ddtrace; print(ddtrace.__version__)" > SYSTEM_TESTS_LIBRARY_VERSION
-python /binaries/get_appsec_rules_version.py > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION
-touch SYSTEM_TESTS_LIBDDWAF_VERSION
-
-echo "dd-trace version is $(cat SYSTEM_TESTS_LIBRARY_VERSION)"
-echo "appsec rules version is $(cat SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION)"
+# python uses the next API to get the library version. See https://github.com/DataDog/system-tests/issues/2799
+echo "0.0.0" > SYSTEM_TESTS_LIBRARY_VERSION
+echo "0.0.0" > SYSTEM_TESTS_LIBDDWAF_VERSION
+echo "0.0.0" > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION
diff --git a/utils/build/docker/python/parametric/system_tests_library_version.sh b/utils/build/docker/python/parametric/system_tests_library_version.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python -c "import ddtrace; print(ddtrace.__version__)"
diff --git a/utils/build/docker/python/pylons.Dockerfile b/utils/build/docker/python/pylons.Dockerfile
@@ -7,7 +7,7 @@ RUN python --version && curl --version
 RUN pip install pylons
 COPY utils/build/docker/python/pylons/app /app
 
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 
 ENV DD_TRACE_HEADER_TAGS='user-agent:http.request.headers.user-agent'

diff --git a/utils/build/docker/python/python3.12.Dockerfile b/utils/build/docker/python/python3.12.Dockerfile
@@ -2,7 +2,7 @@ FROM datadog/system-tests:python3.12.base-v3
 
 WORKDIR /app
 
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 
 COPY utils/build/docker/python/django /app

diff --git a/utils/build/docker/python/uds-flask.Dockerfile b/utils/build/docker/python/uds-flask.Dockerfile
@@ -2,7 +2,7 @@ FROM datadog/system-tests:flask-poc.base-v6
 
 WORKDIR /app
 
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 
 COPY utils/build/docker/python/flask /app

diff --git a/utils/build/docker/python/uwsgi-poc.Dockerfile b/utils/build/docker/python/uwsgi-poc.Dockerfile
@@ -2,7 +2,7 @@ FROM datadog/system-tests:uwsgi-poc.base-v3
 
 WORKDIR /app
 
-COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/
+COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/
 RUN /binaries/install_ddtrace.sh
 
 COPY utils/build/docker/python/flask /app