diff --git a/utils/_context/_scenarios/parametric.py b/utils/_context/_scenarios/parametric.py index c46466457f..e8d409c86e 100644 --- a/utils/_context/_scenarios/parametric.py +++ b/utils/_context/_scenarios/parametric.py @@ -146,7 +146,7 @@ def configure(self, config): self._clean_networks() # https://github.com/DataDog/system-tests/issues/2799 - if library in ("nodejs",): + if library in ("nodejs", "python"): output = _get_client().containers.run( self.apm_test_server_definition.container_tag, remove=True, @@ -338,7 +338,8 @@ def python_library_factory() -> APMLibraryTestServer: WORKDIR /app RUN pyenv global 3.9.16 RUN python3.9 -m pip install fastapi==0.89.1 uvicorn==0.20.0 -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/parametric/system_tests_library_version.sh system_tests_library_version.sh +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh ENV DD_PATCH_MODULES="fastapi:false" """, diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 63240d5faa..5702cca761 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -658,7 +658,7 @@ def configure(self, replay): ) # https://github.com/DataDog/system-tests/issues/2799 - if self.library in ("nodejs",): + if self.library in ("nodejs", "python"): self.healthcheck = { "test": f"curl --fail --silent --show-error localhost:{self.port}/healthcheck", "retries": 60, @@ -685,18 +685,14 @@ def configure(self, replay): else: self.appsec_rules_file = (self.image.env | self.environment).get("DD_APPSEC_RULES", None) - if self.weblog_variant == "python3.12": - if self.library < "python@2.1.0.dev": # profiling causes a seg fault on 2.0.0 - self.environment["DD_PROFILING_ENABLED"] = "false" - def post_start(self): from utils import weblog logger.debug(f"Docker host is {weblog.domain}") - # new way of getting info from the weblog. Only working for nodejs right now + # new way of getting info from the weblog. Only working for nodejs and python right now # https://github.com/DataDog/system-tests/issues/2799 - if self.library == "nodejs": + if self.library in ("nodejs", "python"): with open(self.healthcheck_log_file, mode="r", encoding="utf-8") as f: data = json.load(f) lib = data["library"] @@ -749,6 +745,10 @@ def __init__(self, host_log_folder) -> None: image_name="postgres:alpine", name="postgres", host_log_folder=host_log_folder, + # healthcheck={ + # "test": "pg_isready -q -U postgres -d system_tests_dbname", + # "retries": 30, + # }, user="postgres", environment={"POSTGRES_PASSWORD": "password", "PGPORT": "5433"}, volumes={ diff --git a/utils/build/docker/python/django-poc.Dockerfile b/utils/build/docker/python/django-poc.Dockerfile index 501685f385..568e545c91 100644 --- a/utils/build/docker/python/django-poc.Dockerfile +++ b/utils/build/docker/python/django-poc.Dockerfile @@ -3,7 +3,7 @@ FROM datadog/system-tests:django-poc.base-v2 WORKDIR /app -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh COPY utils/build/docker/python/django /app diff --git a/utils/build/docker/python/django/app/urls.py b/utils/build/docker/python/django/app/urls.py index 06495f80ac..0adc1c9869 100644 --- a/utils/build/docker/python/django/app/urls.py +++ b/utils/build/docker/python/django/app/urls.py @@ -25,6 +25,7 @@ weak_hash_secure_algorithm, ) +import ddtrace from ddtrace import Pin, tracer, patch_all from ddtrace.appsec import trace_utils as appsec_trace_utils @@ -63,6 +64,29 @@ def sample_rate(request, i): _TRACK_CUSTOM_APPSEC_EVENT_NAME = "system_tests_appsec_event" +@csrf_exempt +def healthcheck(request): + with open(ddtrace.appsec.__path__[0] + "/rules.json", encoding="utf-8") as f: + data = json.load(f) + + if "metadata" not in data: + appsec_event_rules_version = "1.2.5" + else: + appsec_event_rules_version = data["metadata"]["rules_version"] + + result = { + "status": "ok", + "library": { + "language": "python", + "version": ddtrace.__version__, + "libddwaf_version": ddtrace.appsec._ddwaf.ddwaf_get_version().decode(), + "appsec_event_rules_version": appsec_event_rules_version, + }, + } + + return HttpResponse(json.dumps(result), content_type="application/json") + + @csrf_exempt def waf(request, *args, **kwargs): if "tag_value" in kwargs: @@ -662,6 +686,7 @@ def create_extra_service(request): urlpatterns = [ path("", hello_world), path("sample_rate_route/", sample_rate), + path("healthcheck", healthcheck), path("waf", waf), path("waf/", waf), path("waf/", waf), diff --git a/utils/build/docker/python/fastapi.Dockerfile b/utils/build/docker/python/fastapi.Dockerfile index 74ecb328f8..0cfe8c6213 100644 --- a/utils/build/docker/python/fastapi.Dockerfile +++ b/utils/build/docker/python/fastapi.Dockerfile @@ -2,7 +2,7 @@ FROM datadog/system-tests:fastapi.base-v1 WORKDIR /app -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh RUN python3.10 -m pip install --upgrade pip RUN python3.10 -m pip install PyYAML uvicorn requests psycopg2-binary python-multipart diff --git a/utils/build/docker/python/fastapi/main.py b/utils/build/docker/python/fastapi/main.py index f7305c3609..74526f27b9 100644 --- a/utils/build/docker/python/fastapi/main.py +++ b/utils/build/docker/python/fastapi/main.py @@ -27,6 +27,7 @@ import urllib3 import xmltodict +import ddtrace from ddtrace import Pin from ddtrace import patch_all from ddtrace import tracer @@ -64,6 +65,27 @@ async def root(): return "Hello, World!" +@app.get("/healthcheck") +async def healthcheck(): + with open(ddtrace.appsec.__path__[0] + "/rules.json", encoding="utf-8") as f: + data = json.load(f) + + if "metadata" not in data: + appsec_event_rules_version = "1.2.5" + else: + appsec_event_rules_version = data["metadata"]["rules_version"] + + return { + "status": "ok", + "library": { + "language": "python", + "version": ddtrace.__version__, + "libddwaf_version": ddtrace.appsec._ddwaf.ddwaf_get_version().decode(), + "appsec_event_rules_version": appsec_event_rules_version, + }, + } + + @app.get("/sample_rate_route/{i}", response_class=PlainTextResponse) async def sample_rate(i): return "OK" diff --git a/utils/build/docker/python/flask-poc.Dockerfile b/utils/build/docker/python/flask-poc.Dockerfile index a3dbab2616..add55518e0 100644 --- a/utils/build/docker/python/flask-poc.Dockerfile +++ b/utils/build/docker/python/flask-poc.Dockerfile @@ -2,7 +2,7 @@ FROM datadog/system-tests:flask-poc.base-v6 WORKDIR /app -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh COPY utils/build/docker/python/flask /app diff --git a/utils/build/docker/python/flask/app.py b/utils/build/docker/python/flask/app.py index 588e049660..1eb3aee650 100644 --- a/utils/build/docker/python/flask/app.py +++ b/utils/build/docker/python/flask/app.py @@ -167,6 +167,28 @@ def hello_world(): return "Hello, World!\\n" +@app.route("/healthcheck") +def healthcheck(): + path = ddtrace.appsec.__path__[0] + "/rules.json" + with open(path, encoding="utf-8") as f: + data = json.load(f) + + if "metadata" not in data: + appsec_event_rules_version = "1.2.5" + else: + appsec_event_rules_version = data["metadata"]["rules_version"] + + return { + "status": "ok", + "library": { + "language": "python", + "version": ddtrace.__version__, + "libddwaf_version": ddtrace.appsec._ddwaf.ddwaf_get_version().decode(), + "appsec_event_rules_version": appsec_event_rules_version, + }, + } + + @app.route("/sample_rate_route/") def sample_rate(i): return "OK" diff --git a/utils/build/docker/python/get_appsec_rules_version.py b/utils/build/docker/python/get_appsec_rules_version.py deleted file mode 100644 index db29f027d8..0000000000 --- a/utils/build/docker/python/get_appsec_rules_version.py +++ /dev/null @@ -1,11 +0,0 @@ -from ddtrace import appsec -import json - - -path = appsec.__path__[0] + "/rules.json" -data = json.load(open(path)) - -if "metadata" not in data: - print("1.2.5") -else: - print(data["metadata"]["rules_version"]) diff --git a/utils/build/docker/python/install_ddtrace.sh b/utils/build/docker/python/install_ddtrace.sh index c27d78adc3..b91ad20bf6 100755 --- a/utils/build/docker/python/install_ddtrace.sh +++ b/utils/build/docker/python/install_ddtrace.sh @@ -24,9 +24,7 @@ fi cd - -python -c "import ddtrace; print(ddtrace.__version__)" > SYSTEM_TESTS_LIBRARY_VERSION -python /binaries/get_appsec_rules_version.py > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION -touch SYSTEM_TESTS_LIBDDWAF_VERSION - -echo "dd-trace version is $(cat SYSTEM_TESTS_LIBRARY_VERSION)" -echo "appsec rules version is $(cat SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION)" +# python uses the next API to get the library version. See https://github.com/DataDog/system-tests/issues/2799 +echo "0.0.0" > SYSTEM_TESTS_LIBRARY_VERSION +echo "0.0.0" > SYSTEM_TESTS_LIBDDWAF_VERSION +echo "0.0.0" > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION \ No newline at end of file diff --git a/utils/build/docker/python/parametric/system_tests_library_version.sh b/utils/build/docker/python/parametric/system_tests_library_version.sh new file mode 100755 index 0000000000..98e87fc189 --- /dev/null +++ b/utils/build/docker/python/parametric/system_tests_library_version.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python -c "import ddtrace; print(ddtrace.__version__)" diff --git a/utils/build/docker/python/pylons.Dockerfile b/utils/build/docker/python/pylons.Dockerfile index 0a6ef45ed9..ba0eaf3930 100644 --- a/utils/build/docker/python/pylons.Dockerfile +++ b/utils/build/docker/python/pylons.Dockerfile @@ -7,7 +7,7 @@ RUN python --version && curl --version RUN pip install pylons COPY utils/build/docker/python/pylons/app /app -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh ENV DD_TRACE_HEADER_TAGS='user-agent:http.request.headers.user-agent' diff --git a/utils/build/docker/python/python3.12.Dockerfile b/utils/build/docker/python/python3.12.Dockerfile index c3f68737ed..5f11043bbb 100644 --- a/utils/build/docker/python/python3.12.Dockerfile +++ b/utils/build/docker/python/python3.12.Dockerfile @@ -2,7 +2,7 @@ FROM datadog/system-tests:python3.12.base-v3 WORKDIR /app -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh COPY utils/build/docker/python/django /app diff --git a/utils/build/docker/python/uds-flask.Dockerfile b/utils/build/docker/python/uds-flask.Dockerfile index 907f036359..4a8f1f6347 100644 --- a/utils/build/docker/python/uds-flask.Dockerfile +++ b/utils/build/docker/python/uds-flask.Dockerfile @@ -2,7 +2,7 @@ FROM datadog/system-tests:flask-poc.base-v6 WORKDIR /app -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh COPY utils/build/docker/python/flask /app diff --git a/utils/build/docker/python/uwsgi-poc.Dockerfile b/utils/build/docker/python/uwsgi-poc.Dockerfile index a3c65bd464..cc64b0ed75 100644 --- a/utils/build/docker/python/uwsgi-poc.Dockerfile +++ b/utils/build/docker/python/uwsgi-poc.Dockerfile @@ -2,7 +2,7 @@ FROM datadog/system-tests:uwsgi-poc.base-v3 WORKDIR /app -COPY utils/build/docker/python/install_ddtrace.sh utils/build/docker/python/get_appsec_rules_version.py binaries* /binaries/ +COPY utils/build/docker/python/install_ddtrace.sh binaries* /binaries/ RUN /binaries/install_ddtrace.sh COPY utils/build/docker/python/flask /app