diff --git a/airflow/tests/test_unit.py b/airflow/tests/test_unit.py index cbdef5dc22efd..d8d70878c741e 100644 --- a/airflow/tests/test_unit.py +++ b/airflow/tests/test_unit.py @@ -26,18 +26,16 @@ def test_service_checks_cannot_connect(aggregator): 'json_resp, expected_healthy_status, expected_healthy_value', [({'status': 'OK'}, AgentCheck.OK, 1), ({'status': 'KO'}, AgentCheck.CRITICAL, 0), ({}, AgentCheck.CRITICAL, 0)], ) -def test_service_checks_healthy_exp(aggregator, json_resp, expected_healthy_status, expected_healthy_value): +def test_service_checks_healthy_exp(aggregator, mock_http, json_resp, expected_healthy_status, expected_healthy_value): instance = common.FULL_CONFIG['instances'][0] check = AirflowCheck('airflow', common.FULL_CONFIG, [instance]) - with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value=None): - mock_session = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=mock_session): - mock_resp = mock.MagicMock(status_code=200) - mock_resp.json.side_effect = [json_resp] - mock_session.get.return_value = mock_resp + mock_resp = mock.MagicMock(status_code=200) + mock_resp.json.side_effect = [json_resp] + mock_http.get.return_value = mock_resp - check.check(None) + with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value=None): + check.check(None) tags = ['key:my-tag', 'url:http://localhost:8080'] @@ -54,22 +52,20 @@ def test_service_checks_healthy_exp(aggregator, json_resp, expected_healthy_stat ], ) def test_service_checks_healthy_stable( - aggregator, metadb_status, scheduler_status, expected_healthy_status, expected_healthy_value + aggregator, mock_http, metadb_status, scheduler_status, expected_healthy_status, expected_healthy_value ): # Stable is only defined in the context of Airflow 2 instance = common.FULL_CONFIG['instances'][0] check = AirflowCheck('airflow', common.FULL_CONFIG, [instance]) - with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'): - mock_session = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=mock_session): - mock_resp = mock.MagicMock(status_code=200) - mock_resp.json.side_effect = [ - {'metadatabase': {'status': metadb_status}, 'scheduler': {'status': scheduler_status}}, - {'status': 'OK'}, - ] - mock_session.get.return_value = mock_resp + mock_resp = mock.MagicMock(status_code=200) + mock_resp.json.side_effect = [ + {'metadatabase': {'status': metadb_status}, 'scheduler': {'status': scheduler_status}}, + {'status': 'OK'}, + ] + mock_http.get.return_value = mock_resp - check.check(None) + with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'): + check.check(None) tags = ['key:my-tag', 'url:http://localhost:8080'] @@ -77,42 +73,39 @@ def test_service_checks_healthy_stable( aggregator.assert_metric('airflow.healthy', expected_healthy_value, tags=tags, count=1) -def test_dag_total_tasks(aggregator, task_instance): +def test_dag_total_tasks(aggregator, mock_http, task_instance): instance = common.FULL_CONFIG['instances'][0] check = AirflowCheck('airflow', common.FULL_CONFIG, [instance]) - with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'): - req = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req): - mock_resp = mock.MagicMock(status_code=200) - mock_resp.json.side_effect = [ - {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}}, - task_instance, - ] - req.get.return_value = mock_resp + mock_resp = mock.MagicMock(status_code=200) + mock_resp.json.side_effect = [ + {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}}, + task_instance, + ] + mock_http.get.return_value = mock_resp - check.check(None) + with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'): + check.check(None) aggregator.assert_metric('airflow.dag.task.total_running', value=1, count=1) -def test_dag_task_ongoing_duration(aggregator, task_instance): +def test_dag_task_ongoing_duration(aggregator, mock_http, task_instance): instance = common.FULL_CONFIG['instances'][0] check = AirflowCheck('airflow', common.FULL_CONFIG, [instance]) + mock_resp = mock.MagicMock(status_code=200) + mock_resp.json.side_effect = [ + {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}}, + ] + mock_http.get.return_value = mock_resp + with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'): - req = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req): - mock_resp = mock.MagicMock(status_code=200) - mock_resp.json.side_effect = [ - {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}}, - ] - req.get.return_value = mock_resp - with mock.patch( - 'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances', - return_value=task_instance.get('task_instances'), - ): - check.check(None) + with mock.patch( + 'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances', + return_value=task_instance.get('task_instances'), + ): + check.check(None) aggregator.assert_metric( 'airflow.dag.task.ongoing_duration', @@ -141,23 +134,21 @@ def test_dag_task_ongoing_duration(aggregator, task_instance): ), ], ) -def test_config_collect_ongoing_duration(collect_ongoing_duration, should_call_method): +def test_config_collect_ongoing_duration(mock_http, collect_ongoing_duration, should_call_method): instance = {**common.FULL_CONFIG['instances'][0], 'collect_ongoing_duration': collect_ongoing_duration} check = AirflowCheck('airflow', common.FULL_CONFIG, [instance]) + mock_resp = mock.MagicMock(status_code=200) + mock_resp.json.side_effect = [ + {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}}, + ] + mock_http.get.return_value = mock_resp + with mock.patch('datadog_checks.airflow.airflow.AirflowCheck._get_version', return_value='2.6.2'): - req = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req): - mock_resp = mock.MagicMock(status_code=200) - mock_resp.json.side_effect = [ - {'metadatabase': {'status': 'healthy'}, 'scheduler': {'status': 'healthy'}}, - ] - req.get.return_value = mock_resp - - with mock.patch( - 'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances' - ) as mock_get_all_task_instances: - check.check(None) - - # Assert method calls - mock_get_all_task_instances.assert_has_calls(should_call_method, any_order=False) + with mock.patch( + 'datadog_checks.airflow.airflow.AirflowCheck._get_all_task_instances' + ) as mock_get_all_task_instances: + check.check(None) + + # Assert method calls + mock_get_all_task_instances.assert_has_calls(should_call_method, any_order=False) diff --git a/appgate_sdp/tests/test_unit.py b/appgate_sdp/tests/test_unit.py index d7ba90e710b2b..ded2350b51ac2 100644 --- a/appgate_sdp/tests/test_unit.py +++ b/appgate_sdp/tests/test_unit.py @@ -31,7 +31,7 @@ def test_check_appgate_sdp(dd_run_check, aggregator, instance, mock_http_respons def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggregator, instance, mock_http_response): mock_http_response(status_code=404) check = AppgateSDPCheck('appgate_sdp', {}, [instance]) - with pytest.raises(Exception, match='requests.exceptions.HTTPError'): + with pytest.raises(Exception, match='HTTPStatusError'): dd_run_check(check) aggregator.assert_service_check('appgate_sdp.openmetrics.health', AppgateSDPCheck.CRITICAL) diff --git a/argo_workflows/tests/test_unit.py b/argo_workflows/tests/test_unit.py index 28b73a6ab547a..58e90ed1632fb 100644 --- a/argo_workflows/tests/test_unit.py +++ b/argo_workflows/tests/test_unit.py @@ -135,6 +135,6 @@ def test_check_with_fixtures(dd_run_check, aggregator, instance, mock_http_respo def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggregator, instance, mock_http_response): mock_http_response(status_code=404) check = ArgoWorkflowsCheck('argo_workflows', {}, [instance]) - with pytest.raises(Exception, match='requests.exceptions.HTTPError'): + with pytest.raises(Exception, match='HTTPStatusError'): dd_run_check(check) aggregator.assert_service_check('argo_workflows.openmetrics.health', ArgoWorkflowsCheck.CRITICAL) diff --git a/bentoml/tests/test_unit.py b/bentoml/tests/test_unit.py index 3ec90e358d026..ebede0719a7fe 100644 --- a/bentoml/tests/test_unit.py +++ b/bentoml/tests/test_unit.py @@ -1,12 +1,12 @@ # (C) Datadog, Inc. 2025-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from unittest.mock import Mock, patch +from unittest.mock import Mock import pytest -import requests from datadog_checks.base.constants import ServiceCheck +from datadog_checks.base.utils.http_exceptions import HTTPStatusError from datadog_checks.bentoml import BentomlCheck from datadog_checks.dev.utils import get_metadata_metrics @@ -21,26 +21,19 @@ def test_bentoml_mock_metrics(dd_run_check, aggregator, mock_http_response): mock_http_response(file_path=get_fixture_path('metrics.txt')) - with patch('datadog_checks.bentoml.check.BentomlCheck.http') as mock_http: - mock_response = type('MockResponse', (), {'status_code': 200})() - mock_http.get.return_value = mock_response - mock_http.get.return_value.raise_for_status = lambda: None - - check = BentomlCheck('bentoml', {}, [OM_MOCKED_INSTANCE]) - dd_run_check(check) + check = BentomlCheck('bentoml', {}, [OM_MOCKED_INSTANCE]) + dd_run_check(check) - for metric in METRICS: - aggregator.assert_metric(metric) + for metric in METRICS: + aggregator.assert_metric(metric) - for metric in ENDPOINT_METRICS: - aggregator.assert_metric(metric, value=1, tags=['test:tag', 'status_code:200']) + for metric in ENDPOINT_METRICS: + aggregator.assert_metric(metric, value=1, tags=['test:tag', 'status_code:200']) - aggregator.assert_all_metrics_covered() - assert mock_http.get.call_count == 2 - aggregator.assert_metrics_using_metadata(get_metadata_metrics()) - aggregator.assert_all_metrics_covered() - aggregator.assert_metric_has_tag('bentoml.service.request.count', 'bentoml_endpoint:/summarize') - aggregator.assert_service_check('bentoml.openmetrics.health', ServiceCheck.OK) + aggregator.assert_all_metrics_covered() + aggregator.assert_metrics_using_metadata(get_metadata_metrics()) + aggregator.assert_metric_has_tag('bentoml.service.request.count', 'bentoml_endpoint:/summarize') + aggregator.assert_service_check('bentoml.openmetrics.health', ServiceCheck.OK) def test_bentoml_mock_invalid_endpoint(dd_run_check, aggregator, mock_http_response): @@ -53,21 +46,25 @@ def test_bentoml_mock_invalid_endpoint(dd_run_check, aggregator, mock_http_respo def test_bentoml_mock_valid_endpoint_invalid_health(dd_run_check, aggregator, mock_http_response): - mock_http_response(file_path=get_fixture_path('metrics.txt')) + session_get_mock = mock_http_response(file_path=get_fixture_path('metrics.txt')) + metrics_response = session_get_mock.return_value _err = Mock() _err.status_code = 500 - _http_err = requests.HTTPError("500 Internal Server Error") - _http_err.response = _err + _http_err = HTTPStatusError("500 Internal Server Error", response=_err) _err.raise_for_status.side_effect = _http_err - with patch('datadog_checks.bentoml.check.BentomlCheck.http') as mock_http: - mock_http.get.return_value = _err + def dispatch(url, **_): + if '/livez' in url or '/readyz' in url: + return _err + return metrics_response - check = BentomlCheck('bentoml', {}, [OM_MOCKED_INSTANCE]) - dd_run_check(check) + session_get_mock.side_effect = dispatch + + check = BentomlCheck('bentoml', {}, [OM_MOCKED_INSTANCE]) + dd_run_check(check) - for metric in ENDPOINT_METRICS: - aggregator.assert_metric(metric, value=0, tags=['test:tag', 'status_code:500']) + for metric in ENDPOINT_METRICS: + aggregator.assert_metric(metric, value=0, tags=['test:tag', 'status_code:500']) - aggregator.assert_service_check('bentoml.openmetrics.health', ServiceCheck.OK) + aggregator.assert_service_check('bentoml.openmetrics.health', ServiceCheck.OK) diff --git a/celery/tests/test_unit.py b/celery/tests/test_unit.py index b1d50c192b122..c5173a14f0894 100644 --- a/celery/tests/test_unit.py +++ b/celery/tests/test_unit.py @@ -44,7 +44,7 @@ def test_emits_critical_openemtrics_service_check_when_service_is_down( """ mock_http_response(status_code=404) check = CeleryCheck("celery", {}, [instance]) - with pytest.raises(Exception, match="requests.exceptions.HTTPError"): + with pytest.raises(Exception, match="HTTPStatusError"): dd_run_check(check) aggregator.assert_all_metrics_covered() diff --git a/consul/tests/test_unit.py b/consul/tests/test_unit.py index 2b844e3601da8..0d3ecb460631e 100644 --- a/consul/tests/test_unit.py +++ b/consul/tests/test_unit.py @@ -3,7 +3,6 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import logging -import mock import pytest from datadog_checks.consul import ConsulCheck @@ -155,25 +154,25 @@ def test_get_nodes_with_service_critical(aggregator): aggregator.assert_metric('consul.catalog.services_count', value=1, tags=expected_tags) -def test_consul_request(aggregator, instance, mocker): +def test_consul_request(aggregator, instance, mocker, mock_http): consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG]) mocker.patch("datadog_checks.base.utils.serialization.json.loads") - with mock.patch("datadog_checks.consul.consul.requests.Session.get") as mock_requests_get: + + consul_check.consul_request("foo") + url = "{}/{}".format(instance["url"], "foo") + aggregator.assert_service_check("consul.can_connect", ConsulCheck.OK, tags=["url:{}".format(url)], count=1) + + aggregator.reset() + mock_http.get.side_effect = Exception("message") + with pytest.raises(Exception): consul_check.consul_request("foo") - url = "{}/{}".format(instance["url"], "foo") - aggregator.assert_service_check("consul.can_connect", ConsulCheck.OK, tags=["url:{}".format(url)], count=1) - - aggregator.reset() - mock_requests_get.side_effect = Exception("message") - with pytest.raises(Exception): - consul_check.consul_request("foo") - aggregator.assert_service_check( - "consul.can_connect", - ConsulCheck.CRITICAL, - tags=["url:{}".format(url)], - count=1, - message="Consul request to {} failed: message".format(url), - ) + aggregator.assert_service_check( + "consul.can_connect", + ConsulCheck.CRITICAL, + tags=["url:{}".format(url)], + count=1, + message="Consul request to {} failed: message".format(url), + ) def test_service_checks(aggregator): @@ -648,26 +647,13 @@ def test_network_latency_node_name( ), ], ) -def test_config(test_case, extra_config, expected_http_kwargs, mocker): +def test_config(test_case, extra_config, expected_http_kwargs): instance = extra_config check = ConsulCheck(common.CHECK_NAME, {}, instances=[instance]) - mocker.patch("datadog_checks.base.utils.serialization.json.loads") - with mock.patch('datadog_checks.base.utils.http.requests.Session') as session: - mock_session = mock.MagicMock() - session.return_value = mock_session - mock_session.get.return_value = mock.MagicMock(status_code=200) - - check.check(None) - - http_wargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_wargs.update(expected_http_kwargs) - mock_session.get.assert_called_with('/v1/status/leader', **http_wargs) + for key, value in expected_http_kwargs.items(): + if key == 'headers': + for h_key, h_value in value.items(): + assert check.http.get_header(h_key) == h_value + else: + assert check.http.options[key] == value diff --git a/couch/tests/test_unit.py b/couch/tests/test_unit.py index 7f2ad8678ae2d..d1bf4b49af3e9 100644 --- a/couch/tests/test_unit.py +++ b/couch/tests/test_unit.py @@ -4,7 +4,6 @@ from copy import deepcopy from unittest.mock import MagicMock -import mock import pytest from datadog_checks.couch import CouchDb @@ -30,24 +29,8 @@ def test_config(test_case, extra_config, expected_http_kwargs): instance.update(extra_config) check = CouchDb(common.CHECK_NAME, {}, instances=[instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200, content='{}') - - check.check(instance) - - http_wargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_wargs.update(expected_http_kwargs) - - r.get.assert_called_with('http://{}:5984/_all_dbs/'.format(common.HOST), **http_wargs) + for key, value in expected_http_kwargs.items(): + assert check.http.options[key] == value def test_new_version_system_metrics(load_test_data): diff --git a/couchbase/tests/conftest.py b/couchbase/tests/conftest.py index 5d5136982e14c..dfda5ee73fbb5 100644 --- a/couchbase/tests/conftest.py +++ b/couchbase/tests/conftest.py @@ -10,10 +10,10 @@ import pytest import requests +from datadog_checks.base.utils.http_testing import MockHTTPResponse # noqa: F401 from datadog_checks.couchbase import Couchbase from datadog_checks.dev import WaitFor, docker_run from datadog_checks.dev.docker import get_container_ip -from datadog_checks.dev.http import MockResponse from .common import ( BUCKET_NAME, @@ -301,4 +301,4 @@ def mock_http_responses(url, **_params): pytest.fail("url `{url}` not registered".format(url=url)) with open(os.path.join(HERE, 'fixtures', metrics_file)) as f: - return MockResponse(content=f.read()) + return MockHTTPResponse(content=f.read()) diff --git a/couchbase/tests/test_unit.py b/couchbase/tests/test_unit.py index 25a437c02f842..1836ea77f1e6e 100644 --- a/couchbase/tests/test_unit.py +++ b/couchbase/tests/test_unit.py @@ -124,8 +124,8 @@ def test_extract_index_tags(instance, test_input, expected_tags): assert eval(str(test_output)) == expected_tags -def test_unit(dd_run_check, check, instance, mocker, aggregator): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_unit(dd_run_check, check, instance, mock_http, aggregator): + mock_http.get.side_effect = mock_http_responses dd_run_check(check(instance)) @@ -140,8 +140,8 @@ def test_unit(dd_run_check, check, instance, mocker, aggregator): aggregator.assert_metrics_using_metadata(get_metadata_metrics()) -def test_unit_query_metrics(dd_run_check, check, instance_query, mocker, aggregator): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_unit_query_metrics(dd_run_check, check, instance_query, mock_http, aggregator): + mock_http.get.side_effect = mock_http_responses dd_run_check(check(instance_query)) diff --git a/datadog_checks_base/changelog.d/22676.added b/datadog_checks_base/changelog.d/22676.added new file mode 100644 index 0000000000000..ff63715e3c69d --- /dev/null +++ b/datadog_checks_base/changelog.d/22676.added @@ -0,0 +1 @@ +Add library-agnostic HTTP mocks/proto/exceptions and migrate intg tests. diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 5783ede4ec247..be4c548dd1005 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -68,7 +68,7 @@ import unicodedata as _module_unicodedata from datadog_checks.base.utils.diagnose import Diagnosis - from datadog_checks.base.utils.http import RequestsWrapper + from datadog_checks.base.utils.http_protocol import HTTPClientProtocol from datadog_checks.base.utils.metadata import MetadataManager inspect: _module_inspect = lazy_loader.load('inspect') @@ -404,7 +404,7 @@ def _get_metric_limit(self, instance=None): return limit @property - def http(self) -> RequestsWrapper: + def http(self) -> HTTPClientProtocol: """ Provides logic to yield consistent network behavior based on user configuration. diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py index 1a476138157de..6552a1723c4d1 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/base.py @@ -8,6 +8,7 @@ from datadog_checks.base.checks import AgentCheck from datadog_checks.base.errors import ConfigurationError +from datadog_checks.base.utils.http_exceptions import HTTPRequestError, HTTPStatusError from datadog_checks.base.utils.tracing import traced_class from .scraper import OpenMetricsScraper @@ -71,7 +72,8 @@ def check(self, _): with self.adopt_namespace(scraper.namespace): try: scraper.scrape() - except (ConnectionError, RequestException) as e: + # Pairs requests-native + library-agnostic exceptions; simplify to HTTPError after migration. + except (ConnectionError, RequestException, HTTPRequestError, HTTPStatusError) as e: self.log.error("There was an error scraping endpoint %s: %s", endpoint, str(e)) raise type(e)("There was an error scraping endpoint {}: {}".format(endpoint, e)) from None diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py index 65be700d603b6..fbd4b852a2aa8 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper/base_scraper.py @@ -23,7 +23,6 @@ from datadog_checks.base.constants import ServiceCheck from datadog_checks.base.errors import ConfigurationError from datadog_checks.base.utils.functions import no_op, return_true -from datadog_checks.base.utils.http import RequestsWrapper class OpenMetricsScraper: @@ -215,18 +214,10 @@ def __init__(self, check, config): self.raw_line_filter = re.compile('|'.join(raw_line_filters)) - self.http = RequestsWrapper(config, self.check.init_config, self.check.HTTP_CONFIG_REMAPPER, self.check.log) + self.http = self.check.http self._content_type = '' self._use_latest_spec = is_affirmative(config.get('use_latest_spec', False)) - if self._use_latest_spec: - accept_header = 'application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1' - else: - accept_header = 'text/plain' - - # Request the appropriate exposition format - if self.http.options['headers'].get('Accept') == '*/*': - self.http.options['headers']['Accept'] = accept_header self.use_process_start_time = is_affirmative(config.get('use_process_start_time')) @@ -463,6 +454,13 @@ def send_request(self, **kwargs): """ kwargs['stream'] = True + extra_headers = kwargs.get('extra_headers', {}) + if self._use_latest_spec: + accept_header = 'application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1' + else: + accept_header = 'text/plain' + extra_headers['Accept'] = accept_header + kwargs['extra_headers'] = extra_headers return self.http.get(self.endpoint, **kwargs) def set_dynamic_tags(self, *tags): diff --git a/datadog_checks_base/datadog_checks/base/utils/http.py b/datadog_checks_base/datadog_checks/base/utils/http.py index ad52272e914a7..5b55effd5828d 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http.py +++ b/datadog_checks_base/datadog_checks/base/utils/http.py @@ -11,6 +11,7 @@ from collections import ChainMap from contextlib import ExitStack, contextmanager from copy import deepcopy +from typing import TYPE_CHECKING from urllib.parse import quote, urlparse, urlunparse import lazy_loader @@ -28,9 +29,22 @@ from .common import ensure_bytes, ensure_unicode from .headers import get_default_headers, update_headers + +# Re-export HTTP exceptions for single import location +from .http_exceptions import ( # noqa: F401 + HTTPConnectionError, + HTTPError, + HTTPRequestError, + HTTPSSLError, + HTTPStatusError, + HTTPTimeoutError, +) from .time import get_timestamp from .tls import SUPPORTED_PROTOCOL_VERSIONS, TlsConfig, create_ssl_context +if TYPE_CHECKING: + from .http_protocol import HTTPClientProtocol, HTTPResponseProtocol # noqa: F401 + # See Performance Optimizations in this package's README.md. requests_kerberos = lazy_loader.load('requests_kerberos') requests_ntlm = lazy_loader.load('requests_ntlm') @@ -434,6 +448,20 @@ def __init__(self, instance, init_config, remapper=None, logger=None, session=No self.tls_config = {key: value for key, value in config.items() if key.startswith('tls_')} self._https_adapters = {} + def get_header(self, name: str, default: str | None = None) -> str | None: + """Look up a request header by name. Lookup is case-insensitive.""" + for key, value in self.options['headers'].items(): + if key.lower() == name.lower(): + return value + return default + + def set_header(self, name: str, value: str) -> None: + for key in self.options['headers']: + if key.lower() == name.lower(): + self.options['headers'][key] = value + return + self.options['headers'][name] = value + def get(self, url, **options): return self._request('get', url, options) diff --git a/datadog_checks_base/datadog_checks/base/utils/http_exceptions.py b/datadog_checks_base/datadog_checks/base/utils/http_exceptions.py new file mode 100644 index 0000000000000..a807c063780c8 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/http_exceptions.py @@ -0,0 +1,40 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Any + +__all__ = [ + 'HTTPError', + 'HTTPRequestError', + 'HTTPStatusError', + 'HTTPTimeoutError', + 'HTTPConnectionError', + 'HTTPSSLError', +] + + +class HTTPError(Exception): + def __init__(self, message: str, response: Any = None, request: Any = None): + super().__init__(message) + self.response = response + self.request = request + + +class HTTPRequestError(HTTPError): + pass + + +class HTTPStatusError(HTTPError): + pass + + +class HTTPTimeoutError(HTTPRequestError): + pass + + +class HTTPConnectionError(HTTPRequestError): + pass + + +class HTTPSSLError(HTTPConnectionError): + pass diff --git a/datadog_checks_base/datadog_checks/base/utils/http_protocol.py b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py new file mode 100644 index 0000000000000..4d103de5930c3 --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/http_protocol.py @@ -0,0 +1,46 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Iterator, Protocol + + +class HTTPResponseProtocol(Protocol): + status_code: int + content: bytes + text: str + headers: Mapping[str, str] + + @property + def ok(self) -> bool: ... + @property + def reason(self) -> str: ... + + def json(self, **kwargs: Any) -> Any: ... + def raise_for_status(self) -> None: ... + def close(self) -> None: ... + def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: ... + def iter_lines( + self, + chunk_size: int | None = None, + decode_unicode: bool = False, + delimiter: bytes | str | None = None, + ) -> Iterator[bytes | str]: ... + def __enter__(self) -> HTTPResponseProtocol: ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: ... + + +class HTTPClientProtocol(Protocol): + options: dict[str, Any] + + def get(self, url: str, **options: Any) -> HTTPResponseProtocol: ... + def post(self, url: str, **options: Any) -> HTTPResponseProtocol: ... + def head(self, url: str, **options: Any) -> HTTPResponseProtocol: ... + def put(self, url: str, **options: Any) -> HTTPResponseProtocol: ... + def patch(self, url: str, **options: Any) -> HTTPResponseProtocol: ... + def delete(self, url: str, **options: Any) -> HTTPResponseProtocol: ... + def options_method(self, url: str, **options: Any) -> HTTPResponseProtocol: ... + def get_header(self, name: str, default: str | None = None) -> str | None: ... + def set_header(self, name: str, value: str) -> None: ... diff --git a/datadog_checks_base/datadog_checks/base/utils/http_testing.py b/datadog_checks_base/datadog_checks/base/utils/http_testing.py new file mode 100644 index 0000000000000..4e887c987eefa --- /dev/null +++ b/datadog_checks_base/datadog_checks/base/utils/http_testing.py @@ -0,0 +1,169 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json +from datetime import timedelta +from http.client import responses as http_responses +from io import BytesIO +from textwrap import dedent +from typing import Any, Iterator +from unittest.mock import MagicMock + +from datadog_checks.base.utils.http_exceptions import HTTPStatusError + +__all__ = ['MockHTTPResponse'] + + +class _CaseInsensitiveDict(dict): + """Case-insensitive dict for HTTP headers. Keys are stored lowercased.""" + + def __init__(self, data=None): + super().__init__() + if data: + for k, v in data.items(): + self[k] = v + + def __setitem__(self, key, value): + super().__setitem__(key.lower() if isinstance(key, str) else key, value) + + def __getitem__(self, key): + return super().__getitem__(key.lower() if isinstance(key, str) else key) + + def __contains__(self, key): + return super().__contains__(key.lower() if isinstance(key, str) else key) + + def __delitem__(self, key): + super().__delitem__(key.lower() if isinstance(key, str) else key) + + def get(self, key, default=None): + return super().get(key.lower() if isinstance(key, str) else key, default) + + def pop(self, key, *args): + return super().pop(key.lower() if isinstance(key, str) else key, *args) + + def update(self, other=(), **kwargs): + if isinstance(other, dict): + other = {(k.lower() if isinstance(k, str) else k): v for k, v in other.items()} + elif other: + other = [(k.lower() if isinstance(k, str) else k, v) for k, v in other] + kwargs = {k.lower(): v for k, v in kwargs.items()} + super().update(other, **kwargs) + + def setdefault(self, key, default=None): + return super().setdefault(key.lower() if isinstance(key, str) else key, default) + + +class MockHTTPResponse: + """Library-agnostic mock HTTP response implementing HTTPResponseProtocol.""" + + # Parameter order differs from MockResponse; not a compatibility concern since all callers use keyword args. + def __init__( + self, + content: str | bytes = '', + status_code: int = 200, + headers: dict[str, str] | None = None, + json_data: Any = None, + file_path: str | None = None, + cookies: dict[str, str] | None = None, + elapsed_seconds: float = 0.1, + normalize_content: bool = True, + url: str = '', + ): + self.url = url + + if json_data is not None: + content = json.dumps(json_data) + # Copy to avoid mutating the caller's dict + headers = dict(headers) if headers is not None else {} + headers.setdefault('Content-Type', 'application/json') + elif file_path is not None: + # Open in binary mode to handle both text and binary files correctly + # This prevents encoding errors and platform-specific newline translation + with open(file_path, 'rb') as f: + content = f.read() + + if normalize_content and ( + (isinstance(content, str) and content.startswith('\n')) + or (isinstance(content, bytes) and content.startswith(b'\n')) + ): + content = dedent(content[1:]) if isinstance(content, str) else content[1:] + + self._content = content.encode('utf-8') if isinstance(content, str) else content + self.status_code = status_code + self.headers = _CaseInsensitiveDict(headers or {}) + self.cookies = cookies or {} + self.encoding: str | None = None + self.elapsed = timedelta(seconds=elapsed_seconds) + self._stream = BytesIO(self._content) + + self.raw = MagicMock() + self.raw.read = self._stream.read + self.raw.connection.sock.getpeercert.side_effect = lambda binary_form=False: b'mock-cert' if binary_form else {} + + @property + def content(self) -> bytes: + return self._content + + @property + def text(self) -> str: + return self._content.decode('utf-8') + + @property + def ok(self) -> bool: + # Transitional: mirrors requests.Response.ok for current production code. + # httpx uses is_success/is_client_error/is_server_error instead. + return self.status_code < 400 + + @property + def reason(self) -> str: + return http_responses.get(self.status_code, '') + + def json(self, **kwargs: Any) -> Any: + return json.loads(self.text, **kwargs) + + def raise_for_status(self) -> None: + if self.status_code >= 400: + message = ( + f'{self.status_code} Client Error' if self.status_code < 500 else f'{self.status_code} Server Error' + ) + raise HTTPStatusError(message, response=self) + + def iter_content(self, chunk_size: int | None = None, decode_unicode: bool = False) -> Iterator[bytes | str]: + # chunk_size=None means return the entire content as a single chunk (matches requests behavior) + chunk_size = chunk_size if chunk_size is not None else len(self._content) or 1 + self._stream.seek(0) + while chunk := self._stream.read(chunk_size): + # Decode to string when decode_unicode=True (matches requests behavior) + yield chunk.decode('utf-8') if decode_unicode else chunk + + def iter_lines( + self, chunk_size: int | None = None, decode_unicode: bool = False, delimiter: bytes | str | None = None + ) -> Iterator[bytes | str]: + # Handle string delimiter by converting to bytes + if isinstance(delimiter, str): + delimiter = delimiter.encode('utf-8') + delimiter = delimiter or b'\n' + + self._stream.seek(0) + lines = self._stream.read().split(delimiter) + # bytes.split() produces a trailing empty element when content ends with the + # delimiter (e.g. b'a\nb\n'.split(b'\n') == [b'a', b'b', b'']). requests uses + # splitlines() for the default case which does not have this behavior, so we + # strip the trailing empty element to match. + if lines and not lines[-1]: + lines.pop() + for line in lines: + # Decode to string when decode_unicode=True (matches requests behavior) + yield line.decode('utf-8') if decode_unicode else line + + def close(self) -> None: + # No-op: requests.Response.close() releases the network connection, but + # content is already buffered in memory. Matching that behaviour here + # so the same instance can be returned by a mock multiple times. + pass + + def __enter__(self) -> 'MockHTTPResponse': + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool | None: + return None diff --git a/datadog_checks_base/tests/base/checks/openmetrics/test_legacy/test_openmetrics.py b/datadog_checks_base/tests/base/checks/openmetrics/test_legacy/test_openmetrics.py index 1435ee2d2f6e1..363de7a8ce3bc 100644 --- a/datadog_checks_base/tests/base/checks/openmetrics/test_legacy/test_openmetrics.py +++ b/datadog_checks_base/tests/base/checks/openmetrics/test_legacy/test_openmetrics.py @@ -19,9 +19,9 @@ from prometheus_client.samples import Sample from datadog_checks.base import ensure_bytes +from datadog_checks.base.utils.http_testing import MockHTTPResponse from datadog_checks.checks.openmetrics import OpenMetricsBaseCheck from datadog_checks.dev import get_here -from datadog_checks.dev.http import MockResponse text_content_type = 'text/plain; version=0.0.4' FIXTURE_PATH = os.path.abspath(os.path.join(get_here(), '..', '..', '..', '..', 'fixtures', 'prometheus')) @@ -113,7 +113,7 @@ def test_config_instance(mocked_prometheus_check): def test_process(text_data, mocked_prometheus_check, mocked_prometheus_scraper_config, ref_gauge): check = mocked_prometheus_check - check.poll = mock.MagicMock(return_value=MockResponse(text_data, headers={'Content-Type': text_content_type})) + check.poll = mock.MagicMock(return_value=MockHTTPResponse(text_data, headers={'Content-Type': text_content_type})) check.process_metric = mock.MagicMock() check.process(mocked_prometheus_scraper_config) check.poll.assert_called_with(mocked_prometheus_scraper_config) @@ -734,7 +734,7 @@ def test_filter_sample_on_gauge(p_check, mocked_prometheus_scraper_config): expected_metric.add_metric(['heapster-v1.4.3'], 1) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check mocked_prometheus_scraper_config['_text_filter_blacklist'] = ["deployment=\"kube-dns\""] metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -767,7 +767,7 @@ def test_parse_one_gauge(p_check, mocked_prometheus_scraper_config): expected_etcd_metric.add_metric([], 1) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -799,7 +799,7 @@ def test_parse_one_counter(p_check, mocked_prometheus_scraper_config): expected_etcd_metric.name = 'go_memstats_mallocs_total' # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -857,7 +857,7 @@ def test_parse_one_histograms_with_label(p_check, mocked_prometheus_scraper_conf ) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -991,7 +991,7 @@ def test_parse_one_histogram(p_check, mocked_prometheus_scraper_config): ) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) assert 1 == len(metrics) @@ -1093,7 +1093,7 @@ def test_parse_two_histograms_with_label(p_check, mocked_prometheus_scraper_conf ) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1131,7 +1131,7 @@ def test_decumulate_histogram_buckets(p_check, mocked_prometheus_scraper_config) 'rest_client_request_latency_seconds_count{url="http://127.0.0.1:8080/api",verb="GET"} 755\n' ) - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1220,7 +1220,7 @@ def test_decumulate_histogram_buckets_single_bucket(p_check, mocked_prometheus_s 'rest_client_request_latency_seconds_count{url="http://127.0.0.1:8080/api",verb="GET"} 755\n' ) - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1283,7 +1283,7 @@ def test_decumulate_histogram_buckets_multiple_contexts(p_check, mocked_promethe 'rest_client_request_latency_seconds_count{url="http://127.0.0.1:8080/api",verb="POST"} 150\n' ) - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1351,7 +1351,7 @@ def test_decumulate_histogram_buckets_negative_buckets(p_check, mocked_prometheu 'random_histogram_count{url="http://127.0.0.1:8080/api",verb="GET"} 70\n' ) - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1403,7 +1403,7 @@ def test_decumulate_histogram_buckets_no_buckets(p_check, mocked_prometheus_scra 'rest_client_request_latency_seconds_count{url="http://127.0.0.1:8080/api",verb="GET"} 755\n' ) - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1474,7 +1474,7 @@ def test_parse_one_summary(p_check, mocked_prometheus_scraper_config): expected_etcd_metric.add_sample("http_response_size_bytes", {"handler": "prometheus", "quantile": "0.99"}, 25763.0) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1517,7 +1517,7 @@ def test_parse_one_summary_with_no_quantile(p_check, mocked_prometheus_scraper_c expected_etcd_metric.add_metric(["prometheus"], 5.0, 120512.0) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1572,7 +1572,7 @@ def test_parse_two_summaries_with_labels(p_check, mocked_prometheus_scraper_conf ) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) @@ -1613,7 +1613,7 @@ def test_parse_one_summary_with_none_values(p_check, mocked_prometheus_scraper_c ) # Iter on the generator to get all metrics - response = MockResponse(text_data, headers={'Content-Type': text_content_type}) + response = MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) check = p_check metrics = list(check.parse_metric_family(response, mocked_prometheus_scraper_config)) assert 1 == len(metrics) @@ -2634,7 +2634,7 @@ def test_filter_metrics( def test_metadata_default(mocked_openmetrics_check_factory, text_data, datadog_agent): instance = dict(OPENMETRICS_CHECK_INSTANCE) check = mocked_openmetrics_check_factory(instance) - check.poll = mock.MagicMock(return_value=MockResponse(text_data, headers={'Content-Type': text_content_type})) + check.poll = mock.MagicMock(return_value=MockHTTPResponse(text_data, headers={'Content-Type': text_content_type})) check.check(instance) datadog_agent.assert_metadata_count(0) @@ -2645,7 +2645,7 @@ def test_metadata_transformer(mocked_openmetrics_check_factory, text_data, datad instance['metadata_metric_name'] = 'kubernetes_build_info' instance['metadata_label_map'] = {'version': 'gitVersion'} check = mocked_openmetrics_check_factory(instance) - check.poll = mock.MagicMock(return_value=MockResponse(text_data, headers={'Content-Type': text_content_type})) + check.poll = mock.MagicMock(return_value=MockHTTPResponse(text_data, headers={'Content-Type': text_content_type})) version_metadata = { 'version.major': '1', @@ -2672,7 +2672,10 @@ def test_ssl_verify_not_raise_warning(caplog, mocked_openmetrics_check_factory, check = mocked_openmetrics_check_factory(instance) scraper_config = check.get_scraper_config(instance) - with caplog.at_level(logging.DEBUG), mock.patch('requests.Session.get', return_value=MockResponse('httpbin.org')): + with ( + caplog.at_level(logging.DEBUG), + mock.patch('requests.Session.get', return_value=MockHTTPResponse('httpbin.org')), + ): resp = check.send_request('https://httpbin.org/get', scraper_config) assert "httpbin.org" in resp.content.decode('utf-8') @@ -2696,7 +2699,10 @@ def test_send_request_with_dynamic_prometheus_url(caplog, mocked_openmetrics_che # `prometheus_url` changed just before calling `send_request` scraper_config['prometheus_url'] = 'https://www.example.com/foo/bar' - with caplog.at_level(logging.DEBUG), mock.patch('requests.Session.get', return_value=MockResponse('httpbin.org')): + with ( + caplog.at_level(logging.DEBUG), + mock.patch('requests.Session.get', return_value=MockHTTPResponse('httpbin.org')), + ): resp = check.send_request('https://httpbin.org/get', scraper_config) assert "httpbin.org" in resp.content.decode('utf-8') @@ -2736,7 +2742,7 @@ def test_simple_type_overrides(aggregator, mocked_prometheus_check, text_data): config = check.get_scraper_config(instance) config['_dry_run'] = False - check.poll = mock.MagicMock(return_value=MockResponse(text_data, headers={'Content-Type': text_content_type})) + check.poll = mock.MagicMock(return_value=MockHTTPResponse(text_data, headers={'Content-Type': text_content_type})) check.process(config) aggregator.assert_metric('prometheus.process.vm.bytes', count=1, metric_type=aggregator.MONOTONIC_COUNT) @@ -2759,7 +2765,7 @@ def test_wildcard_type_overrides(aggregator, mocked_prometheus_check, text_data) config = check.get_scraper_config(instance) config['_dry_run'] = False - check.poll = mock.MagicMock(return_value=MockResponse(text_data, headers={'Content-Type': text_content_type})) + check.poll = mock.MagicMock(return_value=MockHTTPResponse(text_data, headers={'Content-Type': text_content_type})) check.process(config) aggregator.assert_metric('prometheus.process.vm.bytes', count=1, metric_type=aggregator.MONOTONIC_COUNT) @@ -2942,7 +2948,7 @@ def test_use_process_start_time( check = mocked_openmetrics_check_factory(instance) test_data = _make_test_use_process_start_time_data(process_start_time) - check.poll = mock.MagicMock(return_value=MockResponse(test_data, headers={'Content-Type': text_content_type})) + check.poll = mock.MagicMock(return_value=MockHTTPResponse(test_data, headers={'Content-Type': text_content_type})) for _ in range(0, 5): aggregator.reset() @@ -2995,7 +3001,9 @@ def test_refresh_bearer_token(text_data, mocked_openmetrics_check_factory): with patch.object(OpenMetricsBaseCheck, 'KUBERNETES_TOKEN_PATH', os.path.join(TOKENS_PATH, 'default_token')): check = mocked_openmetrics_check_factory(instance) - check.poll = mock.MagicMock(return_value=MockResponse(text_data, headers={'Content-Type': text_content_type})) + check.poll = mock.MagicMock( + return_value=MockHTTPResponse(text_data, headers={'Content-Type': text_content_type}) + ) instance = check.get_scraper_config(instance) assert instance['_bearer_token'] == 'my default token' time.sleep(1.5) diff --git a/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_config.py b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_config.py index 0f3f6b7812e68..4defa72e944c1 100644 --- a/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_config.py +++ b/datadog_checks_base/tests/base/checks/openmetrics/test_v2/test_config.py @@ -417,12 +417,10 @@ def test_strict_latest_spec(self, dd_run_check): check = get_check({'use_latest_spec': True}) check.configure_scrapers() scraper = check.scrapers['test'] - assert scraper.http.options['headers']['Accept'] == ( - 'application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1' - ) + assert scraper._use_latest_spec is True def test_dynamic_spec(self, dd_run_check): check = get_check({'use_latest_spec': False}) check.configure_scrapers() scraper = check.scrapers['test'] - assert scraper.http.options['headers']['Accept'] == 'text/plain' + assert scraper._use_latest_spec is False diff --git a/datadog_checks_base/tests/base/utils/http/test_authtoken.py b/datadog_checks_base/tests/base/utils/http/test_authtoken.py index 3a872371a68f9..710d65a35b4c5 100644 --- a/datadog_checks_base/tests/base/utils/http/test_authtoken.py +++ b/datadog_checks_base/tests/base/utils/http/test_authtoken.py @@ -10,10 +10,10 @@ from datadog_checks.base import ConfigurationError from datadog_checks.base.utils.http import DEFAULT_EXPIRATION, RequestsWrapper +from datadog_checks.base.utils.http_testing import MockHTTPResponse from datadog_checks.base.utils.time import get_timestamp from datadog_checks.dev import TempDir from datadog_checks.dev.fs import read_file, write_file -from datadog_checks.dev.http import MockResponse from .common import DEFAULT_OPTIONS, FIXTURE_PATH @@ -607,14 +607,14 @@ def login(*args, **kwargs): assert isinstance(decoded['exp'], int) assert abs(decoded['exp'] - (get_timestamp() + exp)) < 10 - return MockResponse(json_data={'token': 'auth-token'}) - return MockResponse(status_code=404) + return MockHTTPResponse(json_data={'token': 'auth-token'}) + return MockHTTPResponse(status_code=404) def auth(*args, **kwargs): if args[0] == 'https://leader.mesos/service/some-service': assert kwargs['headers']['Authorization'] == 'token=auth-token' - return MockResponse(json_data={}) - return MockResponse(status_code=404) + return MockHTTPResponse(json_data={}) + return MockHTTPResponse(status_code=404) with mock.patch('requests.post', side_effect=login), mock.patch('requests.Session.get', side_effect=auth): http = RequestsWrapper(instance, init_config) @@ -726,7 +726,7 @@ def raise_error_once(*args, **kwargs): if counter['errors'] <= 1: raise Exception - return MockResponse() + return MockHTTPResponse() expected_headers = {'Authorization': 'Bearer secret2'} expected_headers.update(DEFAULT_OPTIONS['headers']) diff --git a/datadog_checks_base/tests/base/utils/http/test_headers.py b/datadog_checks_base/tests/base/utils/http/test_headers.py index ab42172973153..dd3bca5d4b46a 100644 --- a/datadog_checks_base/tests/base/utils/http/test_headers.py +++ b/datadog_checks_base/tests/base/utils/http/test_headers.py @@ -97,3 +97,34 @@ def test_extra_headers_on_http_method_call(): # make sure the original headers are not modified assert http.options['headers'] == complete_headers assert extra_headers == {"foo": "bar"} + + +def test_get_header_default_for_missing(): + http = RequestsWrapper({}, {}) + assert http.get_header('X-Missing') is None + assert http.get_header('X-Missing', 'fallback') == 'fallback' + + +def test_get_header_case_insensitive(): + http = RequestsWrapper({}, {}) + assert http.get_header('accept') == '*/*' + assert http.get_header('Accept') == '*/*' + assert http.get_header('ACCEPT') == '*/*' + + +def test_set_header(): + http = RequestsWrapper({}, {}) + http.set_header('X-Token', 'abc123') + assert http.get_header('X-Token') == 'abc123' + http.set_header('Accept', 'application/json') + assert http.get_header('Accept') == 'application/json' + + +def test_set_header_case_insensitive(): + http = RequestsWrapper({}, {}) + http.set_header('accept', 'application/json') + # Overwrites the existing 'Accept' key (preserving original casing) + assert http.get_header('Accept') == 'application/json' + assert http.options['headers']['Accept'] == 'application/json' + # No duplicate key created + assert sum(1 for k in http.options['headers'] if k.lower() == 'accept') == 1 diff --git a/datadog_checks_base/tests/base/utils/http/test_http_testing.py b/datadog_checks_base/tests/base/utils/http/test_http_testing.py new file mode 100644 index 0000000000000..c3fe3d583cce1 --- /dev/null +++ b/datadog_checks_base/tests/base/utils/http/test_http_testing.py @@ -0,0 +1,141 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json + +import pytest + +from datadog_checks.base import AgentCheck +from datadog_checks.base.utils.http_exceptions import HTTPStatusError +from datadog_checks.base.utils.http_testing import MockHTTPResponse + + +def test_mock_http_patches_agentcheck(mock_http): + check = AgentCheck('test', {}, [{}]) + assert check.http is mock_http + + +def test_mock_response_json_with_custom_headers(): + headers = {'X-Custom': 'value'} + response = MockHTTPResponse(json_data={'key': 'value'}, headers=headers) + + assert response.headers['content-type'] == 'application/json' + assert response.headers['x-custom'] == 'value' + + +def test_mock_response_json_does_not_mutate_caller_headers(): + headers = {'X-Custom': 'value'} + MockHTTPResponse(json_data={'key': 'value'}, headers=headers) + + assert list(headers.keys()) == ['X-Custom'] + + +def test_mock_response_file_path(tmp_path): + f = tmp_path / 'fixture.txt' + f.write_bytes(b'file content') + + response = MockHTTPResponse(file_path=str(f)) + assert response.content == b'file content' + + +def test_mock_response_raise_for_status(): + response_404 = MockHTTPResponse(content='Not Found', status_code=404) + with pytest.raises(HTTPStatusError) as exc_info: + response_404.raise_for_status() + assert '404 Client Error' in str(exc_info.value) + assert exc_info.value.response is response_404 + + response_500 = MockHTTPResponse(content='Server Error', status_code=500) + with pytest.raises(HTTPStatusError) as exc_info: + response_500.raise_for_status() + assert '500 Server Error' in str(exc_info.value) + assert exc_info.value.response is response_500 + + +def test_mock_response_iter_content_chunks(): + response = MockHTTPResponse(content='hello world') + + chunks = list(response.iter_content(chunk_size=5)) + assert chunks == [b'hello', b' worl', b'd'] + + +def test_mock_response_iter_lines_preserves_empty_lines(): + content = 'line1\n\nline3\n' + response = MockHTTPResponse(content=content) + + lines = list(response.iter_lines()) + assert lines == [b'line1', b'', b'line3'] + + +def test_mock_response_normalize_leading_newline(): + content = '\nActual content' + response = MockHTTPResponse(content=content) + + assert response.text == 'Actual content' + + +def test_mock_response_normalize_leading_newline_with_indent(): + content = """ + line one + line two + """ + response = MockHTTPResponse(content=content) + assert response.text == "line one\nline two\n" + + +def test_mock_response_ok_property(): + assert MockHTTPResponse(status_code=200).ok is True + assert MockHTTPResponse(status_code=399).ok is True + assert MockHTTPResponse(status_code=400).ok is False + assert MockHTTPResponse(status_code=500).ok is False + + +def test_mock_response_reason_property(): + assert MockHTTPResponse(status_code=200).reason == 'OK' + assert MockHTTPResponse(status_code=404).reason == 'Not Found' + assert MockHTTPResponse(status_code=999).reason == '' + + +def test_mock_response_headers_case_insensitive(): + response = MockHTTPResponse(headers={'Content-Type': 'text/plain', 'X-Custom': 'val'}) + + assert response.headers['Content-Type'] == 'text/plain' + assert response.headers['content-type'] == 'text/plain' + assert response.headers.get('Content-Type') == 'text/plain' + assert response.headers.get('cOnTeNt-tYpE') == 'text/plain' + + +def test_mock_response_headers_delete_and_pop(): + response = MockHTTPResponse(headers={'Content-Type': 'text/plain', 'X-Custom': 'val'}) + + del response.headers['Content-Type'] + assert 'content-type' not in response.headers + + assert response.headers.pop('X-Custom') == 'val' + assert response.headers.pop('X-Custom', 'gone') == 'gone' + + +def test_mock_response_headers_update_and_setdefault(): + response = MockHTTPResponse(headers={'Content-Type': 'text/plain'}) + + response.headers.update({'X-New': 'new_val'}) + assert response.headers['x-new'] == 'new_val' + + response.headers.setdefault('X-Default', 'default_val') + assert response.headers['x-default'] == 'default_val' + + response.headers.setdefault('Content-Type', 'should-not-change') + assert response.headers['content-type'] == 'text/plain' + + response.headers.update([('X-Iter', 'iter_val')]) + assert response.headers['x-iter'] == 'iter_val' + + +def test_mock_response_url(): + assert MockHTTPResponse(url='http://example.com').url == 'http://example.com' + assert MockHTTPResponse().url == '' + + +def test_mock_response_raw_readable(): + response = MockHTTPResponse(json_data={'key': 'value'}) + assert json.load(response.raw) == {'key': 'value'} diff --git a/datadog_checks_base/tests/base/utils/http/test_kerberos_unit.py b/datadog_checks_base/tests/base/utils/http/test_kerberos_unit.py index fd4a51a07b0e6..161a837168a8e 100644 --- a/datadog_checks_base/tests/base/utils/http/test_kerberos_unit.py +++ b/datadog_checks_base/tests/base/utils/http/test_kerberos_unit.py @@ -9,8 +9,8 @@ from datadog_checks.base import ConfigurationError from datadog_checks.base.utils.http import RequestsWrapper +from datadog_checks.base.utils.http_testing import MockHTTPResponse from datadog_checks.dev import EnvVars -from datadog_checks.dev.http import MockResponse pytestmark = [pytest.mark.unit] @@ -115,7 +115,7 @@ def test_config_kerberos_keytab_file(): with mock.patch( 'requests.Session.get', - side_effect=lambda *args, **kwargs: MockResponse(os.environ.get('KRB5_CLIENT_KTNAME', '')), + side_effect=lambda *args, **kwargs: MockHTTPResponse(os.environ.get('KRB5_CLIENT_KTNAME', '')), ): response = http.get('https://www.google.com') assert response.text == '/test/file' @@ -132,7 +132,7 @@ def test_config_kerberos_cache(): assert os.environ.get('KRB5CCNAME') is None with mock.patch( - 'requests.Session.get', side_effect=lambda *args, **kwargs: MockResponse(os.environ.get('KRB5CCNAME', '')) + 'requests.Session.get', side_effect=lambda *args, **kwargs: MockHTTPResponse(os.environ.get('KRB5CCNAME', '')) ): response = http.get('https://www.google.com') assert response.text == '/test/file' @@ -148,7 +148,8 @@ def test_config_kerberos_cache_restores_rollback(): with EnvVars({'KRB5CCNAME': 'old'}): with mock.patch( - 'requests.Session.get', side_effect=lambda *args, **kwargs: MockResponse(os.environ.get('KRB5CCNAME', '')) + 'requests.Session.get', + side_effect=lambda *args, **kwargs: MockHTTPResponse(os.environ.get('KRB5CCNAME', '')), ): response = http.get('https://www.google.com') assert response.text == '/test/file' @@ -167,7 +168,7 @@ def test_config_kerberos_keytab_file_rollback(): with mock.patch( 'requests.Session.get', - side_effect=lambda *args, **kwargs: MockResponse(os.environ.get('KRB5_CLIENT_KTNAME', '')), + side_effect=lambda *args, **kwargs: MockHTTPResponse(os.environ.get('KRB5_CLIENT_KTNAME', '')), ): response = http.get('https://www.google.com') assert response.text == '/test/file' diff --git a/datadog_checks_dev/changelog.d/22676.added b/datadog_checks_dev/changelog.d/22676.added new file mode 100644 index 0000000000000..a7037be61a947 --- /dev/null +++ b/datadog_checks_dev/changelog.d/22676.added @@ -0,0 +1 @@ +Add mock_http fixture for library-agnostic HTTP client mocking in integration tests. \ No newline at end of file diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py index e9f9c80a513f0..7588d0cdfb38c 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py @@ -33,7 +33,9 @@ __aggregator = None __datadog_agent = None -MockResponse = None +MockHTTPResponse = None + +_DEFAULT_MOCK_METHOD = 'requests.Session.get' # TODO(httpx-migration): update when backend changes @pytest.fixture @@ -286,29 +288,67 @@ def dd_default_hostname(): @pytest.fixture def mock_response(): - # Lazily import `requests` as it may be costly under certain conditions - global MockResponse - if MockResponse is None: - from datadog_checks.dev.http import MockResponse + global MockHTTPResponse + if MockHTTPResponse is None: + from datadog_checks.base.utils.http_testing import MockHTTPResponse - yield MockResponse + yield MockHTTPResponse @pytest.fixture def mock_http_response(mocker, mock_response): yield lambda *args, **kwargs: mocker.patch( - kwargs.pop('method', 'requests.Session.get'), return_value=mock_response(*args, **kwargs) + kwargs.pop('method', _DEFAULT_MOCK_METHOD), return_value=mock_response(*args, **kwargs) ) +@pytest.fixture +def mock_http(mocker): + from unittest.mock import PropertyMock, create_autospec + + from datadog_checks.base.checks.base import AgentCheck + from datadog_checks.base.utils.http_protocol import HTTPClientProtocol + + client = create_autospec(HTTPClientProtocol) + # Protocol annotations are not picked up by create_autospec, so set options explicitly. + client.options = { + 'auth': None, + 'cert': None, + 'headers': {}, + 'proxies': None, + 'timeout': (10.0, 10.0), + 'verify': True, + 'allow_redirects': True, + } + + def _get_header(name, default=None): + for key, value in client.options['headers'].items(): + if key.lower() == name.lower(): + return value + return default + + def _set_header(name, value): + for key in list(client.options['headers']): + if key.lower() == name.lower(): + client.options['headers'][key] = value + return + client.options['headers'][name] = value + + client.get_header.side_effect = _get_header + client.set_header.side_effect = _set_header + client.options_method.side_effect = NotImplementedError('HTTP OPTIONS not yet supported in mock_http') + mocker.patch.object(AgentCheck, 'http', new_callable=PropertyMock, return_value=client) + return client + + @pytest.fixture def mock_http_response_per_endpoint(mocker, mock_response): @overload def _mock( - responses_by_endpoint: Dict[str, list[MockResponse]], + responses_by_endpoint: Dict[str, list[MockHTTPResponse]], *, mode: Literal["default"], - default_response: MockResponse, + default_response: MockHTTPResponse, method: str = ..., url_arg_index: int = ..., url_kwarg_name: str = ..., @@ -316,7 +356,7 @@ def _mock( ): ... @overload def _mock( - responses_by_endpoint: Dict[str, list[MockResponse]], + responses_by_endpoint: Dict[str, list[MockHTTPResponse]], *, mode: Literal["cycle", "exhaust"], default_response: None = None, @@ -326,10 +366,10 @@ def _mock( strict: bool = ..., ): ... def _mock( - responses_by_endpoint: Dict[str, list[MockResponse]], + responses_by_endpoint: Dict[str, list[MockHTTPResponse]], mode: Literal['cycle', 'exhaust', 'default'] = 'cycle', - default_response: MockResponse | None = None, - method: str = 'requests.Session.get', + default_response: MockHTTPResponse | None = None, + method: str = _DEFAULT_MOCK_METHOD, url_arg_index: int = 1, url_kwarg_name: str = "url", strict: bool = True, @@ -364,7 +404,7 @@ def side_effect(*args, **kwargs): if strict: raise ValueError(f"Endpoint {url} not found in mocked responses") else: - return MockResponse(status_code=404) + return mock_response(status_code=404) else: try: return next(queues[url]) diff --git a/dcgm/tests/test_unit.py b/dcgm/tests/test_unit.py index 911caad2cdd80..f55b92ca5bcf5 100644 --- a/dcgm/tests/test_unit.py +++ b/dcgm/tests/test_unit.py @@ -17,7 +17,7 @@ def test_critical_service_check(dd_run_check, aggregator, mock_http_response, ch When we can't connect to dcgm-exporter for whatever reason we should only submit a CRITICAL service check. """ mock_http_response(status_code=404) - with pytest.raises(Exception, match="requests.exceptions.HTTPError"): + with pytest.raises(Exception, match="HTTPStatusError"): dd_run_check(check) aggregator.assert_service_check('dcgm.openmetrics.health', status=check.CRITICAL) diff --git a/druid/tests/test_unit.py b/druid/tests/test_unit.py index 3a21281fbc8bc..b6381dc9d4903 100644 --- a/druid/tests/test_unit.py +++ b/druid/tests/test_unit.py @@ -19,17 +19,15 @@ def test_missing_url_config(aggregator): check.check({}) -def test_service_check_can_connect_success(aggregator, instance): +def test_service_check_can_connect_success(aggregator, instance, mock_http): check = DruidCheck('druid', {}, [instance]) - req = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req): - mock_resp = mock.MagicMock(status_code=200) - mock_resp.json.return_value = {'abc': '123'} - req.get.return_value = mock_resp + mock_resp = mock.MagicMock(status_code=200) + mock_resp.json.return_value = {'abc': '123'} + mock_http.get.return_value = mock_resp - resp = check._get_process_properties('http://hello-world.com:8899', ['foo:bar']) - assert resp == {'abc': '123'} + resp = check._get_process_properties('http://hello-world.com:8899', ['foo:bar']) + assert resp == {'abc': '123'} aggregator.assert_service_check( 'druid.service.can_connect', @@ -39,17 +37,15 @@ def test_service_check_can_connect_success(aggregator, instance): @pytest.mark.parametrize("exception_class", [requests.exceptions.ConnectionError, requests.exceptions.Timeout]) -def test_service_check_can_connect_failure(aggregator, instance, exception_class): +def test_service_check_can_connect_failure(aggregator, instance, mock_http, exception_class): check = DruidCheck('druid', {}, [instance]) - req = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req): - attrs = {'raise_for_status.side_effect': exception_class} - req.get.side_effect = [mock.MagicMock(status_code=500, **attrs)] + attrs = {'raise_for_status.side_effect': exception_class} + mock_http.get.side_effect = [mock.MagicMock(status_code=500, **attrs)] - with pytest.raises(CheckException): - properties = check._get_process_properties('http://hello-world.com:8899', ['foo:bar']) - assert properties is None + with pytest.raises(CheckException): + properties = check._get_process_properties('http://hello-world.com:8899', ['foo:bar']) + assert properties is None aggregator.assert_service_check( 'druid.service.can_connect', diff --git a/ecs_fargate/tests/test_unit.py b/ecs_fargate/tests/test_unit.py index c4f967166bcec..813e2a5768d51 100644 --- a/ecs_fargate/tests/test_unit.py +++ b/ecs_fargate/tests/test_unit.py @@ -289,24 +289,9 @@ def test_successful_check_wrong_sys_delta(check, aggregator, dd_run_check): [("explicit timeout", {'timeout': 30}, {'timeout': (30, 30)}), ("default timeout", {}, {'timeout': (5, 5)})], ) @pytest.mark.unit -def test_config(test_case, extra_config, expected_http_kwargs, dd_run_check): +def test_config(test_case, extra_config, expected_http_kwargs): instance = extra_config check = FargateCheck('ecs_fargate', {}, instances=[instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200) - - dd_run_check(check) - - http_wargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_wargs.update(expected_http_kwargs) - r.get.assert_called_with('http://169.254.170.2/v2/metadata', **http_wargs) + for key, value in expected_http_kwargs.items(): + assert check.http.options[key] == value diff --git a/envoy/tests/legacy/test_unit.py b/envoy/tests/legacy/test_unit.py index b43b531857fc6..cdcad5ba85923 100644 --- a/envoy/tests/legacy/test_unit.py +++ b/envoy/tests/legacy/test_unit.py @@ -18,7 +18,6 @@ EXT_AUTHZ_METRICS, EXT_PROC_METRICS, FLAVOR, - HOST, INSTANCES, LOCAL_RATE_LIMIT_METRICS, RATE_LIMIT_STAT_PREFIX_TAG, @@ -135,28 +134,13 @@ def test_unknown(fixture_path, mock_http_response, dd_run_check, check): pytest.param({}, {'verify': True}, id="legacy ssl config unset"), ], ) -def test_config(extra_config, expected_http_kwargs, check, dd_run_check): +def test_config(extra_config, expected_http_kwargs, check): instance = deepcopy(INSTANCES['main']) instance.update(extra_config) check = check(instance) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200) - - dd_run_check(check) - - http_wargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_wargs.update(expected_http_kwargs) - r.get.assert_called_with('http://{}:8001/stats'.format(HOST), **http_wargs) + for key, value in expected_http_kwargs.items(): + assert check.http.options[key] == value @pytest.mark.parametrize( diff --git a/etcd/tests/test_integration.py b/etcd/tests/test_integration.py index da94be9a527c3..b5387de0d9467 100644 --- a/etcd/tests/test_integration.py +++ b/etcd/tests/test_integration.py @@ -3,7 +3,6 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from copy import deepcopy -import mock import pytest from datadog_checks.etcd import Etcd @@ -63,41 +62,6 @@ def test_service_check(aggregator, instance, dd_run_check): aggregator.assert_service_check('etcd.prometheus.health', Etcd.OK, tags=tags, count=1) -@pytest.mark.parametrize( - 'test_case, extra_config, expected_http_kwargs', - [ - ("new auth config", {'username': 'new_foo', 'password': 'new_bar'}, {'auth': ('new_foo', 'new_bar')}), - ("legacy ssl config True", {'ssl_verify': True}, {'verify': True}), - ("legacy ssl config False", {'ssl_verify': False}, {'verify': False}), - ("legacy ssl config unset", {}, {'verify': False}), - ("timeout", {'prometheus_timeout': 100}, {'timeout': (100.0, 100.0)}), - ], -) -@pytest.mark.integration -def test_config(instance, test_case, extra_config, expected_http_kwargs, dd_run_check): - instance.update(extra_config) - check = Etcd(CHECK_NAME, {}, [instance]) - - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200) - - dd_run_check(check) - - http_kwargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'data': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_kwargs.update(expected_http_kwargs) - r.post.assert_called_with(URL + '/v3/maintenance/status', **http_kwargs) - - @pytest.mark.integration def test_version_metadata(aggregator, instance, dd_environment, datadog_agent, dd_run_check): check_instance = Etcd(CHECK_NAME, {}, [instance]) diff --git a/etcd/tests/test_unit.py b/etcd/tests/test_unit.py new file mode 100644 index 0000000000000..5987830c6efeb --- /dev/null +++ b/etcd/tests/test_unit.py @@ -0,0 +1,26 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.etcd import Etcd + +CHECK_NAME = 'etcd' + + +@pytest.mark.parametrize( + 'test_case, extra_config, expected_http_kwargs', + [ + ("new auth config", {'username': 'new_foo', 'password': 'new_bar'}, {'auth': ('new_foo', 'new_bar')}), + ("legacy ssl config True", {'ssl_verify': True}, {'verify': True}), + ("legacy ssl config False", {'ssl_verify': False}, {'verify': False}), + ("legacy ssl config unset", {}, {'verify': False}), + ("timeout", {'prometheus_timeout': 100}, {'timeout': (100.0, 100.0)}), + ], +) +def test_config(instance, test_case, extra_config, expected_http_kwargs): + instance.update(extra_config) + check = Etcd(CHECK_NAME, {}, [instance]) + + for key, value in expected_http_kwargs.items(): + assert check.http.options[key] == value diff --git a/falco/tests/test_unit.py b/falco/tests/test_unit.py index c2a544b1f780a..fdb03f05bbc30 100644 --- a/falco/tests/test_unit.py +++ b/falco/tests/test_unit.py @@ -1,12 +1,10 @@ # (C) Datadog, Inc. 2025-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from unittest import mock - import pytest from datadog_checks.base.constants import ServiceCheck -from datadog_checks.dev.http import MockResponse +from datadog_checks.base.utils.http_testing import MockHTTPResponse from datadog_checks.dev.utils import get_metadata_metrics from datadog_checks.falco import FalcoCheck @@ -22,13 +20,11 @@ def test_empty_instance(dd_run_check): dd_run_check(check) -def test_check_falco(dd_run_check, aggregator, instance): - mock_responses = [ - MockResponse(file_path=get_fixture_path("falco_metrics.txt")), +def test_check_falco(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = [ + MockHTTPResponse(file_path=get_fixture_path("falco_metrics.txt")), ] - - with mock.patch('requests.Session.get', side_effect=mock_responses): - dd_run_check(FalcoCheck('falco', {}, [instance])) + dd_run_check(FalcoCheck('falco', {}, [instance])) for metric in METRICS: aggregator.assert_metric(metric) diff --git a/gitlab_runner/tests/test_unit.py b/gitlab_runner/tests/test_unit.py index cc72d9a5ad861..35b89bc2109d2 100644 --- a/gitlab_runner/tests/test_unit.py +++ b/gitlab_runner/tests/test_unit.py @@ -4,7 +4,6 @@ from copy import deepcopy -import mock import pytest from datadog_checks.dev.utils import get_metadata_metrics @@ -29,22 +28,7 @@ def test_timeout(test_case, timeout_config, expected_timeout): gitlab_runner = GitlabRunnerCheck('gitlab_runner', common.CONFIG['init_config'], instances=config['instances']) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200) - - gitlab_runner.check(config['instances'][0]) - - r.get.assert_called_with( - 'http://localhost:8085/ci', - auth=mock.ANY, - cert=mock.ANY, - headers=mock.ANY, - proxies=mock.ANY, - timeout=expected_timeout, - verify=mock.ANY, - allow_redirects=mock.ANY, - ) + assert gitlab_runner.http.options['timeout'] == expected_timeout @pytest.mark.unit diff --git a/kubevirt_api/tests/conftest.py b/kubevirt_api/tests/conftest.py index 8556cf4d3956d..81af61164d28d 100644 --- a/kubevirt_api/tests/conftest.py +++ b/kubevirt_api/tests/conftest.py @@ -7,8 +7,8 @@ import pytest import yaml +from datadog_checks.base.utils.http_testing import MockHTTPResponse # noqa: F401 from datadog_checks.dev import get_here, run_command -from datadog_checks.dev.http import MockResponse from datadog_checks.dev.kind import kind_run from datadog_checks.dev.kube_port_forward import port_forward @@ -105,4 +105,4 @@ def mock_http_responses(url, **_params): raise Exception(f"url `{url}` not registered") with open(os.path.join(HERE, "fixtures", fixtures_file)) as f: - return MockResponse(content=f.read()) + return MockHTTPResponse(content=f.read()) diff --git a/kubevirt_api/tests/test_unit.py b/kubevirt_api/tests/test_unit.py index 1a4b040e86d05..0179b6c93586b 100644 --- a/kubevirt_api/tests/test_unit.py +++ b/kubevirt_api/tests/test_unit.py @@ -17,8 +17,8 @@ pytestmark = [pytest.mark.unit] -def test_check_collects_all_metrics(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check_collects_all_metrics(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtApiCheck("kubevirt_api", {}, [instance]) @@ -94,8 +94,8 @@ def test_check_collects_all_metrics(dd_run_check, aggregator, instance, mocker): aggregator.assert_metrics_using_metadata(get_metadata_metrics()) -def test_check_sends_zero_count_for_vms(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check_sends_zero_count_for_vms(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtApiCheck("kubevirt_api", {}, [instance]) @@ -115,8 +115,8 @@ def test_check_sends_zero_count_for_vms(dd_run_check, aggregator, instance, mock aggregator.assert_metric("kubevirt_api.vm.count", value=0) -def test_check_sends_zero_count_for_vmis(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check_sends_zero_count_for_vmis(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtApiCheck("kubevirt_api", {}, [instance]) @@ -153,8 +153,8 @@ def test_emits_zero_can_connect_when_service_is_down(dd_run_check, aggregator, i ) -def test_emits_one_can_connect_when_service_is_up(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_emits_one_can_connect_when_service_is_up(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtApiCheck("kubevirt_api", {}, [instance]) check._setup_kube_client = lambda: None @@ -170,8 +170,8 @@ def test_emits_one_can_connect_when_service_is_up(dd_run_check, aggregator, inst ) -def test_raise_exception_when_metrics_endpoint_is_bad(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_raise_exception_when_metrics_endpoint_is_bad(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtApiCheck("kubevirt_api", {}, [BAD_METRICS_HOSTNAME_INSTANCE]) check._setup_kube_client = lambda: None @@ -189,8 +189,8 @@ def test_raise_exception_when_metrics_endpoint_is_bad(dd_run_check, aggregator, ) -def test_raise_exception_cannot_connect_to_kubernetes_api(dd_run_check, aggregator, instance, mocker, caplog): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_raise_exception_cannot_connect_to_kubernetes_api(dd_run_check, aggregator, instance, mock_http, caplog): + mock_http.get.side_effect = mock_http_responses check = KubeVirtApiCheck("kubevirt_api", {}, [instance]) with pytest.raises( @@ -201,8 +201,8 @@ def test_raise_exception_cannot_connect_to_kubernetes_api(dd_run_check, aggregat assert "Cannot connect to Kubernetes API:" in caplog.text -def test_log_warning_healthz_endpoint_not_provided(dd_run_check, aggregator, instance, mocker, caplog): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_log_warning_healthz_endpoint_not_provided(dd_run_check, aggregator, instance, mock_http, caplog): + mock_http.get.side_effect = mock_http_responses new_instance = deepcopy(instance) new_instance.pop("kubevirt_api_healthz_endpoint") diff --git a/kubevirt_controller/tests/conftest.py b/kubevirt_controller/tests/conftest.py index e0e0174775b60..b2512fba1a7b4 100644 --- a/kubevirt_controller/tests/conftest.py +++ b/kubevirt_controller/tests/conftest.py @@ -7,8 +7,8 @@ import pytest import yaml +from datadog_checks.base.utils.http_testing import MockHTTPResponse # noqa: F401 from datadog_checks.dev import get_here, run_command -from datadog_checks.dev.http import MockResponse from datadog_checks.dev.kind import kind_run from datadog_checks.dev.kube_port_forward import port_forward @@ -101,4 +101,4 @@ def mock_http_responses(url, **_params): raise Exception(f"url `{url}` not registered") with open(os.path.join(HERE, "fixtures", fixtures_file)) as f: - return MockResponse(content=f.read()) + return MockHTTPResponse(content=f.read()) diff --git a/kubevirt_controller/tests/test_unit.py b/kubevirt_controller/tests/test_unit.py index 0a6bcb988a40c..1b4aa53fe488a 100644 --- a/kubevirt_controller/tests/test_unit.py +++ b/kubevirt_controller/tests/test_unit.py @@ -15,8 +15,8 @@ ] -def test_emits_can_connect_one_when_service_is_up(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_emits_can_connect_one_when_service_is_up(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtControllerCheck("kubevirt_controller", {}, [instance]) dd_run_check(check) aggregator.assert_metric( @@ -35,8 +35,8 @@ def test_emits_can_connect_zero_when_service_is_down(dd_run_check, aggregator, i ) -def test_check_collects_all_metrics(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check_collects_all_metrics(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtControllerCheck("kubevirt_controller", {}, [instance]) diff --git a/kubevirt_handler/tests/conftest.py b/kubevirt_handler/tests/conftest.py index 7bc0bc1bad6d3..294abc17ab1e9 100644 --- a/kubevirt_handler/tests/conftest.py +++ b/kubevirt_handler/tests/conftest.py @@ -6,8 +6,8 @@ import pytest +from datadog_checks.base.utils.http_testing import MockHTTPResponse # noqa: F401 from datadog_checks.dev import get_here, run_command -from datadog_checks.dev.http import MockResponse from datadog_checks.dev.kind import kind_run from datadog_checks.dev.kube_port_forward import port_forward @@ -89,4 +89,4 @@ def mock_http_responses(url, **_params): raise Exception(f"url `{url}` not registered") with open(os.path.join(HERE, "fixtures", fixtures_file)) as f: - return MockResponse(content=f.read()) + return MockHTTPResponse(content=f.read()) diff --git a/kubevirt_handler/tests/test_unit.py b/kubevirt_handler/tests/test_unit.py index 01e8e42c4a9ba..28561e3e5a327 100644 --- a/kubevirt_handler/tests/test_unit.py +++ b/kubevirt_handler/tests/test_unit.py @@ -16,8 +16,8 @@ ] -def test_check_collects_metrics(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check_collects_metrics(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtHandlerCheck("kubevirt_handler", {}, [instance]) dd_run_check(check) @@ -121,8 +121,8 @@ def test_check_collects_metrics(dd_run_check, aggregator, instance, mocker): aggregator.assert_metrics_using_metadata(get_metadata_metrics()) -def test_logs_warning_when_healthz_endpoint_is_missing(dd_run_check, aggregator, instance, mocker, caplog): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_logs_warning_when_healthz_endpoint_is_missing(dd_run_check, aggregator, instance, mock_http, caplog): + mock_http.get.side_effect = mock_http_responses del instance["kubevirt_handler_healthz_endpoint"] check = KubeVirtHandlerCheck("kubevirt_handler", {}, [instance]) dd_run_check(check) @@ -133,8 +133,8 @@ def test_logs_warning_when_healthz_endpoint_is_missing(dd_run_check, aggregator, ) -def test_emits_can_connect_one_when_service_is_up(dd_run_check, aggregator, instance, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_emits_can_connect_one_when_service_is_up(dd_run_check, aggregator, instance, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtHandlerCheck("kubevirt_handler", {}, [instance]) dd_run_check(check) aggregator.assert_metric( @@ -156,8 +156,8 @@ def test_emits_can_connect_zero_when_service_is_down(dd_run_check, aggregator, i ) -def test_version_metadata(instance, dd_run_check, datadog_agent, aggregator, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_version_metadata(instance, dd_run_check, datadog_agent, aggregator, mock_http): + mock_http.get.side_effect = mock_http_responses check = KubeVirtHandlerCheck("kubevirt_handler", {}, [instance]) check.check_id = "test:123" dd_run_check(check) diff --git a/marathon/tests/test_unit.py b/marathon/tests/test_unit.py index ef104e6264e90..042ab6f38ccd5 100644 --- a/marathon/tests/test_unit.py +++ b/marathon/tests/test_unit.py @@ -2,8 +2,8 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) from copy import deepcopy +from unittest.mock import MagicMock -import mock import pytest from datadog_checks.marathon import Marathon @@ -38,13 +38,13 @@ def test_process_apps_ko(check, aggregator): If the check can't hit the Marathon master Url, no metric should be collected """ - check.get_apps_json = mock.MagicMock(return_value=None) + check.get_apps_json = MagicMock(return_value=None) check.process_apps('url', 'acs_url', [], [], None) assert len(aggregator.metric_names) == 0 def test_process_apps(check, aggregator): - check.get_apps_json = mock.MagicMock( + check.get_apps_json = MagicMock( return_value={ 'apps': [ {'id': '/', 'version': '', 'backoffSeconds': 99}, @@ -107,20 +107,5 @@ def test_config(test_case, init_config, extra_config, expected_http_kwargs): instance.update(extra_config) check = Marathon('marathon', init_config, instances=[instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200) - - check.check(instance) - - http_wargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_wargs.update(expected_http_kwargs) - r.get.assert_called_with('http://localhost:8080/v2/queue', **http_wargs) + for key, value in expected_http_kwargs.items(): + assert check.http.options[key] == value diff --git a/mesos_master/tests/test_check.py b/mesos_master/tests/test_check.py index 4c026dcabc9c2..daff6b4033480 100644 --- a/mesos_master/tests/test_check.py +++ b/mesos_master/tests/test_check.py @@ -1,7 +1,8 @@ # (C) Datadog, Inc. 2018-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import mock +from unittest.mock import MagicMock + import pytest import requests @@ -84,28 +85,28 @@ def test_instance_timeout(check, instance): [ ( 'OK case for /state endpoint', - [mock.MagicMock(status_code=200, content='{}')], + [MagicMock(status_code=200, content='{}')], AgentCheck.OK, ['my:tag', 'url:http://hello.com/state'], False, ), ( 'OK case with failing /state due to bad status and fallback on /state.json', - [mock.MagicMock(status_code=500), mock.MagicMock(status_code=200, content='{}')], + [MagicMock(status_code=500), MagicMock(status_code=200, content='{}')], AgentCheck.OK, ['my:tag', 'url:http://hello.com/state.json'], False, ), ( 'OK case with failing /state due to Timeout and fallback on /state.json', - [requests.exceptions.Timeout, mock.MagicMock(status_code=200, content='{}')], + [requests.exceptions.Timeout, MagicMock(status_code=200, content='{}')], AgentCheck.OK, ['my:tag', 'url:http://hello.com/state.json'], False, ), ( 'OK case with failing /state due to Exception and fallback on /state.json', - [Exception, mock.MagicMock(status_code=200, content='{}')], + [Exception, MagicMock(status_code=200, content='{}')], AgentCheck.OK, ['my:tag', 'url:http://hello.com/state.json'], False, @@ -119,7 +120,7 @@ def test_instance_timeout(check, instance): ), ( 'NOK case with failing /state and /state.json with bad status', - [mock.MagicMock(status_code=500), mock.MagicMock(status_code=500)], + [MagicMock(status_code=500), MagicMock(status_code=500)], AgentCheck.CRITICAL, ['my:tag', 'url:http://hello.com/state.json'], True, @@ -127,8 +128,8 @@ def test_instance_timeout(check, instance): ( 'OK case with non-leader master on /state', [ - mock.MagicMock(status_code=401, history=[mock.MagicMock(status_code=307)]), - mock.MagicMock(content='{}', history=[], status_code=500), + MagicMock(status_code=401, history=[MagicMock(status_code=307)]), + MagicMock(content='{}', history=[], status_code=500), ], AgentCheck.UNKNOWN, ['my:tag', 'url:http://hello.com/state.json'], @@ -137,8 +138,8 @@ def test_instance_timeout(check, instance): ( 'OK case with non-leader master on /state.json', [ - mock.MagicMock(status_code=500, history=[]), - mock.MagicMock(content='{}', history=[mock.MagicMock(status_code=307)], status_code=401), + MagicMock(status_code=500, history=[]), + MagicMock(content='{}', history=[MagicMock(status_code=307)], status_code=401), ], AgentCheck.UNKNOWN, ['my:tag', 'url:http://hello.com/state.json'], @@ -148,20 +149,25 @@ def test_instance_timeout(check, instance): ) @pytest.mark.integration def test_can_connect_service_check( - instance, aggregator, test_case_name, request_mock_side_effects, expected_status, expected_tags, expect_exception + instance, + aggregator, + mock_http, + test_case_name, + request_mock_side_effects, + expected_status, + expected_tags, + expect_exception, ): check = MesosMaster('mesos_master', {}, [instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.side_effect = request_mock_side_effects + mock_http.get.side_effect = request_mock_side_effects - try: - check._get_master_state('http://hello.com', ['my:tag']) - exception_raised = False - except CheckException: - exception_raised = True + try: + check._get_master_state('http://hello.com', ['my:tag']) + exception_raised = False + except CheckException: + exception_raised = True - assert expect_exception == exception_raised + assert expect_exception == exception_raised aggregator.assert_service_check('mesos_master.can_connect', count=1, status=expected_status, tags=expected_tags) diff --git a/mesos_slave/tests/test_unit.py b/mesos_slave/tests/test_unit.py index 098e3683313cf..e04738108caed 100644 --- a/mesos_slave/tests/test_unit.py +++ b/mesos_slave/tests/test_unit.py @@ -177,37 +177,40 @@ def test_config(check, instance, test_case, extra_config, expected_http_kwargs): @pytest.mark.parametrize(PARAMETERS, state_test_data) @pytest.mark.integration def test_can_connect_service_check_state( - instance, aggregator, test_case_name, request_mock_effects, expected_tags, expect_exception, expected_status + instance, + aggregator, + mock_http, + test_case_name, + request_mock_effects, + expected_tags, + expect_exception, + expected_status, ): check = MesosSlave('mesos_slave', {}, [instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.side_effect = request_mock_effects - try: - check._process_state_info('http://hello.com', instance['tasks'], 5050, instance['tags']) - assert not expect_exception - except Exception: - if not expect_exception: - raise + mock_http.get.side_effect = request_mock_effects + try: + check._process_state_info('http://hello.com', instance['tasks'], 5050, instance['tags']) + assert not expect_exception + except Exception: + if not expect_exception: + raise aggregator.assert_service_check('mesos_slave.can_connect', count=1, status=expected_status, tags=expected_tags) @pytest.mark.integration -def test_can_connect_service_with_instance_cluster_name(instance, aggregator): +def test_can_connect_service_with_instance_cluster_name(instance, aggregator, mock_http): instance['cluster_name'] = 'test-cluster' expected_tags = ['url:http://hello.com/state'] + cluster_name_tag + additional_tags expected_status = AgentCheck.OK check = MesosSlave('mesos_slave', {}, [instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.side_effect = [mock.MagicMock(status_code=200, content='{}')] - try: - check._process_state_info('http://hello.com', instance['tasks'], 5050, instance['tags']) - assert not False - except Exception: - if not False: - raise + mock_http.get.side_effect = [mock.MagicMock(status_code=200, content='{}')] + try: + check._process_state_info('http://hello.com', instance['tasks'], 5050, instance['tags']) + assert not False + except Exception: + if not False: + raise aggregator.assert_service_check('mesos_slave.can_connect', count=1, status=expected_status, tags=expected_tags) @@ -215,17 +218,22 @@ def test_can_connect_service_with_instance_cluster_name(instance, aggregator): @pytest.mark.parametrize(PARAMETERS, stats_test_data) @pytest.mark.integration def test_can_connect_service_check_stats( - instance, aggregator, test_case_name, request_mock_effects, expected_tags, expect_exception, expected_status + instance, + aggregator, + mock_http, + test_case_name, + request_mock_effects, + expected_tags, + expect_exception, + expected_status, ): check = MesosSlave('mesos_slave', {}, [instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.side_effect = request_mock_effects - try: - check._process_stats_info('http://hello.com', instance['tags']) - assert not expect_exception - except Exception: - if not expect_exception: - raise + mock_http.get.side_effect = request_mock_effects + try: + check._process_stats_info('http://hello.com', instance['tags']) + assert not expect_exception + except Exception: + if not expect_exception: + raise aggregator.assert_service_check('mesos_slave.can_connect', count=1, status=expected_status, tags=expected_tags) diff --git a/nginx/tests/test_unit.py b/nginx/tests/test_unit.py index 66de9c5dfd748..655c41583eb63 100644 --- a/nginx/tests/test_unit.py +++ b/nginx/tests/test_unit.py @@ -68,34 +68,16 @@ def test_config(check, instance, test_case, extra_config, expected_http_kwargs): c = check(instance) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200, content=b'{}') + for key, value in expected_http_kwargs.items(): + assert c.http.options[key] == value - c.check(instance) - http_wargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_wargs.update(expected_http_kwargs) - - r.get.assert_called_with('http://localhost:8080/nginx_status', **http_wargs) - - -def test_no_version(check, instance, caplog): +def test_no_version(check, instance, caplog, mock_http): c = check(instance) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200, content=b'{}', headers={'server': 'nginx'}) + mock_http.get.return_value = mock.MagicMock(status_code=200, content=b'{}', headers={'server': 'nginx'}) - c.check(instance) + c.check(instance) errors = [record for record in caplog.records if record.levelname == "ERROR"] assert not errors diff --git a/nvidia_nim/tests/test_unit.py b/nvidia_nim/tests/test_unit.py index deaeffedf3ae8..c6ea3eb977340 100644 --- a/nvidia_nim/tests/test_unit.py +++ b/nvidia_nim/tests/test_unit.py @@ -54,7 +54,7 @@ def test_emits_critical_openemtrics_service_check_when_service_is_down( """ mock_http_response(status_code=404) check = NvidiaNIMCheck("nvidia_nim", {}, [instance]) - with pytest.raises(Exception, match="requests.exceptions.HTTPError"): + with pytest.raises(Exception, match="HTTPStatusError"): dd_run_check(check) aggregator.assert_all_metrics_covered() diff --git a/nvidia_triton/tests/test_unit.py b/nvidia_triton/tests/test_unit.py index 31c6b7e0551d2..e5c8fff456612 100644 --- a/nvidia_triton/tests/test_unit.py +++ b/nvidia_triton/tests/test_unit.py @@ -35,7 +35,7 @@ def test_emits_critical_openemtrics_service_check_when_service_is_down( """ mock_http_response(status_code=404) check = NvidiaTritonCheck('nvidia_triton', {}, [instance]) - with pytest.raises(Exception, match="requests.exceptions.HTTPError"): + with pytest.raises(Exception, match="HTTPStatusError"): dd_run_check(check) aggregator.assert_all_metrics_covered() diff --git a/openmetrics/tests/test_openmetrics.py b/openmetrics/tests/test_openmetrics.py index f0212951b1bde..b55f8693f7711 100644 --- a/openmetrics/tests/test_openmetrics.py +++ b/openmetrics/tests/test_openmetrics.py @@ -33,12 +33,9 @@ @pytest.mark.parametrize('poll_mock_fixture', ['prometheus_poll_mock', 'openmetrics_poll_mock']) def test_openmetrics(aggregator, dd_run_check, request, poll_mock_fixture): - from datadog_checks.base.checks.openmetrics.v2.scraper import OpenMetricsScraper - request.getfixturevalue(poll_mock_fixture) check = OpenMetricsCheck('openmetrics', {}, [instance_new]) - scraper = OpenMetricsScraper(check, instance_new) dd_run_check(check) aggregator.assert_metric( @@ -63,19 +60,15 @@ def test_openmetrics(aggregator, dd_run_check, request, poll_mock_fixture): ) aggregator.assert_all_metrics_covered() - assert check.http.options['headers']['Accept'] == '*/*' - assert scraper.http.options['headers']['Accept'] == 'text/plain' + assert check.http.get_header('Accept') == '*/*' def test_openmetrics_use_latest_spec(aggregator, dd_run_check, mock_http_response, openmetrics_payload, caplog): - from datadog_checks.base.checks.openmetrics.v2.scraper import OpenMetricsScraper - # We want to make sure that when `use_latest_spec` is enabled, we use the OpenMetrics parser # even when the response's `Content-Type` doesn't declare the appropriate media type. - mock_http_response(openmetrics_payload, normalize_content=False) + get_mock = mock_http_response(openmetrics_payload, normalize_content=False) check = OpenMetricsCheck('openmetrics', {}, [instance_new_strict]) - scraper = OpenMetricsScraper(check, instance_new_strict) dd_run_check(check) aggregator.assert_metric( @@ -95,9 +88,9 @@ def test_openmetrics_use_latest_spec(aggregator, dd_run_check, mock_http_respons ) aggregator.assert_all_metrics_covered() - assert check.http.options['headers']['Accept'] == '*/*' + assert check.http.get_header('Accept') == '*/*' assert caplog.text == '' - assert scraper.http.options['headers']['Accept'] == ( + assert get_mock.call_args.kwargs['headers']['Accept'] == ( 'application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1' ) diff --git a/php_fpm/tests/test_unit.py b/php_fpm/tests/test_unit.py index b0f46d613c077..c70af599f53a4 100644 --- a/php_fpm/tests/test_unit.py +++ b/php_fpm/tests/test_unit.py @@ -33,50 +33,44 @@ def test_bad_ping(aggregator, dd_run_check): aggregator.all_metrics_asserted() -def test_should_not_retry(check, instance): +def test_should_not_retry(check, instance, mock_http): """ backoff only works when response code is 503, otherwise the error should bubble up """ - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.side_effect = FooException("Generic http error here") - with pytest.raises(FooException): - check._process_status(instance['status_url'], [], None, False) + mock_http.get.side_effect = FooException("Generic http error here") + with pytest.raises(FooException): + check._process_status(instance['status_url'], [], None, False) -def test_should_bail_out(check, instance): +def test_should_bail_out(check, instance, mock_http): """ backoff should give up after 3 attempts """ - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - attrs = {'raise_for_status.side_effect': FooException()} - r.get.side_effect = [ - mock.MagicMock(status_code=503, **attrs), - mock.MagicMock(status_code=503, **attrs), - mock.MagicMock(status_code=503, **attrs), - mock.MagicMock(status_code=200), - ] - with pytest.raises(FooException): - check._process_status(instance['status_url'], [], None, False) - - -def test_backoff_success(check, instance, aggregator, payload): + attrs = {'raise_for_status.side_effect': FooException()} + mock_http.get.side_effect = [ + mock.MagicMock(status_code=503, **attrs), + mock.MagicMock(status_code=503, **attrs), + mock.MagicMock(status_code=503, **attrs), + mock.MagicMock(status_code=200), + ] + with pytest.raises(FooException): + check._process_status(instance['status_url'], [], None, False) + + +def test_backoff_success(check, instance, aggregator, payload, mock_http): """ Success after 2 failed attempts """ instance['ping_url'] = None - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - attrs = {'json.return_value': payload} - r.get.side_effect = [ - mock.MagicMock(status_code=503), - mock.MagicMock(status_code=503), - mock.MagicMock(status_code=200, **attrs), - ] - pool_name = check._process_status(instance['status_url'], [], None, False) - assert pool_name == 'www' + attrs = {'json.return_value': payload} + mock_http.get.side_effect = [ + mock.MagicMock(status_code=503), + mock.MagicMock(status_code=503), + mock.MagicMock(status_code=200, **attrs), + ] + pool_name = check._process_status(instance['status_url'], [], None, False) + assert pool_name == 'www' @pytest.mark.parametrize( @@ -101,25 +95,14 @@ def test_backoff_success(check, instance, aggregator, payload): ), ], ) -def test_config(test_case, extra_config, expected_http_kwargs, dd_run_check): +def test_config(test_case, extra_config, expected_http_kwargs): instance = {'ping_url': 'http://foo:9001/ping'} instance.update(extra_config) check = PHPFPMCheck('php_fpm', {}, instances=[instance]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200) - - dd_run_check(check) - - http_kwargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_kwargs.update(expected_http_kwargs) - r.get.assert_called_with('http://foo:9001/ping', **http_kwargs) + for key, value in expected_http_kwargs.items(): + if key == 'headers': + for h_key, h_value in value.items(): + assert check.http.get_header(h_key) == h_value + else: + assert check.http.options[key] == value diff --git a/quarkus/tests/test_unit.py b/quarkus/tests/test_unit.py index 9f96137aa2e8c..b1df052419d7d 100644 --- a/quarkus/tests/test_unit.py +++ b/quarkus/tests/test_unit.py @@ -83,7 +83,7 @@ def test_emits_critical_service_check_when_service_is_down(dd_run_check, aggrega mock_http_response(status_code=404) check = QuarkusCheck('quarkus', {}, [instance]) # When - with pytest.raises(Exception, match="requests.exceptions.HTTPError"): + with pytest.raises(Exception, match="HTTPStatusError"): dd_run_check(check) # Then aggregator.assert_service_check('quarkus.openmetrics.health', QuarkusCheck.CRITICAL) diff --git a/rabbitmq/tests/test_openmetrics.py b/rabbitmq/tests/test_openmetrics.py index ce916324e5891..36990aa17f8e4 100644 --- a/rabbitmq/tests/test_openmetrics.py +++ b/rabbitmq/tests/test_openmetrics.py @@ -352,6 +352,6 @@ def test_config(prom_plugin_settings, err): def test_service_check_critical(aggregator, dd_run_check, mock_http_response): mock_http_response(status_code=404) check = _rmq_om_check({'url': 'http://fail'}) - with pytest.raises(Exception, match="requests.exceptions.HTTPError"): + with pytest.raises(Exception, match="HTTPStatusError"): dd_run_check(check) aggregator.assert_service_check('rabbitmq.openmetrics.health', status=check.CRITICAL) diff --git a/rabbitmq/tests/test_unit.py b/rabbitmq/tests/test_unit.py index 4e0d95ea9f06a..bffa86fc8cb6e 100644 --- a/rabbitmq/tests/test_unit.py +++ b/rabbitmq/tests/test_unit.py @@ -24,16 +24,12 @@ pytestmark = [pytest.mark.unit, common.requires_management] -def test__get_data(check): - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.side_effect = [requests.exceptions.HTTPError, ValueError] - with pytest.raises(RabbitMQException) as e: - check._get_data('') - assert isinstance(e, RabbitMQException) - with pytest.raises(RabbitMQException) as e: - check._get_data('') - assert isinstance(e, RabbitMQException) +def test__get_data(check, mock_http): + mock_http.get.side_effect = [requests.exceptions.HTTPError, ValueError] + with pytest.raises(RabbitMQException): + check._get_data('') + with pytest.raises(RabbitMQException): + check._get_data('') def test_status_check(check, aggregator): @@ -139,24 +135,8 @@ def test_config(check, test_case, extra_config, expected_http_kwargs): config.update(extra_config) check = RabbitMQ('rabbitmq', {}, instances=[config]) - r = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=r): - r.get.return_value = mock.MagicMock(status_code=200) - - check.check(config) - - http_wargs = { - 'auth': mock.ANY, - 'cert': mock.ANY, - 'headers': mock.ANY, - 'proxies': mock.ANY, - 'timeout': mock.ANY, - 'verify': mock.ANY, - 'allow_redirects': mock.ANY, - } - http_wargs.update(expected_http_kwargs) - - r.get.assert_called_with('http://localhost:15672/api/connections', **http_wargs) + for key, value in expected_http_kwargs.items(): + assert check.http.options[key] == value def test_nodes(aggregator, check): diff --git a/ray/tests/common.py b/ray/tests/common.py index 19b0c9f911678..d51a4faaf0061 100644 --- a/ray/tests/common.py +++ b/ray/tests/common.py @@ -4,8 +4,8 @@ import os +from datadog_checks.base.utils.http_testing import MockHTTPResponse from datadog_checks.dev import get_docker_hostname, get_here -from datadog_checks.dev.http import MockResponse HERE = get_here() @@ -333,4 +333,4 @@ def mock_http_responses(url, **_params): raise Exception(f"url `{url}` not registered") with open(os.path.join(HERE, 'fixtures', metrics_file)) as f: - return MockResponse(content=f.read()) + return MockHTTPResponse(content=f.read()) diff --git a/ray/tests/test_unit.py b/ray/tests/test_unit.py index f2e7725b5287f..4be5c8aa13ba9 100644 --- a/ray/tests/test_unit.py +++ b/ray/tests/test_unit.py @@ -16,8 +16,8 @@ pytest.param(MOCKED_WORKER_INSTANCE, WORKER_METRICS, id='worker'), ], ) -def test_check(dd_run_check, aggregator, mocker, check, instance, metrics): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check(dd_run_check, aggregator, mock_http, check, instance, metrics): + mock_http.get.side_effect = mock_http_responses dd_run_check(check(instance)) for expected_metric in metrics: @@ -30,10 +30,10 @@ def test_check(dd_run_check, aggregator, mocker, check, instance, metrics): assert len(aggregator.service_check_names) == 1 -def test_invalid_url(dd_run_check, aggregator, check, mocked_head_instance, mocker): +def test_invalid_url(dd_run_check, aggregator, check, mocked_head_instance, mock_http): mocked_head_instance["openmetrics_endpoint"] = "http://unknowwn" - mocker.patch("requests.Session.get", wraps=mock_http_responses) + mock_http.get.side_effect = mock_http_responses with pytest.raises(Exception): dd_run_check(check(mocked_head_instance)) diff --git a/sonatype_nexus/tests/test_check.py b/sonatype_nexus/tests/test_check.py index eda941ff13609..2171b10c995db 100644 --- a/sonatype_nexus/tests/test_check.py +++ b/sonatype_nexus/tests/test_check.py @@ -3,7 +3,6 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.dev.http import MockResponse from datadog_checks.dev.utils import get_metadata_metrics from datadog_checks.sonatype_nexus import constants from datadog_checks.sonatype_nexus.check import SonatypeNexusCheck @@ -12,25 +11,30 @@ from .conftest import instance -@pytest.fixture -def mock_http_response(mocker): - yield lambda *args, **kwargs: mocker.patch( - kwargs.pop("method", "requests.Session.get"), return_value=MockResponse(*args, **kwargs) - ) - - def test_successful_metrics_collection(dd_run_check, mock_http_response, aggregator): status_metrics_response_data = {key: {"healthy": True} for key in constants.STATUS_METRICS_MAP.keys()} + status_metrics_response_data["gauges"] = { + "jvm.memory.heap.used": {"value": 123456789}, + "nexus.analytics.bytes_transferred_by_format": { + "value": [{"maven": {"bytes_uploaded": 100, "bytes_downloaded": 200}}], + }, + "nexus.analytics.blobstore_type_counts": {"value": {"file": 5}}, + } mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=200, - json_data=status_metrics_response_data.update({"gauges": {"jvm.memory.heap.used": {"value": 123456789}}}), + json_data=status_metrics_response_data, ) check = SonatypeNexusCheck("sonatype_nexus", {}, [instance]) dd_run_check(check) + for metric_name in constants.STATUS_METRICS_MAP.values(): + aggregator.assert_metric(f"sonatype_nexus.{metric_name}") + aggregator.assert_metric("sonatype_nexus.analytics.jvm.heap_memory_used") + aggregator.assert_metric("sonatype_nexus.analytics.uploaded_bytes_by_format") + aggregator.assert_metric("sonatype_nexus.analytics.downloaded_bytes_by_format") + aggregator.assert_metric("sonatype_nexus.analytics.blob_store.count_by_type") aggregator.assert_all_metrics_covered() aggregator.assert_metrics_using_metadata(get_metadata_metrics()) @@ -83,7 +87,6 @@ def test_empty_instance(dd_run_check): def test_invalid_credentials(dd_run_check, mock_http_response): mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=401, json_data={"error": "Invalid credentials"}, ) @@ -103,7 +106,6 @@ def test_invalid_credentials(dd_run_check, mock_http_response): def test_bad_request_error(dd_run_check, mock_http_response): mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=400, json_data={"error": "Bad request"}, ) @@ -116,7 +118,6 @@ def test_bad_request_error(dd_run_check, mock_http_response): def test_license_expired_error(dd_run_check, mock_http_response): mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=402, json_data={"error": "License expired"}, ) @@ -129,7 +130,6 @@ def test_license_expired_error(dd_run_check, mock_http_response): def test_insufficient_permission_error(dd_run_check, mock_http_response): mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=403, json_data={"error": "Insufficient permissions"}, ) @@ -142,7 +142,6 @@ def test_insufficient_permission_error(dd_run_check, mock_http_response): def test_not_found_error(dd_run_check, mock_http_response): mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=404, json_data={"error": "Resource not found"}, ) @@ -155,7 +154,6 @@ def test_not_found_error(dd_run_check, mock_http_response): def test_server_error(dd_run_check, mock_http_response): mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=500, json_data={"error": "Internal server error"}, ) @@ -168,7 +166,6 @@ def test_server_error(dd_run_check, mock_http_response): def test_timeout_error(dd_run_check, mock_http_response): mock_http_response( - "https://example.com/service/rest/v1/status/check", status_code=408, json_data={"error": "TimeoutError"}, ) diff --git a/squid/tests/test_unit.py b/squid/tests/test_unit.py index 332e8c9123e2d..f1941b042eec9 100644 --- a/squid/tests/test_unit.py +++ b/squid/tests/test_unit.py @@ -103,17 +103,4 @@ def test_legacy_username_password(instance, auth_config): instance.update(auth_config) check = SquidCheck(common.CHECK_NAME, {}, {}, [instance]) - with mock.patch('datadog_checks.base.utils.http.requests.Session.get') as g: - with mock.patch('datadog_checks.squid.SquidCheck.submit_version'): - check.get_counters('host', 'port', []) - - g.assert_called_with( - 'http://host:port/squid-internal-mgr/counters', - auth=('datadog_user', 'datadog_pass'), - cert=mock.ANY, - headers=mock.ANY, - proxies=mock.ANY, - timeout=mock.ANY, - verify=mock.ANY, - allow_redirects=mock.ANY, - ) + assert check.http.options['auth'] == ('datadog_user', 'datadog_pass') diff --git a/strimzi/tests/conftest.py b/strimzi/tests/conftest.py index 0499c622ed9bb..16ce3da9f2b10 100644 --- a/strimzi/tests/conftest.py +++ b/strimzi/tests/conftest.py @@ -8,8 +8,8 @@ import pytest +from datadog_checks.base.utils.http_testing import MockHTTPResponse # noqa: F401 from datadog_checks.dev import run_command -from datadog_checks.dev.http import MockResponse from datadog_checks.dev.kind import kind_run from datadog_checks.dev.kube_port_forward import port_forward from datadog_checks.strimzi import StrimziCheck @@ -103,4 +103,4 @@ def mock_http_responses(url, **_params): pytest.fail(f"url `{url}` not registered") with open(os.path.join(HERE, 'fixtures', STRIMZI_VERSION, metrics_file)) as f: - return MockResponse(content=f.read()) + return MockHTTPResponse(content=f.read()) diff --git a/strimzi/tests/test_unit.py b/strimzi/tests/test_unit.py index 0c9475cefab1a..1cfb28a5478c1 100644 --- a/strimzi/tests/test_unit.py +++ b/strimzi/tests/test_unit.py @@ -54,9 +54,9 @@ def test_check_unique_operator( instance, metrics, tag, - mocker, + mock_http, ): - mocker.patch("requests.Session.get", wraps=mock_http_responses) + mock_http.get.side_effect = mock_http_responses dd_run_check(check(instance)) for expected_metric in metrics: @@ -75,8 +75,8 @@ def test_check_unique_operator( assert len(aggregator.service_check_names) == 1 -def test_check_all_operators(dd_run_check, aggregator, check, mocker): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check_all_operators(dd_run_check, aggregator, check, mock_http): + mock_http.get.side_effect = mock_http_responses dd_run_check( check( { diff --git a/teamcity/tests/test_teamcity.py b/teamcity/tests/test_teamcity.py index 65c23a4a12dab..2800bd02efcef 100644 --- a/teamcity/tests/test_teamcity.py +++ b/teamcity/tests/test_teamcity.py @@ -4,7 +4,6 @@ from copy import deepcopy import pytest -from mock import ANY, patch from datadog_checks.teamcity.constants import ( SERVICE_CHECK_BUILD_PROBLEMS, @@ -17,7 +16,6 @@ BUILD_STATS_METRICS, BUILD_TAGS, EXPECTED_SERVICE_CHECK_TEST_RESULTS, - LEGACY_REST_INSTANCE, USE_OPENMETRICS, ) @@ -53,36 +51,6 @@ def test_build_event(dd_run_check, aggregator, rest_instance): aggregator.assert_event(msg_title="", msg_text="", count=0) -@pytest.mark.parametrize( - 'extra_config, expected_http_kwargs', - [ - pytest.param({'ssl_validation': True}, {'verify': True}, id="legacy ssl config True"), - pytest.param({'ssl_validation': False}, {'verify': False}, id="legacy ssl config False"), - pytest.param({}, {'verify': True}, id="legacy ssl config unset"), - ], -) -def test_config(dd_run_check, extra_config, expected_http_kwargs): - instance = deepcopy(LEGACY_REST_INSTANCE) - instance.update(extra_config) - check = TeamCityRest('teamcity', {}, [instance]) - - with patch('datadog_checks.base.utils.http.requests.Session.get') as r: - dd_run_check(check) - - http_wargs = { - 'auth': ANY, - 'cert': ANY, - 'headers': ANY, - 'proxies': ANY, - 'timeout': ANY, - 'verify': ANY, - 'allow_redirects': ANY, - } - http_wargs.update(expected_http_kwargs) - - r.assert_called_with(ANY, **http_wargs) - - @pytest.mark.parametrize( 'build_config, expected_error', [ diff --git a/teamcity/tests/test_unit.py b/teamcity/tests/test_unit.py index 20fd19a482ac6..10414201ffcb2 100644 --- a/teamcity/tests/test_unit.py +++ b/teamcity/tests/test_unit.py @@ -2,10 +2,12 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) from collections import OrderedDict +from copy import deepcopy import pytest from datadog_checks.teamcity.common import filter_build_configs, filter_items, filter_projects, normalize_server_url +from datadog_checks.teamcity.teamcity_rest import TeamCityRest from .common import ( CONFIG_ALL_BUILD_CONFIGS, @@ -18,6 +20,7 @@ CONFIG_ONLY_EXCLUDE_ONE_BUILD_CONFIG, CONFIG_ONLY_EXCLUDE_ONE_PROJECT, CONFIG_ONLY_INCLUDE_ONE_BUILD_CONFIG, + LEGACY_REST_INSTANCE, TEAMCITY_SERVER_VALUES, USE_OPENMETRICS, ) @@ -348,3 +351,20 @@ def test_filter_build_configs( filtered = filter_build_configs(check, build_configs_to_filter, 'ProjectID', {'ProjectID': filter_config}) assert filtered == expected_result + + +@pytest.mark.parametrize( + 'extra_config, expected_http_kwargs', + [ + pytest.param({'ssl_validation': True}, {'verify': True}, id="legacy ssl config True"), + pytest.param({'ssl_validation': False}, {'verify': False}, id="legacy ssl config False"), + pytest.param({}, {'verify': True}, id="legacy ssl config unset"), + ], +) +def test_config(extra_config, expected_http_kwargs): + instance = deepcopy(LEGACY_REST_INSTANCE) + instance.update(extra_config) + check = TeamCityRest('teamcity', {}, [instance]) + + for key, value in expected_http_kwargs.items(): + assert check.http.options[key] == value diff --git a/tekton/tests/common.py b/tekton/tests/common.py index 0203124213c86..a1d36404d371a 100644 --- a/tekton/tests/common.py +++ b/tekton/tests/common.py @@ -3,8 +3,8 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import os +from datadog_checks.base.utils.http_testing import MockHTTPResponse from datadog_checks.dev import get_here -from datadog_checks.dev.http import MockResponse from datadog_checks.tekton import TektonCheck HERE = get_here() @@ -152,4 +152,4 @@ def mock_http_responses(url, **_params): raise Exception(f"url `{url}` not registered") with open(os.path.join(HERE, 'fixtures', metrics_file)) as f: - return MockResponse(content=f.read()) + return MockHTTPResponse(content=f.read()) diff --git a/tekton/tests/test_unit.py b/tekton/tests/test_unit.py index 0010a2eae466d..de85f1b0857fb 100644 --- a/tekton/tests/test_unit.py +++ b/tekton/tests/test_unit.py @@ -16,8 +16,8 @@ pytest.param('triggers_instance', TRIGGERS_METRICS, 'triggers_controller', id='triggers'), ], ) -def test_check(dd_run_check, aggregator, mocker, instance, metrics, request, namespace): - mocker.patch("requests.Session.get", wraps=mock_http_responses) +def test_check(dd_run_check, aggregator, mock_http, instance, metrics, request, namespace): + mock_http.get.side_effect = mock_http_responses dd_run_check(check(request.getfixturevalue(instance))) for expected_metric in metrics: @@ -30,10 +30,10 @@ def test_check(dd_run_check, aggregator, mocker, instance, metrics, request, nam assert len(aggregator.service_check_names) == 1 -def test_invalid_url(dd_run_check, aggregator, pipelines_instance, mocker): +def test_invalid_url(dd_run_check, aggregator, pipelines_instance, mock_http): pipelines_instance["pipelines_controller_endpoint"] = "http://unknowwn" - mocker.patch("requests.Session.get", wraps=mock_http_responses) + mock_http.get.side_effect = mock_http_responses with pytest.raises(Exception): dd_run_check(check(pipelines_instance)) diff --git a/torchserve/tests/management/test_model_discovery.py b/torchserve/tests/management/test_model_discovery.py index ea17df181c441..cb02407f916e5 100644 --- a/torchserve/tests/management/test_model_discovery.py +++ b/torchserve/tests/management/test_model_discovery.py @@ -95,7 +95,7 @@ ), ], ) -def test_get_models(check, mocked_management_instance, expected_models, fixture_folder, status_codes): +def test_get_models(check, mocked_management_instance, expected_models, fixture_folder, status_codes, mock_http): # Build all the responses our mock will return responses = [] full_path = get_fixture_path(os.path.join("management", "pagination", fixture_folder)) @@ -109,21 +109,19 @@ def test_get_models(check, mocked_management_instance, expected_models, fixture_ mock_resp.raise_for_status.side_effect = HTTPError() if status_code != 200 else None responses.append(mock_resp) - req = mock.MagicMock() - with mock.patch('datadog_checks.base.utils.http.requests.Session', return_value=req): - discovery = ModelDiscovery(check(mocked_management_instance), include=[".*"]) - req.get.side_effect = responses - assert [('.*', model['modelName'], model, None) for model in expected_models] == list(discovery.get_items()) - assert req.get.call_count == len(status_codes) + mock_http.get.side_effect = responses + discovery = ModelDiscovery(check(mocked_management_instance), include=[".*"]) + assert [('.*', model['modelName'], model, None) for model in expected_models] == list(discovery.get_items()) + assert mock_http.get.call_count == len(status_codes) - # Validate we used the right params - assert req.get.call_args_list[0].kwargs["params"] == {"limit": 100} + # Validate we used the right params + assert mock_http.get.call_args_list[0].kwargs["params"] == {"limit": 100} - for index, _ in enumerate(status_codes[1:], start=1): - # The nextPageToken from the call n comes from the answer n-1 - assert req.get.call_args_list[index].kwargs["params"] == { - "limit": 100, - "nextPageToken": responses[index - 1].json.return_value["nextPageToken"], - } + for index, _ in enumerate(status_codes[1:], start=1): + # The nextPageToken from the call n comes from the answer n-1 + assert mock_http.get.call_args_list[index].kwargs["params"] == { + "limit": 100, + "nextPageToken": responses[index - 1].json.return_value["nextPageToken"], + } - assert discovery.api_status == (AgentCheck.CRITICAL if status_codes[0] != 200 else AgentCheck.OK) + assert discovery.api_status == (AgentCheck.CRITICAL if status_codes[0] != 200 else AgentCheck.OK) diff --git a/traefik_mesh/changelog.d/22676.changed b/traefik_mesh/changelog.d/22676.changed new file mode 100644 index 0000000000000..f39347eb80c40 --- /dev/null +++ b/traefik_mesh/changelog.d/22676.changed @@ -0,0 +1 @@ +Remove unused ``url`` parameter from ``get_version``. \ No newline at end of file diff --git a/traefik_mesh/datadog_checks/traefik_mesh/check.py b/traefik_mesh/datadog_checks/traefik_mesh/check.py index a1e1549f9c566..ad1394c8a9aca 100644 --- a/traefik_mesh/datadog_checks/traefik_mesh/check.py +++ b/traefik_mesh/datadog_checks/traefik_mesh/check.py @@ -8,6 +8,9 @@ import requests from datadog_checks.base import AgentCheck, OpenMetricsBaseCheckV2 +from datadog_checks.base.utils.http_exceptions import HTTPConnectionError as _HTTPConnectionError +from datadog_checks.base.utils.http_exceptions import HTTPStatusError +from datadog_checks.base.utils.http_exceptions import HTTPTimeoutError as _HTTPTimeoutError from datadog_checks.traefik_mesh.config_models import ConfigMixin from datadog_checks.traefik_mesh.metrics import METRIC_MAP, RENAME_LABELS @@ -84,7 +87,7 @@ def get_mesh_ready_status(self): return node_status - def get_version(self, url): + def get_version(self): """Fetches Traefik Proxy version from the Proxy API""" version_url = urljoin(self.traefik_proxy_api_endpoint, PROXY_VERSION) @@ -112,10 +115,15 @@ def _get_json(self, url): resp = self.http.get(url) resp.raise_for_status() return resp.json() - except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e: + except ( + requests.exceptions.HTTPError, + requests.exceptions.ConnectionError, + HTTPStatusError, + _HTTPConnectionError, + ) as e: self.warning( "Couldn't connect to URL: %s with exception: %s. Please verify the address is reachable", url, e ) - except requests.exceptions.Timeout as e: + except (requests.exceptions.Timeout, _HTTPTimeoutError) as e: self.warning("Connection timeout when connecting to %s: %s", url, e) return None diff --git a/traefik_mesh/tests/test_unit.py b/traefik_mesh/tests/test_unit.py index 6a7cb29d2507f..c6c3acbc697df 100644 --- a/traefik_mesh/tests/test_unit.py +++ b/traefik_mesh/tests/test_unit.py @@ -5,6 +5,9 @@ import pytest from datadog_checks.base.constants import ServiceCheck +from datadog_checks.base.utils.http_exceptions import HTTPConnectionError as _HTTPConnectionError +from datadog_checks.base.utils.http_exceptions import HTTPStatusError +from datadog_checks.base.utils.http_exceptions import HTTPTimeoutError as _HTTPTimeoutError from datadog_checks.dev.utils import assert_service_checks, get_metadata_metrics from datadog_checks.traefik_mesh import TraefikMeshCheck @@ -125,3 +128,21 @@ def test_submit_version(datadog_agent, dd_run_check, mock_http_response): } datadog_agent.assert_metadata('test:123', version_metadata) + + +def test_get_json_handles_http_status_error(): + check = TraefikMeshCheck('traefik_mesh', {}, [OM_MOCKED_INSTANCE]) + with mock.patch('requests.Session.get', side_effect=HTTPStatusError('404 Client Error')): + assert check._get_json('http://example.com/api') is None + + +def test_get_json_handles_http_connection_error(): + check = TraefikMeshCheck('traefik_mesh', {}, [OM_MOCKED_INSTANCE]) + with mock.patch('requests.Session.get', side_effect=_HTTPConnectionError('Connection refused')): + assert check._get_json('http://example.com/api') is None + + +def test_get_json_handles_http_timeout_error(): + check = TraefikMeshCheck('traefik_mesh', {}, [OM_MOCKED_INSTANCE]) + with mock.patch('requests.Session.get', side_effect=_HTTPTimeoutError('Read timed out')): + assert check._get_json('http://example.com/api') is None diff --git a/vault/changelog.d/22676.added b/vault/changelog.d/22676.added new file mode 100644 index 0000000000000..5bcdc39844d6b --- /dev/null +++ b/vault/changelog.d/22676.added @@ -0,0 +1 @@ +Reapply auth headers to check.http after scraper started sharing it \ No newline at end of file diff --git a/vault/datadog_checks/vault/check.py b/vault/datadog_checks/vault/check.py index 5726d1eb4bc35..2e83f811a4d2b 100644 --- a/vault/datadog_checks/vault/check.py +++ b/vault/datadog_checks/vault/check.py @@ -43,7 +43,7 @@ def __init__(self, name, init_config, instances): self.scraper_configs.clear() # https://www.vaultproject.io/api-docs#the-x-vault-request-header - self.http.options['headers']['X-Vault-Request'] = 'true' + self.http.set_header('X-Vault-Request', 'true') # Before scrapers are configured self.check_initializations.insert(-1, self.parse_config) @@ -238,8 +238,12 @@ def configure_scrapers(self): }, } } + if hasattr(self, '_http'): + del self._http + self.http.set_header('X-Vault-Request', 'true') if self.config.client_token: config['headers']['X-Vault-Token'] = self.config.client_token + self.http.set_header('X-Vault-Token', self.config.client_token) self.scraper_configs.clear() self.scraper_configs.append(config) diff --git a/vault/datadog_checks/vault/vault.py b/vault/datadog_checks/vault/vault.py index b216644712494..f5bb7c1f263d9 100644 --- a/vault/datadog_checks/vault/vault.py +++ b/vault/datadog_checks/vault/vault.py @@ -318,7 +318,7 @@ def renew_client_token(self): self.set_client_token(f.read().decode('utf-8')) def _set_header(self, http_wrapper, header, value): - http_wrapper.options['headers'][header] = value + http_wrapper.set_header(header, value) def get_scraper_config(self, instance): # This validation is called during `__init__` but we don't need it diff --git a/vllm/tests/test_unit.py b/vllm/tests/test_unit.py index f6aa9f5cca85f..1fa5454eedf65 100644 --- a/vllm/tests/test_unit.py +++ b/vllm/tests/test_unit.py @@ -81,7 +81,7 @@ def test_emits_critical_openemtrics_service_check_when_service_is_down( """ mock_http_response(status_code=404) check = vLLMCheck("vllm", {}, [instance]) - with pytest.raises(Exception, match='requests.exceptions.HTTPError'): + with pytest.raises(Exception, match='HTTPStatusError'): dd_run_check(check) aggregator.assert_all_metrics_covered()