diff --git a/datadog_checks_base/changelog.d/22704.added.1 b/datadog_checks_base/changelog.d/22704.added.1 new file mode 100644 index 0000000000000..9a04d91e6705a --- /dev/null +++ b/datadog_checks_base/changelog.d/22704.added.1 @@ -0,0 +1 @@ +Forward instance config (auth, TLS, timeout, proxy, headers) to HTTPXWrapper. \ No newline at end of file diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index 7757cedda888c..db6de7e1eed7e 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -387,11 +387,14 @@ def http(self) -> HTTPClientProtocol: if not hasattr(self, '_http'): # See Performance Optimizations in this package's README.md. if is_affirmative((self.instance or {}).get('use_httpx', False)): - import httpx - from datadog_checks.base.utils.http_httpx import HTTPXWrapper - self._http = HTTPXWrapper(httpx.Client()) + self._http = HTTPXWrapper( + self.instance or {}, + self.init_config, + self.HTTP_CONFIG_REMAPPER, + self.log, + ) else: from datadog_checks.base.utils.http import RequestsWrapper diff --git a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py index 59b3e44d97242..58595d9af8539 100644 --- a/datadog_checks_base/datadog_checks/base/utils/http_httpx.py +++ b/datadog_checks_base/datadog_checks/base/utils/http_httpx.py @@ -3,10 +3,14 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations +import logging from typing import Any, Iterator import httpx +from datadog_checks.base.config import is_affirmative +from datadog_checks.base.utils.headers import get_default_headers, update_headers + from .http_exceptions import ( HTTPConnectionError, HTTPError, @@ -15,6 +19,181 @@ HTTPTimeoutError, ) +LOGGER = logging.getLogger(__name__) + +DEFAULT_TIMEOUT = 10 + +# Fields recognized from instance/init_config for httpx client construction +_STANDARD_FIELDS = { + 'allow_redirects': True, + 'auth_type': 'basic', + 'connect_timeout': None, + 'extra_headers': None, + 'headers': None, + 'kerberos_auth': None, + 'kerberos_delegate': False, + 'kerberos_force_initiate': False, + 'kerberos_hostname': None, + 'kerberos_keytab': None, + 'kerberos_principal': None, + 'ntlm_domain': None, + 'password': None, + 'proxy': None, + 'read_timeout': None, + 'skip_proxy': False, + 'timeout': DEFAULT_TIMEOUT, + 'tls_ca_cert': None, + 'tls_cert': None, + 'tls_private_key': None, + 'tls_verify': True, + 'username': None, +} + +# Legacy field aliases applied before reading standard fields +_DEFAULT_REMAPPED_FIELDS = { + 'kerberos': {'name': 'kerberos_auth'}, + 'no_proxy': {'name': 'skip_proxy'}, +} + + +def _build_httpx_client( + instance: dict, + init_config: dict, + remapper: dict | None = None, + logger: logging.Logger | None = None, +) -> httpx.Client: + log = logger or LOGGER + + # Merge default fields; init_config provides global overrides + default_fields = dict(_STANDARD_FIELDS) + default_fields['skip_proxy'] = init_config.get('skip_proxy', default_fields['skip_proxy']) + default_fields['timeout'] = init_config.get('timeout', default_fields['timeout']) + + # Populate config from instance, using defaults for missing fields + config = {field: instance.get(field, value) for field, value in default_fields.items()} + + # Apply remapper: normalize legacy/integration-specific field names + if remapper is None: + remapper = {} + remapper.update(_DEFAULT_REMAPPED_FIELDS) + + for remapped_field, data in remapper.items(): + field = data.get('name') + if field not in _STANDARD_FIELDS: + continue + # Standard field already explicitly set — skip remapped alias + if field in instance: + continue + default = default_fields[field] + if data.get('invert'): + default = not default + value = instance.get(remapped_field, data.get('default', default)) + if data.get('invert'): + value = not is_affirmative(value) + config[field] = value + + # --- Timeouts --- + connect_timeout = read_timeout = float(config['timeout']) + if config['connect_timeout'] is not None: + connect_timeout = float(config['connect_timeout']) + if config['read_timeout'] is not None: + read_timeout = float(config['read_timeout']) + # read_timeout is the default; connect overrides the connect-phase only + timeout = httpx.Timeout(read_timeout, connect=connect_timeout) + + # --- Headers --- + headers = get_default_headers() + if config['headers']: + headers.clear() + update_headers(headers, config['headers']) + if config['extra_headers']: + update_headers(headers, config['extra_headers']) + + # --- Auth --- + auth_type = (config['auth_type'] or 'basic').lower() + if auth_type == 'basic': + if config['kerberos_auth']: + log.warning( + 'The ability to use Kerberos auth without explicitly setting auth_type to ' + '`kerberos` is deprecated and will be removed in Agent 8' + ) + auth_type = 'kerberos' + elif config['ntlm_domain']: + log.warning( + 'The ability to use NTLM auth without explicitly setting auth_type to ' + '`ntlm` is deprecated and will be removed in Agent 8' + ) + auth_type = 'ntlm' + + auth: httpx.Auth | None = None + if auth_type == 'basic': + if config['username'] is not None: + auth = httpx.BasicAuth(config['username'], config['password'] or '') + elif auth_type == 'digest': + if config['username'] is not None: + auth = httpx.DigestAuth(config['username'], config['password'] or '') + elif auth_type == 'kerberos': + from datadog_checks.base.utils.httpx_auth import KerberosAuth + + auth = KerberosAuth( + mutual_authentication=config.get('kerberos_auth') or 'required', + delegate=is_affirmative(config['kerberos_delegate']), + force_preemptive=is_affirmative(config['kerberos_force_initiate']), + hostname_override=config['kerberos_hostname'], + principal=config['kerberos_principal'], + keytab=config['kerberos_keytab'], + ) + elif auth_type == 'ntlm': + from datadog_checks.base.utils.httpx_auth import NTLMAuth + + auth = NTLMAuth(config['ntlm_domain'], config['password']) + + # --- TLS / verify --- + verify: bool | str = True + if isinstance(config['tls_ca_cert'], str): + verify = config['tls_ca_cert'] + elif not is_affirmative(config['tls_verify']): + verify = False + + cert: tuple[str, str] | str | None = None + if isinstance(config['tls_cert'], str): + if isinstance(config['tls_private_key'], str): + cert = (config['tls_cert'], config['tls_private_key']) + else: + cert = config['tls_cert'] + + # --- Proxies --- + # trust_env=True lets httpx fall back to HTTP_PROXY/HTTPS_PROXY env vars (same as requests) + trust_env = True + mounts: dict[str, httpx.BaseTransport | None] | None = None + + if is_affirmative(config['skip_proxy']): + trust_env = False + else: + raw_proxy = config['proxy'] or init_config.get('proxy') + if raw_proxy: + mounts = {} + for scheme, url in raw_proxy.items(): + # 'no_proxy' entries are not proxy URLs — skip them + if scheme == 'no_proxy' or not url: + continue + # Convert requests format {'http': url} to httpx mount format {'http://': transport} + key = scheme if scheme.endswith('://') else f'{scheme}://' + mounts[key] = httpx.HTTPTransport(proxy=url) + if not mounts: + mounts = None + + return httpx.Client( + auth=auth, + verify=verify, + cert=cert, + timeout=timeout, + headers=headers, + follow_redirects=is_affirmative(config['allow_redirects']), + mounts=mounts, + trust_env=trust_env, + ) + def _translate_httpx_error(e: httpx.HTTPError) -> HTTPError: if isinstance(e, httpx.HTTPStatusError): @@ -65,8 +244,14 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: class HTTPXWrapper: - def __init__(self, client: httpx.Client) -> None: - self._client = client + def __init__( + self, + instance: dict, + init_config: dict, + remapper: dict | None = None, + logger: logging.Logger | None = None, + ) -> None: + self._client = _build_httpx_client(instance, init_config, remapper, logger) def __del__(self) -> None: # no cov try: diff --git a/datadog_checks_base/tests/base/utils/http/test_http_backend_equivalence.py b/datadog_checks_base/tests/base/utils/http/test_http_backend_equivalence.py index 11b213c46799c..267613f0b759b 100644 --- a/datadog_checks_base/tests/base/utils/http/test_http_backend_equivalence.py +++ b/datadog_checks_base/tests/base/utils/http/test_http_backend_equivalence.py @@ -40,7 +40,11 @@ def http_client(request): with patch.object(requests.Session, "get", return_value=_requests_response(_BODY)): yield RequestsWrapper({}, {}) else: - yield HTTPXWrapper(httpx.Client(transport=_httpx_transport(_BODY))) + with patch( + 'datadog_checks.base.utils.http_httpx._build_httpx_client', + return_value=httpx.Client(transport=_httpx_transport(_BODY)), + ): + yield HTTPXWrapper({}, {}) def test_status_code(http_client): diff --git a/datadog_checks_base/tests/base/utils/http/test_http_httpx.py b/datadog_checks_base/tests/base/utils/http/test_http_httpx.py index 279d0ff32185e..041197c8faaa2 100644 --- a/datadog_checks_base/tests/base/utils/http/test_http_httpx.py +++ b/datadog_checks_base/tests/base/utils/http/test_http_httpx.py @@ -1,7 +1,7 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import httpx import pytest @@ -12,7 +12,7 @@ HTTPStatusError, HTTPTimeoutError, ) -from datadog_checks.base.utils.http_httpx import HTTPXResponseAdapter, HTTPXWrapper +from datadog_checks.base.utils.http_httpx import HTTPXResponseAdapter, HTTPXWrapper, _build_httpx_client class TestHTTPXResponseAdapter: @@ -73,28 +73,34 @@ def test_response_attributes_accessible(self): class TestHTTPXWrapper: def test_successful_request_returns_response_adapter(self): - client = MagicMock(spec=httpx.Client) - client.request.return_value = MagicMock(spec=httpx.Response) - wrapper = HTTPXWrapper(client) + mock_client = MagicMock(spec=httpx.Client) + mock_client.request.return_value = MagicMock(spec=httpx.Response) + + with patch('datadog_checks.base.utils.http_httpx._build_httpx_client', return_value=mock_client): + wrapper = HTTPXWrapper({}, {}) result = wrapper.get("http://example.com") assert isinstance(result, HTTPXResponseAdapter) def test_timeout_raises_http_timeout_error(self): - client = MagicMock(spec=httpx.Client) + mock_client = MagicMock(spec=httpx.Client) request = httpx.Request("GET", "http://example.com") - client.request.side_effect = httpx.TimeoutException("timed out", request=request) - wrapper = HTTPXWrapper(client) + mock_client.request.side_effect = httpx.TimeoutException("timed out", request=request) + + with patch('datadog_checks.base.utils.http_httpx._build_httpx_client', return_value=mock_client): + wrapper = HTTPXWrapper({}, {}) with pytest.raises(HTTPTimeoutError): wrapper.get("http://example.com") def test_connect_error_raises_http_connection_error(self): - client = MagicMock(spec=httpx.Client) + mock_client = MagicMock(spec=httpx.Client) request = httpx.Request("GET", "http://example.com") - client.request.side_effect = httpx.ConnectError("connection refused", request=request) - wrapper = HTTPXWrapper(client) + mock_client.request.side_effect = httpx.ConnectError("connection refused", request=request) + + with patch('datadog_checks.base.utils.http_httpx._build_httpx_client', return_value=mock_client): + wrapper = HTTPXWrapper({}, {}) with pytest.raises(HTTPConnectionError): wrapper.get("http://example.com") @@ -108,9 +114,11 @@ def test_invalid_url_raises_http_request_error(self): wrapper.get("not a url") def test_all_http_methods_delegate_to_client(self): - client = MagicMock(spec=httpx.Client) - client.request.return_value = MagicMock(spec=httpx.Response) - wrapper = HTTPXWrapper(client) + mock_client = MagicMock(spec=httpx.Client) + mock_client.request.return_value = MagicMock(spec=httpx.Response) + + with patch('datadog_checks.base.utils.http_httpx._build_httpx_client', return_value=mock_client): + wrapper = HTTPXWrapper({}, {}) url = "http://example.com" wrapper.get(url) @@ -121,5 +129,131 @@ def test_all_http_methods_delegate_to_client(self): wrapper.delete(url) wrapper.options_method(url) - methods = [call.args[0] for call in client.request.call_args_list] + methods = [call.args[0] for call in mock_client.request.call_args_list] assert methods == ["GET", "POST", "HEAD", "PUT", "PATCH", "DELETE", "OPTIONS"] + + +class TestBuildHttpxClient: + @staticmethod + def _get_client_kwargs(instance, init_config=None, **kwargs): + """Capture the kwargs passed to httpx.Client() without constructing a real client. + + Uses mock.patch so the real httpx.Client is not instantiated, allowing tests to + inspect constructor arguments (e.g. 'verify') that aren't exposed as attributes. + """ + if init_config is None: + init_config = {} + with patch('datadog_checks.base.utils.http_httpx.httpx.Client') as mock_cls: + mock_cls.return_value = MagicMock() + _build_httpx_client(instance, init_config, **kwargs) + return mock_cls.call_args.kwargs + + # --- Auth --- + + def test_basic_auth_sets_client_auth(self): + client = _build_httpx_client({'username': 'user', 'password': 'pass'}, {}) + assert isinstance(client.auth, httpx.BasicAuth) + + def test_basic_auth_not_set_without_username(self): + client = _build_httpx_client({}, {}) + assert client.auth is None + + def test_digest_auth(self): + client = _build_httpx_client({'username': 'u', 'password': 'p', 'auth_type': 'digest'}, {}) + assert isinstance(client.auth, httpx.DigestAuth) + + def test_kerberos_auth_type_builds_kerberos_adapter(self): + from datadog_checks.base.utils.httpx_auth import KerberosAuth + + client = _build_httpx_client({'auth_type': 'kerberos'}, {}) + assert isinstance(client.auth, KerberosAuth) + + def test_ntlm_auth_type_builds_ntlm_adapter(self): + from datadog_checks.base.utils.httpx_auth import NTLMAuth + + client = _build_httpx_client({'auth_type': 'ntlm', 'ntlm_domain': 'DOMAIN\\user', 'password': 'pass'}, {}) + assert isinstance(client.auth, NTLMAuth) + + # --- TLS --- + + def test_tls_verify_false(self): + kwargs = self._get_client_kwargs({'tls_verify': False}) + assert kwargs['verify'] is False + + def test_tls_ca_cert(self): + # _get_client_kwargs avoids real SSL context loading + kwargs = self._get_client_kwargs({'tls_ca_cert': '/path/to/ca.pem'}) + assert kwargs['verify'] == '/path/to/ca.pem' + + def test_tls_verify_true_by_default(self): + kwargs = self._get_client_kwargs({}) + assert kwargs['verify'] is True + + # --- Redirects --- + + def test_follow_redirects_default_true(self): + client = _build_httpx_client({}, {}) + assert client.follow_redirects is True + + def test_follow_redirects_false(self): + client = _build_httpx_client({'allow_redirects': False}, {}) + assert client.follow_redirects is False + + # --- Timeouts --- + + def test_timeout_from_instance(self): + client = _build_httpx_client({'timeout': 30}, {}) + assert client.timeout.read == 30.0 + assert client.timeout.connect == 30.0 + + def test_timeout_read_connect_split(self): + client = _build_httpx_client({'read_timeout': 20, 'connect_timeout': 5}, {}) + assert client.timeout.read == 20.0 + assert client.timeout.connect == 5.0 + + def test_timeout_from_init_config(self): + client = _build_httpx_client({}, {'timeout': 42}) + assert client.timeout.read == 42.0 + + # --- Remapper --- + + def test_remapper_applied(self): + # 'user' should be remapped to 'username' via remapper dict + remapper = {'user': {'name': 'username'}} + client = _build_httpx_client({'user': 'alice', 'password': 'secret'}, {}, remapper=remapper) + assert isinstance(client.auth, httpx.BasicAuth) + + def test_remapper_invert(self): + # ssl_validation=False with invert=True → tls_verify=True (not disabled) + remapper = {'ssl_validation': {'name': 'tls_verify', 'default': False, 'invert': True}} + kwargs_false = self._get_client_kwargs({'ssl_validation': False}, remapper=remapper) + assert kwargs_false['verify'] is True + + kwargs_true = self._get_client_kwargs({'ssl_validation': True}, remapper=remapper) + assert kwargs_true['verify'] is False + + # --- Proxies --- + + def test_skip_proxy_disables_trust_env(self): + client = _build_httpx_client({'skip_proxy': True}, {}) + assert client.trust_env is False + + def test_proxy_config_converted_to_mounts(self): + proxy_url = 'http://proxy.example.com:8080' + kwargs = self._get_client_kwargs({'proxy': {'http': proxy_url}}) + mounts = kwargs.get('mounts') or {} + assert 'http://' in mounts + + # --- Headers --- + + def test_headers_replaced_when_headers_set(self): + # When 'headers' is set, it replaces our default Datadog headers; + # the custom header must be present. + client = _build_httpx_client({'headers': {'X-Custom': 'value'}}, {}) + assert client.headers.get('x-custom') == 'value' + + def test_extra_headers_merged(self): + # 'extra_headers' are merged on top of defaults; Datadog User-Agent still present + client = _build_httpx_client({'extra_headers': {'X-Extra': 'yes'}}, {}) + assert client.headers.get('x-extra') == 'yes' + assert 'user-agent' in client.headers diff --git a/nginx/tests/test_unit.py b/nginx/tests/test_unit.py index 94bbb5493c1a6..ecaa2e0f85102 100644 --- a/nginx/tests/test_unit.py +++ b/nginx/tests/test_unit.py @@ -88,6 +88,43 @@ def test_config(check, instance, test_case, extra_config, expected_http_kwargs): r.get.assert_called_with('http://localhost:8080/nginx_status', **http_wargs) +@pytest.mark.parametrize( + 'test_case, extra_config, expected', + [ + ("legacy auth config", {'user': 'legacy_foo', 'password': 'legacy_bar'}, {'auth_is_basic': True}), + ("new auth config", {'username': 'new_foo', 'password': 'new_bar'}, {'auth_is_basic': True}), + ("legacy ssl config True", {'ssl_validation': True}, {'verify': True}), + ("legacy ssl config False", {'ssl_validation': False}, {'verify': False}), + ], +) +def test_config_httpx(check, instance, test_case, extra_config, expected): + import httpx as _httpx + + instance = deepcopy(instance) + instance.update(extra_config) + instance['use_httpx'] = True + + c = check(instance) + + client_kwargs = {} + + def spy_httpx_client(**kwargs): + client_kwargs.update(kwargs) + return mock.MagicMock() + + with mock.patch('datadog_checks.base.utils.http_httpx.httpx.Client', side_effect=spy_httpx_client): + _ = c.http + + if expected.get('auth_is_basic'): + assert isinstance(client_kwargs.get('auth'), _httpx.BasicAuth), ( + f"{test_case}: expected BasicAuth, got {client_kwargs.get('auth')!r}" + ) + if 'verify' in expected: + assert client_kwargs.get('verify') == expected['verify'], ( + f"{test_case}: verify expected {expected['verify']!r}, got {client_kwargs.get('verify')!r}" + ) + + def test_no_version(check, instance, caplog): c = check(instance)