Add coustomized ja3 and akamai fingerprints support (#331)

Makefile

@@ -2,7 +2,7 @@
 SHELL := bash
 
 # this is the upstream libcurl-impersonate version
-VERSION := 0.7.0b6
+VERSION := 0.7.0
 CURL_VERSION := curl-8_7_1
 
 $(CURL_VERSION):

curl_cffi/__init__.py

@@ -8,6 +8,7 @@
     "CurlMOpt",
     "CurlECode",
     "CurlHttpVersion",
+    "CurlSslVersion",
     "CurlWsFlag",
     "ffi",
     "lib",
@@ -20,5 +21,5 @@
 # This line includes _wrapper.so into the wheel
 from ._wrapper import ffi, lib
 from .aio import AsyncCurl
-from .const import CurlECode, CurlHttpVersion, CurlInfo, CurlMOpt, CurlOpt, CurlWsFlag
+from .const import CurlECode, CurlHttpVersion, CurlInfo, CurlMOpt, CurlOpt, CurlWsFlag, CurlSslVersion
 from .curl import Curl, CurlError, CurlMime

curl_cffi/const.py

@@ -328,6 +328,8 @@ class CurlOpt(IntEnum):
     TLS_EXTENSION_ORDER = 10000 + 1012
     STREAM_EXCLUSIVE = 0 + 1013
     TLS_KEY_USAGE_NO_CHECK = 0 + 1014
+    TLS_SIGNED_CERT_TIMESTAMPS = 0 + 1015
+    TLS_STATUS_REQUEST = 0 + 1016
 
     if locals().get("WRITEDATA"):
         FILE = locals().get("WRITEDATA")
@@ -563,3 +565,17 @@ class CurlWsFlag(IntEnum):
     CLOSE = 1 << 3
     PING = 1 << 4
     OFFSET = 1 << 5
+
+
+class CurlSslVersion(IntEnum):
+    """``CURL_SSLVERSION`` constants extracted from libcurl, see comments for details."""
+
+    DEFAULT = 0
+    TLSv1 = 1
+    SSLv2 = 2
+    SSLv3 = 3
+    TLSv1_0 = 4
+    TLSv1_1 = 5
+    TLSv1_2 = 6
+    TLSv1_3 = 7
+    MAX_DEFAULT = 1 << 16

curl_cffi/requests/__init__.py

@@ -19,17 +19,20 @@
     "WebSocket",
     "WebSocketError",
     "WsCloseCode",
+    "ExtraFingerprints",
 ]
 
 from functools import partial
 from io import BytesIO
 from typing import Callable, Dict, List, Optional, Tuple, Union
 
+
 from ..const import CurlHttpVersion, CurlWsFlag
 from ..curl import CurlMime
 from .cookies import Cookies, CookieTypes
 from .errors import RequestsError
 from .headers import Headers, HeaderTypes
+from .impersonate import ExtraFingerprints, ExtraFpDict
 from .models import Request, Response
 from .session import AsyncSession, BrowserType, ProxySpec, Session, ThreadType
 from .websockets import WebSocket, WebSocketError, WsCloseCode
@@ -56,6 +59,9 @@ def request(
     accept_encoding: Optional[str] = "gzip, deflate, br",
     content_callback: Optional[Callable] = None,
     impersonate: Optional[Union[str, BrowserType]] = None,
+    ja3: Optional[str] = None,
+    akamai: Optional[str] = None,
+    extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
     thread: Optional[ThreadType] = None,
     default_headers: Optional[bool] = None,
     default_encoding: Union[str, Callable[[bytes], str]] = "utf-8",
@@ -65,7 +71,7 @@ def request(
     interface: Optional[str] = None,
     cert: Optional[Union[str, Tuple[str, str]]] = None,
     stream: bool = False,
-    max_recv_speed: int = 0,    
+    max_recv_speed: int = 0,
     multipart: Optional[CurlMime] = None,
 ) -> Response:
     """Send an http request.
@@ -95,6 +101,9 @@ def request(
         content_callback: a callback function to receive response body.
             ``def callback(chunk: bytes) -> None:``
         impersonate: which browser version to impersonate.
+        ja3: ja3 string to impersonate.
+        akamai: akamai string to impersonate.
+        extra_fp: extra fingerprints options, in complement to ja3 and akamai strings.
         thread: work with other thread implementations. choices: eventlet, gevent.
         default_headers: whether to set default browser headers.
         default_encoding: encoding for decoding response content if charset is not found in headers.
@@ -130,6 +139,9 @@ def request(
             accept_encoding=accept_encoding,
             content_callback=content_callback,
             impersonate=impersonate,
+            ja3=ja3,
+            akamai=akamai,
+            extra_fp=extra_fp,
             default_headers=default_headers,
             default_encoding=default_encoding,
             http_version=http_version,

curl_cffi/requests/impersonate.py

@@ -0,0 +1,280 @@
+from dataclasses import dataclass
+from typing import List, Literal, Optional, TypedDict
+import warnings
+from enum import Enum
+
+from ..const import CurlSslVersion, CurlOpt
+
+
+class BrowserType(str, Enum):
+    edge99 = "edge99"
+    edge101 = "edge101"
+    chrome99 = "chrome99"
+    chrome100 = "chrome100"
+    chrome101 = "chrome101"
+    chrome104 = "chrome104"
+    chrome107 = "chrome107"
+    chrome110 = "chrome110"
+    chrome116 = "chrome116"
+    chrome119 = "chrome119"
+    chrome120 = "chrome120"
+    chrome123 = "chrome123"
+    chrome124 = "chrome124"
+    chrome99_android = "chrome99_android"
+    safari15_3 = "safari15_3"
+    safari15_5 = "safari15_5"
+    safari17_0 = "safari17_0"
+    safari17_2_ios = "safari17_2_ios"
+
+    chrome = "chrome124"
+    safari = "safari17_0"
+    safari_ios = "safari17_2_ios"
+
+    @classmethod
+    def has(cls, item):
+        return item in cls.__members__
+
+    @classmethod
+    def normalize(cls, item):
+        if item == "chrome":  # noqa: SIM116
+            return cls.chrome
+        elif item == "safari":
+            return cls.safari
+        elif item == "safari_ios":
+            return cls.safari_ios
+        else:
+            return item
+
+
+@dataclass
+class ExtraFingerprints:
+    tls_min_version: int = CurlSslVersion.TLSv1_2
+    tls_grease: bool = False
+    tls_permute_extensions: bool = False
+    tls_cert_compression: Literal["zlib", "brotli"] = "brotli"
+    tls_signature_algorithms: Optional[List[str]] = None
+    http2_stream_weight: int = 256
+    http2_stream_exclusive: int = 1
+
+
+class ExtraFpDict(TypedDict, total=False):
+    tls_min_version: int
+    tls_grease: bool
+    tls_permute_extensions: bool
+    tls_cert_compression: Literal["zlib", "brotli"]
+    tls_signature_algorithms: Optional[List[str]]
+    http2_stream_weight: int
+    http2_stream_exclusive: int
+
+
+# TLS version are in the format of 0xAABB, where AA is major version and BB is minor
+# version. As of today, the major version is always 03.
+TLS_VERSION_MAP = {
+    0x0301: CurlSslVersion.TLSv1_0,  # 769
+    0x0302: CurlSslVersion.TLSv1_1,  # 770
+    0x0303: CurlSslVersion.TLSv1_2,  # 771
+    0x0304: CurlSslVersion.TLSv1_3,  # 772
+}
+
+# A list of the possible cipher suite ids. Taken from
+# http://www.iana.org/assignments/tls-parameters/tls-parameters.xml
+# via BoringSSL
+TLS_CIPHER_NAME_MAP = {
+    0x000A: "TLS_RSA_WITH_3DES_EDE_CBC_SHA",
+    0x002F: "TLS_RSA_WITH_AES_128_CBC_SHA",
+    0x0035: "TLS_RSA_WITH_AES_256_CBC_SHA",
+    0x003C: "TLS_RSA_WITH_AES_128_CBC_SHA256",
+    0x003D: "TLS_RSA_WITH_AES_256_CBC_SHA256",
+    0x008C: "TLS_PSK_WITH_AES_128_CBC_SHA",
+    0x008D: "TLS_PSK_WITH_AES_256_CBC_SHA",
+    0x009C: "TLS_RSA_WITH_AES_128_GCM_SHA256",
+    0x009D: "TLS_RSA_WITH_AES_256_GCM_SHA384",
+    0xC009: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
+    0xC00A: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
+    0xC012: "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA",
+    0xC013: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
+    0xC014: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
+    0xC023: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",
+    0xC024: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384",
+    0xC027: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
+    0xC028: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
+    0xC02B: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+    0xC02C: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+    0xC02F: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+    0xC030: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+    0xC035: "TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA",
+    0xC036: "TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA",
+    0xCCA8: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
+    0xCCA9: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+    0xCCAC: "TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256",
+    0x1301: "TLS_AES_128_GCM_SHA256",
+    0x1302: "TLS_AES_256_GCM_SHA384",
+    0x1303: "TLS_CHACHA20_POLY1305_SHA256",
+}
+
+
+# RFC tls extensions: https://datatracker.ietf.org/doc/html/rfc6066
+# IANA list: https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml
+TLS_EXTENSION_NAME_MAP = {
+    0: "server_name",
+    1: "max_fragment_length",
+    2: "client_certificate_url",
+    3: "trusted_ca_keys",
+    4: "truncated_hmac",
+    5: "status_request",
+    6: "user_mapping",
+    7: "client_authz",
+    8: "server_authz",
+    9: "cert_type",
+    10: "supported_groups",  # (renamed from "elliptic_curves")
+    11: "ec_point_formats",
+    12: "srp",
+    13: "signature_algorithms",
+    14: "use_srtp",
+    15: "heartbeat",
+    16: "application_layer_protocol_negotiation",
+    17: "status_request_v2",
+    18: "signed_certificate_timestamp",
+    19: "client_certificate_type",
+    20: "server_certificate_type",
+    21: "padding",
+    22: "encrypt_then_mac",
+    23: "extended_master_secret",
+    24: "token_binding",
+    25: "cached_info",
+    26: "tls_lts",
+    27: "compress_certificate",
+    28: "record_size_limit",
+    29: "pwd_protect",
+    30: "pwd_clear",
+    31: "password_salt",
+    32: "ticket_pinning",
+    33: "tls_cert_with_extern_psk",
+    34: "delegated_credential",
+    35: "session_ticket",  # (renamed from "SessionTicket TLS")
+    36: "TLMSP",
+    37: "TLMSP_proxying",
+    38: "TLMSP_delegate",
+    39: "supported_ekt_ciphers",
+    # 40:"Reserved",
+    41: "pre_shared_key",
+    42: "early_data",
+    43: "supported_versions",
+    44: "cookie",
+    45: "psk_key_exchange_modes",
+    # 46:"Reserved",
+    47: "certificate_authorities",
+    48: "oid_filters",
+    49: "post_handshake_auth",
+    50: "signature_algorithms_cert",
+    51: "key_share",
+    52: "transparency_info",
+    # 53:"connection_id", # (deprecated)
+    54: "connection_id",
+    55: "external_id_hash",
+    56: "external_session_id",
+    57: "quic_transport_parameters",
+    58: "ticket_request",
+    59: "dnssec_chain",
+    60: "sequence_number_encryption_algorithms",
+    61: "rrc",
+    17513: "application_settings",  # BoringSSL private usage
+    # 62-2569:"Unassigned
+    # 2570:"Reserved
+    # 2571-6681:"Unassigned
+    # 6682:"Reserved
+    # 6683-10793:"Unassigned
+    # 10794:"Reserved
+    # 10795-14905:"Unassigned
+    # 14906:"Reserved
+    # 14907-19017:"Unassigned
+    # 19018:"Reserved
+    # 19019-23129:"Unassigned
+    # 23130:"Reserved
+    # 23131-27241:"Unassigned
+    # 27242:"Reserved
+    # 27243-31353:"Unassigned
+    # 31354:"Reserved
+    # 31355-35465:"Unassigned
+    # 35466:"Reserved
+    # 35467-39577:"Unassigned
+    # 39578:"Reserved
+    # 39579-43689:"Unassigned
+    # 43690:"Reserved
+    # 43691-47801:"Unassigned
+    # 47802:"Reserved
+    # 47803-51913:"Unassigned
+    # 51914:"Reserved
+    # 51915-56025:"Unassigned
+    # 56026:"Reserved
+    # 56027-60137:"Unassigned
+    # 60138:"Reserved
+    # 60139-64249:"Unassigned
+    # 64250:"Reserved
+    # 64251-64767:"Unassigned
+    64768: "ech_outer_extensions",
+    # 64769-65036:"Unassigned
+    65037:"encrypted_client_hello",
+    # 65038-65279:"Unassigned
+    # 65280:"Reserved for Private Use
+    65281:"renegotiation_info",
+    # 65282-65535:"Reserved for Private Use
+}
+
+
+TLS_EC_CURVES_MAP = {
+    19: "P-192",
+    21: "P-224",
+    23: "P-256",
+    24: "P-384",
+    25: "P-521",
+    29: "X25519",
+    25497: "X25519Kyber768Draft00",
+}
+
+
+def toggle_extension(curl, extension_id: int, enable: bool):
+    # ECH
+    if extension_id == 65037:
+        if enable:
+            curl.setopt(CurlOpt.ECH, "GREASE")
+        else:
+            curl.setopt(CurlOpt.ECH, "")
+    # compress certificate
+    elif extension_id == 27:
+        if enable:
+            warnings.warn("Cert compression setting to brotli, you had better specify which to use: zlib/brotli")
+            curl.setopt(CurlOpt.SSL_CERT_COMPRESSION, "brotli")
+        else:
+            curl.setopt(CurlOpt.SSL_CERT_COMPRESSION, "")
+    # ALPS: application settings
+    elif extension_id == 17513:
+        if enable:
+            curl.setopt(CurlOpt.SSL_ENABLE_ALPS, 1)
+        else:
+            curl.setopt(CurlOpt.SSL_ENABLE_ALPS, 0)
+    # server_name
+    elif extension_id == 0:
+        raise NotImplementedError("It's unlikely that the server_name(0) extension being changed.")
+    # ALPN
+    elif extension_id == 16:
+        raise NotImplementedError("It's unlikely that the ALPN(16) extension being changed.")
+    # status_request
+    elif extension_id == 5:
+        if enable:
+            curl.setopt(CurlOpt.TLS_STATUS_REQUEST, 1)
+    # signed_certificate_timestamps
+    elif extension_id == 18:
+        if enable:
+            curl.setopt(CurlOpt.TLS_SIGNED_CERT_TIMESTAMPS, 1)
+    # session_ticket
+    elif extension_id == 35:
+        if enable:
+            curl.setopt(CurlOpt.SSL_ENABLE_TICKET, 1)
+        else:
+            curl.setopt(CurlOpt.SSL_ENABLE_TICKET, 0)
+    # padding
+    elif extension_id == 21:
+        pass
+    else:
+        raise NotImplementedError(f"This extension({extension_id}) can not be toggled for now, it may be updated later.")