feat!: use native Request / Response interface (#3163)

Phasing out `got-scraping`-specific interfaces in favour of native
`fetch` API.

Related to #3071

Author: barjin

docs/upgrading/upgrading_v4.md

@@ -94,3 +94,16 @@ This experimental option relied on an outdated manifest version for browser exte
 ## Available resource detection
 
 In v3, we introduced a new way to detect available resources for the crawler, available via `systemInfoV2` flag. In v4, this is the default way to detect available resources. The old way is removed completely together with the `systemInfoV2` flag.
+
+## `HttpClient` instances return `Response` objects
+
+The interface of `HttpClient` instances was changed to return the [native `Response` objects](https://developer.mozilla.org/en-US/docs/Web/API/Response) instead of custom `HttpResponse` objects.
+
+## `CrawlingContext.response` is now of type `Response`
+
+The `CrawlingContext.response` property is now of type [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) instead of `HttpResponse`. `CrawlingContext.sendRequest` method now returns `Response` objects as well.
+
+## Crawling context in the `FileDownload` crawler no longer includes `body` and `stream` properties
+
+The crawling context in the `FileDownload` crawler no longer includes the `body` and `stream` properties. These can be accessed directly via the `response` property instead, e.g. `context.response.bytes()` or `context.response.body`.
+

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -1578,7 +1578,7 @@ export class BasicCrawler<
             },
             pushData: this.pushData.bind(this),
             useState: this.useState.bind(this),
-            sendRequest: createSendRequest(this.httpClient, request!, session),
+            sendRequest: createSendRequest(this.httpClient, request!, session) as CrawlingContext['sendRequest'],
             getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName, { config: this.config }),
             registerDeferredCleanup: (cleanup) => {
                 deferredCleanup.push(cleanup);

packages/basic-crawler/src/internals/send-request.ts

@@ -5,7 +5,6 @@ import {
     type Request,
     type Session,
 } from '@crawlee/core';
-import type { Method, Response as GotResponse } from 'got-scraping';
 
 /**
  * Prepares a function to be used as the `sendRequest` context helper.
@@ -17,10 +16,7 @@ import type { Method, Response as GotResponse } from 'got-scraping';
  * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request.
  */
 export function createSendRequest(httpClient: BaseHttpClient, originRequest: Request, session: Session | undefined) {
-    return async <Response = string>(
-        // TODO the type information here (and in crawler_commons) is outright wrong... for BC - replace this with generic HttpResponse in v4
-        overrideOptions: Partial<HttpRequestOptions> = {},
-    ): Promise<GotResponse<Response>> => {
+    return async (overrideOptions: Partial<HttpRequestOptions> = {}): Promise<Response> => {
         const cookieJar = session
             ? {
                   getCookieString: async (url: string) => session.getCookieString(url),
@@ -31,7 +27,7 @@ export function createSendRequest(httpClient: BaseHttpClient, originRequest: Req
 
         const requestOptions = processHttpRequestOptions({
             url: originRequest.url,
-            method: originRequest.method as Method, // Narrow type to omit CONNECT
+            method: originRequest.method,
             headers: originRequest.headers,
             proxyUrl: session?.proxyInfo?.url,
             sessionToken: session,
@@ -43,6 +39,6 @@ export function createSendRequest(httpClient: BaseHttpClient, originRequest: Req
         // Fill in body as the last step - `processHttpRequestOptions` may use either `body`, `json` or `form` so we cannot override it beforehand
         requestOptions.body ??= originRequest.payload;
 
-        return httpClient.sendRequest<any>(requestOptions) as unknown as GotResponse<Response>;
+        return httpClient.sendRequest(requestOptions);
     };
 }

packages/core/src/cookie_utils.ts

@@ -12,16 +12,14 @@ export interface ResponseLike {
 /**
  * @internal
  */
-export function getCookiesFromResponse(response: ResponseLike): Cookie[] {
-    const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
-    const cookieHeader = headers?.['set-cookie'] || '';
+export function getCookiesFromResponse(response: Response): Cookie[] {
+    const headers = response.headers;
+    const cookieHeaders = headers.getSetCookie();
 
     try {
-        return Array.isArray(cookieHeader)
-            ? cookieHeader.map((cookie) => Cookie.parse(cookie)!)
-            : [Cookie.parse(cookieHeader)!];
+        return cookieHeaders.map((cookie) => Cookie.parse(cookie)!);
     } catch (e) {
-        throw new CookieParseError(cookieHeader);
+        throw new CookieParseError(cookieHeaders);
     }
 }
 

packages/core/src/crawlers/crawler_commons.ts

@@ -1,5 +1,5 @@
 import type { Dictionary } from '@crawlee/types';
-import type { OptionsInit, Response as GotResponse } from 'got-scraping';
+import type { OptionsInit } from 'got-scraping';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
 
 import type { Configuration } from '../configuration.js';
@@ -156,7 +156,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * },
      * ```
      */
-    sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;
+    sendRequest(overrideOptions?: Partial<OptionsInit>): Promise<Response>;
 
     /**
      * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.

packages/core/src/http_clients/base-http-client.ts

@@ -1,5 +1,6 @@
 import type { Readable } from 'node:stream';
 
+import type { AllowedHttpMethods } from '@crawlee/types';
 import { applySearchParams, type SearchParams } from '@crawlee/utils';
 
 import type { FormDataLike } from './form-data-like.js';
@@ -15,24 +16,6 @@ type Timeout =
       }
     | { request: number };
 
-type Method =
-    | 'GET'
-    | 'POST'
-    | 'PUT'
-    | 'PATCH'
-    | 'HEAD'
-    | 'DELETE'
-    | 'OPTIONS'
-    | 'TRACE'
-    | 'get'
-    | 'post'
-    | 'put'
-    | 'patch'
-    | 'head'
-    | 'delete'
-    | 'options'
-    | 'trace';
-
 /**
  * Maps permitted values of the `responseType` option on {@apilink HttpRequest} to the types that they produce.
  */
@@ -79,7 +62,7 @@ export interface HttpRequest<TResponseType extends keyof ResponseTypes = 'text'>
     [k: string]: unknown; // TODO BC with got - remove in 4.0
 
     url: string | URL;
-    method?: Method;
+    method?: AllowedHttpMethods;
     headers?: SimpleHeaders;
     body?: string | Buffer | Readable | Generator | AsyncGenerator | FormDataLike;
 
@@ -146,6 +129,14 @@ interface HttpResponseWithoutBody<TResponseType extends keyof ResponseTypes = ke
     request: HttpRequest<TResponseType>;
 }
 
+export class ResponseWithUrl extends Response {
+    override url: string;
+    constructor(body: BodyInit | null, init: ResponseInit & { url?: string }) {
+        super(body, init);
+        this.url = init.url ?? '';
+    }
+}
+
 /**
  * HTTP response data as returned by the {@apilink BaseHttpClient.sendRequest} method.
  */
@@ -169,7 +160,7 @@ export interface StreamingHttpResponse extends HttpResponseWithoutBody {
  * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
  */
 export type RedirectHandler = (
-    redirectResponse: BaseHttpResponseData,
+    redirectResponse: Response,
     updatedRequest: { url?: string | URL; headers: SimpleHeaders },
 ) => void;
 
@@ -182,12 +173,12 @@ export interface BaseHttpClient {
      */
     sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(
         request: HttpRequest<TResponseType>,
-    ): Promise<HttpResponse<TResponseType>>;
+    ): Promise<Response>;
 
     /**
      * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response.
      */
-    stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
+    stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<Response>;
 }
 
 /**

packages/core/src/http_clients/got-scraping-http-client.ts

@@ -1,25 +1,40 @@
+import { Readable } from 'node:stream';
+
 import type { Options, PlainResponse } from 'got-scraping';
 import { gotScraping } from 'got-scraping';
 
-import type {
-    BaseHttpClient,
-    HttpRequest,
-    HttpResponse,
-    RedirectHandler,
-    ResponseTypes,
-    StreamingHttpResponse,
+import {
+    type BaseHttpClient,
+    type HttpRequest,
+    type RedirectHandler,
+    type ResponseTypes,
+    ResponseWithUrl,
 } from './base-http-client.js';
 
 /**
  * A HTTP client implementation based on the `got-scraping` library.
  */
 export class GotScrapingHttpClient implements BaseHttpClient {
+    /**
+     * Type guard that validates the HTTP method (excluding CONNECT).
+     * @param request - The HTTP request to validate
+     */
+    private validateRequest<TResponseType extends keyof ResponseTypes, T extends HttpRequest<TResponseType>>(
+        request: T,
+    ): request is T & { method: Exclude<T['method'], 'CONNECT' | 'connect'> } {
+        return !['CONNECT', 'connect'].includes(request.method!);
+    }
+
     /**
      * @inheritDoc
      */
     async sendRequest<TResponseType extends keyof ResponseTypes>(
         request: HttpRequest<TResponseType>,
-    ): Promise<HttpResponse<TResponseType>> {
+    ): Promise<Response> {
+        if (!this.validateRequest(request)) {
+            throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
+        }
+
         const gotResult = await gotScraping({
             ...request,
             // `HttpCrawler` reads the cookies beforehand and sets them in `request.gotOptions`.
@@ -31,23 +46,45 @@ export class GotScrapingHttpClient implements BaseHttpClient {
             },
         });
 
-        return {
-            ...gotResult,
-            body: gotResult.body as ResponseTypes[TResponseType],
-            request: { url: request.url, ...gotResult.request },
-        };
+        const parsedHeaders = Object.entries(gotResult.headers)
+            .map(([key, value]) => {
+                if (value === undefined) return [];
+
+                if (Array.isArray(value)) {
+                    return value.map((v) => [key, v]);
+                }
+
+                return [[key, value]];
+            })
+            .flat() as [string, string][];
+
+        return new ResponseWithUrl(new Uint8Array(gotResult.rawBody), {
+            headers: new Headers(parsedHeaders),
+            status: gotResult.statusCode,
+            statusText: gotResult.statusMessage ?? '',
+            url: gotResult.url,
+        });
     }
 
     /**
      * @inheritDoc
      */
-    async stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {
+    async stream(request: HttpRequest, handleRedirect?: RedirectHandler): Promise<Response> {
+        if (!this.validateRequest(request)) {
+            throw new Error(`The HTTP method CONNECT is not supported by the GotScrapingHttpClient.`);
+        }
         // eslint-disable-next-line no-async-promise-executor
         return new Promise(async (resolve, reject) => {
             const stream = gotScraping({ ...request, isStream: true, cookieJar: undefined });
 
-            stream.on('redirect', (updatedOptions: Options, redirectResponse: PlainResponse) => {
-                handleRedirect?.(redirectResponse, updatedOptions);
+            stream.on('redirect', (updatedOptions: Options, redirectResponse: any) => {
+                const nativeRedirectResponse = new ResponseWithUrl(redirectResponse.rawBody, {
+                    headers: redirectResponse.headers,
+                    status: redirectResponse.statusCode,
+                    statusText: redirectResponse.statusMessage,
+                    url: redirectResponse.url,
+                });
+                handleRedirect?.(nativeRedirectResponse, updatedOptions);
             });
 
             // We need to end the stream for DELETE requests, otherwise it will hang.
@@ -58,37 +95,15 @@ export class GotScrapingHttpClient implements BaseHttpClient {
             stream.on('error', reject);
 
             stream.on('response', (response: PlainResponse) => {
-                const result: StreamingHttpResponse = {
-                    stream,
-                    request,
-                    redirectUrls: response.redirectUrls,
-                    url: response.url,
-                    ip: response.ip,
-                    statusCode: response.statusCode,
-                    headers: response.headers,
-                    trailers: response.trailers,
-                    complete: response.complete,
-                    get downloadProgress() {
-                        return stream.downloadProgress;
-                    },
-                    get uploadProgress() {
-                        return stream.uploadProgress;
-                    },
-                };
-
-                Object.assign(result, response); // TODO BC - remove in 4.0
-
-                resolve(result);
-
-                stream.on('end', () => {
-                    result.complete = response.complete;
-
-                    result.trailers ??= {};
-                    Object.assign(result.trailers, response.trailers);
-
-                    (result as any).rawTrailers ??= []; // TODO BC - remove in 4.0
-                    Object.assign((result as any).rawTrailers, response.rawTrailers);
-                });
+                // Cast shouldn't be needed here, undici might have a different `ReadableStream` type
+                resolve(
+                    new ResponseWithUrl(Readable.toWeb(stream) as any, {
+                        status: response.statusCode,
+                        statusText: response.statusMessage ?? '',
+                        headers: response.headers as HeadersInit,
+                        url: response.url,
+                    }),
+                );
             });
         });
     }

packages/core/src/request.ts

@@ -81,7 +81,7 @@ export enum RequestState {
  * ```
  * @category Sources
  */
-export class Request<UserData extends Dictionary = Dictionary> {
+class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     /** Request ID */
     id?: string;
 
@@ -196,7 +196,8 @@ export class Request<UserData extends Dictionary = Dictionary> {
         this.url = url;
         this.loadedUrl = loadedUrl;
         this.uniqueKey =
-            uniqueKey || Request.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+            uniqueKey ||
+            CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
@@ -259,6 +260,18 @@ export class Request<UserData extends Dictionary = Dictionary> {
         }
     }
 
+    /**
+     * Converts the Crawlee Request object to a `fetch` API Request object.
+     * @returns The native `fetch` API Request object.
+     */
+    public intoFetchAPIRequest(): Request {
+        return new Request(this.url, {
+            method: this.method,
+            headers: this.headers,
+            body: this.payload,
+        });
+    }
+
     /** Tells the crawler processing this request to skip the navigation and process the request directly. */
     get skipNavigation(): boolean {
         return this.userData.__crawlee?.skipNavigation ?? false;
@@ -419,7 +432,7 @@ export class Request<UserData extends Dictionary = Dictionary> {
             }
             return normalizedUrl;
         }
-        const payloadHash = payload ? Request.hashPayload(payload) : '';
+        const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
         return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
     }
 
@@ -561,10 +574,12 @@ interface ComputeUniqueKeyOptions {
     useExtendedUniqueKey?: boolean;
 }
 
-export type Source = (Partial<RequestOptions> & { requestsFromUrl?: string; regex?: RegExp }) | Request;
+export type Source = (Partial<RequestOptions> & { requestsFromUrl?: string; regex?: RegExp }) | CrawleeRequest;
 
 /** @internal */
 export interface InternalSource {
     requestsFromUrl: string;
     regex?: RegExp;
 }
+
+export { CrawleeRequest as Request };