diff --git a/README.md b/README.md index 1ddbb1b..955a3ab 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,12 @@ SDK to access [Zenrows](https://www.zenrows.com/) API directly from Node.js. Zen - [Batch](#batch) - [Failed runs and extensible values](#failed-runs-and-extensible-values) - [Extract in a batch](#extract-in-a-batch) + - [Crawl](#crawl) - [Concurrency](#concurrency) - [An important note about Promise.allSettled() on TypeScript](#an-important-note-about-promiseallsettled-on-typescript) - [Examples](#examples) - [Contributing](#contributing) + - [Running the Crawl e2e test](#running-the-crawl-e2e-test) - [License](#license) ## Installation @@ -265,6 +267,63 @@ const { ZenRowsBatchClient } = require("zenrows"); const batch = new ZenRowsBatchClient(apiKey, { baseURL: "https://async.api.zenrows.com/v1" }); // baseURL is optional ``` +### Crawl + +[Crawl](https://docs.zenrows.com) takes one start URL and returns the URLs behind it, and is reachable via `client.crawl`. It follows the links on each page up to `depth` hops, staying on the start URL's domain. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it ends, and `iterResults()` reads the URLs it kept. With `outputFormat: "html"` it also fetches each kept page, which you read with `getContent()` or all at once with `download()`. + +```javascript +const { ZenRows } = require("zenrows"); + +const apiKey = "YOUR-API-KEY"; + +(async () => { + const client = new ZenRows(apiKey); + + const crawl = await client.crawl.create({ + url: "https://example.com/products/", + depth: 1, // link hops from the start URL + maxItems: 10, // stop once this many URLs are kept + maxPages: 20, // stop once this many pages are fetched; bounds the cost + includePatterns: ["/product/"], // keep only URLs containing one of these + outputFormat: "html", // omit for URLs only + }); + + const ended = await client.crawl.wait(crawl.crawl_id); // completed, stopped or failed + if (ended.status === "failed") throw new Error(ended.error?.detail); + + for await (const result of client.crawl.iterResults(crawl.crawl_id)) { + console.log(result.url); + if (result.content_status === "fetched") { + const html = await client.crawl.getContent(crawl.crawl_id, result); + } + } + + // Or every result, with its page, as parsed NDJSON lines. + for await (const line of client.crawl.download(crawl.crawl_id)) { + console.log(line.url, line.content?.length); + } +})(); +``` + +`client.crawl` also exposes `get()` (status, coverage and one page of results), `list()` and `iterCrawls()` (the account's crawls, newest first), and `stop()` (idempotent; a crawl that already ended answers as it ended). `create()` takes an `idempotencyKey` option, so a retried create does not start a second crawl. Read results after `wait()`: while a crawl runs, `iterResults()` yields only what it has kept so far. `wait()` throws `WaiterTimeoutError` after `timeout` seconds (default 300) and leaves the crawl running. + +Errors throw a `ZenRowsCrawlError` with `status`, `code` and `problem`. Branch on `status` and `code`: + +- `403` `REQS008`: Crawl is not enabled for this account. +- `404` `crawl_not_found` / `content_not_found`. +- `422` `invalid_parameter` / `invalid_start_url`: `problem.detail` names the field. +- `429` `too_many_crawls`: the account has too many crawls running. Nothing was created; retry after `error.retryAfter` seconds. + +`outputFormat` takes `"html"`; without it a crawl returns URLs only. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A failed crawl carries `error.code` and `error.detail`. + +The crawl client (`ZenRowsCrawlClient`) also works standalone: + +```javascript +const { ZenRowsCrawlClient } = require("zenrows"); + +const crawl = new ZenRowsCrawlClient(apiKey, { baseURL: "https://api.zenrows.com/v1" }); // baseURL is optional +``` + ### Concurrency To limit the concurrency, it uses [fastq](https://github.com/mcollina/fastq), which will simultaneously send a maximum of requests. The concurrency is determined by the plan you are in, so take a look at the [pricing](https://www.zenrows.com/pricing) and set it accordingly. Take into account that each client instance will have its own limit, meaning that two different scripts will not share it, and 429 (Too Many Requests) errors might arise. @@ -340,6 +399,27 @@ npx tsx index.ts # TS example Pull requests are welcome. For significant changes, please open an issue first to discuss what you would like to change. +Before opening a pull request, run `pnpm check` and `pnpm test -- --run`. Neither touches the network. + +### Running the Crawl e2e test + +`tests/e2e/crawl.e2e.test.ts` runs a small real crawl (depth 1, up to 3 items and 5 pages, HTML output) and checks every Crawl method against the live API. It is skipped unless the first three variables are set: + +- `ZENROWS_API_KEY`: a key with Crawl access. Keep it out of your shell history, e.g. read it from a file. +- `ZENROWS_CRAWL_BASE_URL`: the API base, e.g. `https://api.zenrows.com/v1`. Point it at a local or staging deployment to test that instead. +- `ZENROWS_E2E_CRAWL_URL`: the start URL to crawl, a site you may crawl whose start page links to at least one page. +- `ZENROWS_E2E_CRAWL_INCLUDE` (optional): an include pattern, e.g. `/product/`. When set, the crawl keeps only matching URLs and the test asserts every result contains it. + +```bash +export ZENROWS_API_KEY="$(cat path/to/key-file)" +export ZENROWS_CRAWL_BASE_URL=https://api.zenrows.com/v1 +export ZENROWS_E2E_CRAWL_URL=https://example.com/products/ +export ZENROWS_E2E_CRAWL_INCLUDE=/product/ +pnpm test:e2e +``` + +The run creates one crawl on the account and uses up to a few pages of credits. If the account has too many crawls running, the test waits and retries (up to 5 minutes). + ## License [MIT](./LICENSE) diff --git a/package.json b/package.json index 81873eb..76793e7 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "scripts": { "build": "tsup src/index.ts --dts --format cjs,esm --clean", "test": "vitest", + "test:e2e": "vitest run tests/e2e", "format": "biome format --write .", "check": "biome check ." }, diff --git a/src/batch/transport.ts b/src/batch/transport.ts index d8328dd..02e7360 100644 --- a/src/batch/transport.ts +++ b/src/batch/transport.ts @@ -1,5 +1,6 @@ import packageJson from "../../package.json" with { type: "json" }; import { ZenRowsBatchError, parseProblem } from "./errors.js"; +import type { ProblemJson } from "./types.js"; const RETRYABLE_STATUSES = new Set([429, 502, 503, 504]); const IDEMPOTENT_METHODS = new Set(["GET", "PUT", "DELETE", "HEAD", "OPTIONS"]); @@ -33,6 +34,16 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** Builds the error thrown for a non-2xx response. Defaults to `ZenRowsBatchError`. */ +export type ErrorFactory = ( + response: Response, + problem: ProblemJson | undefined, + extras: Record | undefined, +) => Error; + +const batchError: ErrorFactory = (response, problem, extras) => + new ZenRowsBatchError(response.status, problem, extras); + export interface RequestOptions { query?: object; body?: unknown; @@ -51,14 +62,15 @@ export class BatchTransport { private readonly baseURL: string, private readonly apiKey: string, private readonly retries: number = DEFAULT_RETRIES, + private readonly errorFactory: ErrorFactory = batchError, ) {} - /** Send a request, parse the response, throw `ZenRowsBatchError` on non-2xx. */ + /** Send a request, parse the response, throw the factory's error on non-2xx. */ async requestJson(method: string, path: string, options: RequestOptions = {}): Promise { const response = await this.send(method, path, options); if (!response.ok) { const { problem, extras } = await parseProblem(response); - throw new ZenRowsBatchError(response.status, problem, extras); + throw this.errorFactory(response, problem, extras); } const text = await response.text(); if (!text) { @@ -72,7 +84,7 @@ export class BatchTransport { const response = await this.send(method, path, options); if (!response.ok) { const { problem, extras } = await parseProblem(response); - throw new ZenRowsBatchError(response.status, problem, extras); + throw this.errorFactory(response, problem, extras); } return response; } diff --git a/src/crawl/client.ts b/src/crawl/client.ts new file mode 100644 index 0000000..855d5f9 --- /dev/null +++ b/src/crawl/client.ts @@ -0,0 +1,207 @@ +import { BatchTransport } from "../batch/transport.js"; +import { pollUntil } from "../batch/waiters.js"; +import { crawlError } from "./errors.js"; +import type { + Crawl, + CrawlExportLine, + CrawlList, + CrawlOutputFormat, + CrawlResult, + CrawlStop, + CrawlWithResults, +} from "./types.js"; + +const DEFAULT_CRAWL_API_URL = "https://api.zenrows.com/v1"; + +export interface CrawlClientConfig { + /** Override the Crawl API base URL. Default `https://api.zenrows.com/v1`. */ + baseURL?: string; + /** Retries for transient failures (429/502/503/504 + network errors) on idempotent requests. Default 3. */ + retries?: number; +} + +export interface CreateCrawlOptions { + /** The page the crawl starts from: an absolute public `http`/`https` URL. */ + url: string; + /** Link hops to follow from the start URL, 1–100000. */ + depth: number; + /** Stop once this many URLs are kept. Server default 10. */ + maxItems?: number; + /** Stop once this many pages are fetched; bounds the cost. Server default 10. */ + maxPages?: number; + /** Keep a URL only if it contains at least one of these substrings. */ + includePatterns?: string[]; + /** Drop a URL that contains any of these substrings, even if it matches an include pattern. */ + excludePatterns?: string[]; + /** Also fetch each kept URL's page. Absent means URLs only. */ + outputFormat?: CrawlOutputFormat; +} + +export interface WaitForCrawlOptions { + /** Seconds before `WaiterTimeoutError`. Default 300. The crawl keeps running on timeout. */ + timeout?: number; + /** Seconds before the first re-poll. Default 2; grows ×1.5 per poll. */ + pollInterval?: number; + /** Cap on the poll interval in seconds. Default 15. */ + maxPollInterval?: number; +} + +function contentIdOf(content: string | CrawlResult): string { + if (typeof content === "string") return content; + if (!content.content_url) { + throw new Error( + `getContent: result for ${content.url} has no content_url (content_status: ${content.content_status ?? "absent"})`, + ); + } + const id = content.content_url.split("/").pop(); + if (!id) throw new Error(`getContent: cannot read a content id from ${content.content_url}`); + return id; +} + +/** + * Client for the Zenrows Crawl API: give it one start URL, read back the URLs behind it. + * Usable standalone (`new ZenRowsCrawlClient(apiKey)`) or via `client.crawl` on a `ZenRows` + * instance. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it + * ends, and `iterResults()` / `download()` read what it kept. + */ +export class ZenRowsCrawlClient { + readonly apiKey: string; + private readonly transport: BatchTransport; + + constructor(apiKey: string, config: CrawlClientConfig = {}) { + this.apiKey = apiKey; + this.transport = new BatchTransport( + (config.baseURL ?? DEFAULT_CRAWL_API_URL).replace(/\/+$/, ""), + apiKey, + config.retries, + crawlError, + ); + } + + /** + * Start a crawl. Sends only the fields you set. With an `idempotencyKey`, a retry returns the + * crawl the first request created instead of starting another, and transient failures are + * retried. A 429 `too_many_crawls` (the account has too many crawls running) throws + * `ZenRowsCrawlError` with `retryAfter` set. + */ + create(options: CreateCrawlOptions, opts: { idempotencyKey?: string } = {}): Promise { + const body = { + url: options.url, + depth: options.depth, + max_items: options.maxItems, + max_pages: options.maxPages, + include_patterns: options.includePatterns, + exclude_patterns: options.excludePatterns, + output_format: options.outputFormat, + }; + return this.transport.requestJson("POST", "/crawls", { + body, + idempotencyKey: opts.idempotencyKey, + }); + } + + /** Read a crawl's status, coverage and one page of results (`limit` 1–10000, default 1000). */ + get( + crawlId: string, + options: { cursor?: string; limit?: number } = {}, + ): Promise { + return this.transport.requestJson("GET", `/crawls/${encodeURIComponent(crawlId)}`, { + query: options, + }); + } + + /** One page of the account's crawls, newest first (`limit` 1–100, default 20). */ + list(options: { cursor?: string; limit?: number } = {}): Promise { + return this.transport.requestJson("GET", "/crawls", { query: options }); + } + + /** Every crawl of the account, newest first, following `next_cursor`. */ + async *iterCrawls(options: { limit?: number } = {}): AsyncGenerator { + let cursor: string | undefined; + while (true) { + const page = await this.list({ ...options, cursor }); + yield* page.crawls; + cursor = page.next_cursor; + if (!cursor) return; + } + } + + /** + * Every URL the crawl kept, following `next_cursor` until it is null. Call it once the crawl + * has ended (see `wait()`): on a running crawl it yields the URLs kept so far and returns. + */ + async *iterResults( + crawlId: string, + options: { limit?: number } = {}, + ): AsyncGenerator { + let cursor: string | undefined; + while (true) { + const page = await this.get(crawlId, { ...options, cursor }); + yield* page.results; + // A running crawl's cursor is never null; an empty page means nothing more is kept yet. + if (page.next_cursor === null || page.results.length === 0) return; + cursor = page.next_cursor; + } + } + + /** Stop a running crawl. Idempotent: a crawl that already ended answers as it ended. */ + stop(crawlId: string): Promise { + return this.transport.requestJson("POST", `/crawls/${encodeURIComponent(crawlId)}/stop`); + } + + /** + * One kept URL's page as fetched (HTML). Pass the content id, or the `CrawlResult` itself + * (its `content_url` names the content; present once `content_status` is `fetched`). + */ + async getContent(crawlId: string, content: string | CrawlResult): Promise { + const contentId = contentIdOf(content); + const response = await this.transport.requestRaw( + "GET", + `/crawls/${encodeURIComponent(crawlId)}/contents/${encodeURIComponent(contentId)}`, + ); + return response.text(); + } + + /** + * Every result of the crawl in one download, one parsed line at a time (with the page when + * the crawl has an `output_format`). On a running crawl it holds what was kept so far. + */ + async *download(crawlId: string): AsyncGenerator { + const response = await this.transport.requestRaw( + "GET", + `/crawls/${encodeURIComponent(crawlId)}/download`, + ); + if (!response.body) return; + const decoder = new TextDecoder(); + let buffered = ""; + for await (const chunk of response.body as AsyncIterable) { + buffered += decoder.decode(chunk, { stream: true }); + let newline = buffered.indexOf("\n"); + while (newline !== -1) { + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + if (line) yield JSON.parse(line) as CrawlExportLine; + newline = buffered.indexOf("\n"); + } + } + const last = (buffered + decoder.decode()).trim(); + if (last) yield JSON.parse(last) as CrawlExportLine; + } + + /** + * Poll until the crawl ends (any status but `running`) and return it, without results. + * A `failed` crawl resolves too: read `error`. Throws `WaiterTimeoutError` after `timeout` + * seconds; the crawl is not stopped. + */ + async wait(crawlId: string, options: WaitForCrawlOptions = {}): Promise { + // `limit: 1` keeps each poll cheap; the one result it reads is dropped. + const ended = await pollUntil(() => this.get(crawlId, { limit: 1 }), { + isDone: (c) => c.status !== "running", + timeout: options.timeout ?? 300, + initialInterval: options.pollInterval ?? 2, + maxInterval: options.maxPollInterval ?? 15, + }); + const { results: _results, next_cursor: _cursor, ...crawl } = ended; + return crawl; + } +} diff --git a/src/crawl/errors.ts b/src/crawl/errors.ts new file mode 100644 index 0000000..f640841 --- /dev/null +++ b/src/crawl/errors.ts @@ -0,0 +1,58 @@ +import type { ProblemJson } from "../batch/types.js"; + +/** The API's code for an account Crawl is not enabled on (403). */ +export const CRAWL_NOT_ENABLED_CODE = "REQS008"; + +/** + * A non-2xx response from the Crawl API, decoded as RFC 9457 `application/problem+json` + * where possible. Branch on `status` and `code` (e.g. `crawl_not_found`, `too_many_crawls`, + * or `REQS008` when Crawl is not enabled for the account); `code` defaults to + * `"internal"` when the body wasn't valid Problem JSON. + */ +export class ZenRowsCrawlError extends Error { + readonly status: number; + readonly problem: ProblemJson | undefined; + readonly code: string; + readonly extras: Record | undefined; + /** Seconds from the `Retry-After` header, e.g. 30 on a 429 `too_many_crawls`. */ + readonly retryAfter: number | undefined; + + constructor( + status: number, + problem: ProblemJson | undefined, + extras?: Record, + retryAfter?: number, + ) { + let message: string; + if (status === 403 && problem?.code === CRAWL_NOT_ENABLED_CODE) { + message = `403 Crawl is not enabled for this account (${CRAWL_NOT_ENABLED_CODE})${problem.detail ? `: ${problem.detail}` : ""}`; + } else if (problem) { + message = `${status} ${problem.title ?? "Error"}: ${problem.detail ?? problem.code ?? "unknown"}`; + } else { + message = `${status} (no problem body)`; + } + super(message); + this.name = "ZenRowsCrawlError"; + this.status = status; + this.problem = problem; + this.code = problem?.code ?? "internal"; + this.extras = extras; + this.retryAfter = retryAfter; + } +} + +/** @internal Builds a `ZenRowsCrawlError` for the shared transport. */ +export function crawlError( + response: Response, + problem: ProblemJson | undefined, + extras: Record | undefined, +): ZenRowsCrawlError { + const raw = response.headers.get("Retry-After"); + const secs = raw === null ? Number.NaN : Number(raw); + return new ZenRowsCrawlError( + response.status, + problem, + extras, + Number.isNaN(secs) || secs < 0 ? undefined : secs, + ); +} diff --git a/src/crawl/types.ts b/src/crawl/types.ts new file mode 100644 index 0000000..6f44a49 --- /dev/null +++ b/src/crawl/types.ts @@ -0,0 +1,79 @@ +import type { Extensible } from "../batch/types.js"; + +/** `running` until the crawl ends; the other three are terminal. */ +export type CrawlStatus = "running" | "completed" | "stopped" | "failed"; +/** What ended a crawl early: a limit (`status: completed`) or `stop()` (`status: stopped`). */ +export type CrawlStopReason = "max_items" | "max_pages" | "user"; +/** Why a crawl failed (`status: failed`). */ +export type CrawlErrorCode = "insufficient_credits" | "seed_unreachable" | "no_items_found"; +/** Where a kept URL's page stands, when the crawl has an `output_format`. */ +export type CrawlContentStatus = "pending" | "fetched" | "failed"; +/** Return each kept URL's page as fetched. Absent means URLs only. */ +export type CrawlOutputFormat = "html"; + +export interface CrawlCoverage { + pages_fetched: number; + pages_failed: number; + items_found: number; +} + +/** Why a crawl failed. Part of the crawl, not an error response. */ +export interface CrawlRunError { + code: Extensible; + detail: string; +} + +export interface Crawl { + crawl_id: string; + status: Extensible; + url: string; + depth: number; + max_items: number; + max_pages: number; + coverage: CrawlCoverage; + created_at: string; + stop_reason?: Extensible; + error?: CrawlRunError; + include_patterns?: string[]; + exclude_patterns?: string[]; + output_format?: Extensible; + duplicates_removed?: number; + /** Absent while the crawl runs. */ + finished_at?: string; +} + +export interface CrawlResult { + url: string; + content_status?: Extensible; + /** The page's path, e.g. `/v1/crawls/c_x/contents/ct_y`. Present when `content_status` is `fetched`. */ + content_url?: string; +} + +export interface CrawlWithResults extends Crawl { + results: CrawlResult[]; + /** Never null while the crawl runs; null once it ended and this page holds its last URLs. */ + next_cursor: string | null; +} + +export interface CrawlList { + crawls: Crawl[]; + /** Absent on the last page. */ + next_cursor?: string; +} + +/** What `stop()` returns: where the crawl stands, without counts. */ +export interface CrawlStop { + crawl_id: string; + status: Extensible; + stop_reason?: Extensible; + error?: CrawlRunError; + finished_at?: string; +} + +/** One line of a crawl's download. */ +export interface CrawlExportLine { + url: string; + content_status?: Extensible; + /** The page HTML, present when `content_status` is `fetched`. */ + content?: string; +} diff --git a/src/index.ts b/src/index.ts index 72e82ed..8a518d1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import fastq from "fastq"; import fetchRetry from "fetch-retry"; import packageJson from "../package.json" with { type: "json" }; import { ZenRowsBatchClient } from "./batch/client.js"; +import { ZenRowsCrawlClient } from "./crawl/client.js"; export * from "./batch/client.js"; export * from "./batch/errors.js"; @@ -10,6 +11,9 @@ export * from "./batch/schedule.js"; export * from "./batch/waiters.js"; export * from "./batch/download.js"; export type * from "./batch/types.js"; +export * from "./crawl/client.js"; +export { CRAWL_NOT_ENABLED_CODE, ZenRowsCrawlError } from "./crawl/errors.js"; +export type * from "./crawl/types.js"; const API_URL = "https://api.zenrows.com/v1/"; @@ -66,11 +70,14 @@ export class ZenRows { readonly fetchWithRetry; /** Client for the Batch API (async job/run/task model) — see `./batch.ts`. */ readonly batch: ZenRowsBatchClient; + /** Client for the Crawl API (one start URL in, the URLs behind it out) — see `./crawl/client.ts`. */ + readonly crawl: ZenRowsCrawlClient; constructor(apiKey: string, clientConfig: ClientConfig = {}) { this.apiKey = apiKey; this.clientConfig = clientConfig; this.batch = new ZenRowsBatchClient(apiKey); + this.crawl = new ZenRowsCrawlClient(apiKey); const retries = this.clientConfig.retries ?? 0; this.queue = fastq.promise(this, this.worker, this.clientConfig.concurrency ?? 5); diff --git a/tests/crawl-client.test.ts b/tests/crawl-client.test.ts new file mode 100644 index 0000000..e2fac37 --- /dev/null +++ b/tests/crawl-client.test.ts @@ -0,0 +1,381 @@ +import { http, HttpResponse } from "msw"; +import { beforeEach, describe, expect, test } from "vitest"; +import { ZenRows } from "../src"; +import { WaiterTimeoutError } from "../src/batch/waiters"; +import { ZenRowsCrawlClient } from "../src/crawl/client"; +import { ZenRowsCrawlError } from "../src/crawl/errors"; +import type { CrawlWithResults } from "../src/crawl/types"; +import { server } from "./_setup"; + +const BASE = "https://api.zenrows.com/v1"; + +function crawl(overrides: Partial = {}): CrawlWithResults { + return { + crawl_id: "c_1", + status: "completed", + url: "https://example.com/", + depth: 1, + max_items: 10, + max_pages: 10, + coverage: { pages_fetched: 1, pages_failed: 0, items_found: 2 }, + created_at: "2026-10-08T09:00:00Z", + results: [], + next_cursor: null, + ...overrides, + }; +} + +function problem(status: number, code: string, title: string, headers?: Record) { + return HttpResponse.json( + { code, title, detail: `${title}.`, status, instance: "urn:zenrows:request:x" }, + { status, headers: { "Content-Type": "application/problem+json", ...headers } }, + ); +} + +describe("ZenRowsCrawlClient — requests", () => { + let client: ZenRowsCrawlClient; + + beforeEach(() => { + client = new ZenRowsCrawlClient("API_KEY"); + }); + + test("create sends snake_case fields, the API key and the Idempotency-Key", async () => { + let seen: { headers: Headers; body: unknown } | undefined; + server.use( + http.post(`${BASE}/crawls`, async ({ request }) => { + seen = { headers: request.headers, body: await request.json() }; + return HttpResponse.json(crawl({ status: "running" }), { + status: 202, + headers: { Location: "/v1/crawls/c_1" }, + }); + }), + ); + const created = await client.create( + { + url: "https://example.com/", + depth: 1, + maxItems: 3, + maxPages: 5, + includePatterns: ["/product/"], + excludePatterns: ["/cart"], + outputFormat: "html", + }, + { idempotencyKey: "idem-1" }, + ); + expect(created.crawl_id).toBe("c_1"); + expect(seen?.headers.get("X-API-Key")).toBe("API_KEY"); + expect(seen?.headers.get("Idempotency-Key")).toBe("idem-1"); + expect(seen?.body).toEqual({ + url: "https://example.com/", + depth: 1, + max_items: 3, + max_pages: 5, + include_patterns: ["/product/"], + exclude_patterns: ["/cart"], + output_format: "html", + }); + }); + + test("create sends only the fields the caller set", async () => { + let body: unknown; + server.use( + http.post(`${BASE}/crawls`, async ({ request }) => { + body = await request.json(); + return HttpResponse.json(crawl({ status: "running" }), { status: 202 }); + }), + ); + await client.create({ url: "https://example.com/", depth: 2 }); + expect(body).toEqual({ url: "https://example.com/", depth: 2 }); + }); + + test("get passes cursor and limit, and parses results", async () => { + let query: URLSearchParams | undefined; + server.use( + http.get(`${BASE}/crawls/c_1`, ({ request }) => { + query = new URL(request.url).searchParams; + return HttpResponse.json( + crawl({ results: [{ url: "https://example.com/product/a" }], next_cursor: "cur_1" }), + ); + }), + ); + const page = await client.get("c_1", { cursor: "cur_0", limit: 50 }); + expect(query?.get("cursor")).toBe("cur_0"); + expect(query?.get("limit")).toBe("50"); + expect(page.results).toEqual([{ url: "https://example.com/product/a" }]); + expect(page.next_cursor).toBe("cur_1"); + }); + + test("a baseURL override is used, with or without a trailing slash", async () => { + server.use(http.get("https://example.com/v1/crawls/c_1", () => HttpResponse.json(crawl()))); + const local = new ZenRowsCrawlClient("API_KEY", { baseURL: "https://example.com/v1/" }); + expect((await local.get("c_1")).crawl_id).toBe("c_1"); + }); + + test("tolerates unknown fields and enum values in responses", async () => { + server.use( + http.get(`${BASE}/crawls/c_1`, () => + HttpResponse.json({ ...crawl({ status: "archived" }), new_field: 1 }), + ), + ); + const got = await client.get("c_1"); + expect(got.status).toBe("archived"); + }); + + test("list and iterCrawls follow next_cursor until it is absent", async () => { + server.use( + http.get(`${BASE}/crawls`, ({ request }) => { + const cursor = new URL(request.url).searchParams.get("cursor"); + if (!cursor) { + return HttpResponse.json({ crawls: [crawl({ crawl_id: "c_2" })], next_cursor: "p2" }); + } + return HttpResponse.json({ crawls: [crawl({ crawl_id: "c_1" })] }); + }), + ); + const ids: string[] = []; + for await (const c of client.iterCrawls()) ids.push(c.crawl_id); + expect(ids).toEqual(["c_2", "c_1"]); + }); + + test("iterResults follows next_cursor and stops on null", async () => { + const cursors: (string | null)[] = []; + server.use( + http.get(`${BASE}/crawls/c_1`, ({ request }) => { + const cursor = new URL(request.url).searchParams.get("cursor"); + cursors.push(cursor); + if (!cursor) { + return HttpResponse.json(crawl({ results: [{ url: "u1" }], next_cursor: "cur_1" })); + } + return HttpResponse.json(crawl({ results: [{ url: "u2" }], next_cursor: null })); + }), + ); + const urls: string[] = []; + for await (const r of client.iterResults("c_1")) urls.push(r.url); + expect(urls).toEqual(["u1", "u2"]); + expect(cursors).toEqual([null, "cur_1"]); + }); + + test("iterResults on a running crawl returns once nothing more is kept yet", async () => { + let calls = 0; + server.use( + http.get(`${BASE}/crawls/c_1`, ({ request }) => { + calls += 1; + const cursor = new URL(request.url).searchParams.get("cursor"); + const results = cursor ? [] : [{ url: "u1" }]; + return HttpResponse.json(crawl({ status: "running", results, next_cursor: "cur_1" })); + }), + ); + const urls: string[] = []; + for await (const r of client.iterResults("c_1")) urls.push(r.url); + expect(urls).toEqual(["u1"]); + expect(calls).toBe(2); + }); + + test("stop posts with no body and returns the stop answer", async () => { + let hadBody = true; + server.use( + http.post(`${BASE}/crawls/c_1/stop`, async ({ request }) => { + hadBody = (await request.text()).length > 0; + return HttpResponse.json({ crawl_id: "c_1", status: "stopped", stop_reason: "user" }); + }), + ); + const stopped = await client.stop("c_1"); + expect(hadBody).toBe(false); + expect(stopped).toEqual({ crawl_id: "c_1", status: "stopped", stop_reason: "user" }); + }); + + test("getContent takes a content id or a result's content_url", async () => { + server.use( + http.get(`${BASE}/crawls/c_1/contents/ct_9`, () => + HttpResponse.text("page", { headers: { "Content-Type": "text/html" } }), + ), + ); + expect(await client.getContent("c_1", "ct_9")).toBe("page"); + expect( + await client.getContent("c_1", { + url: "https://example.com/product/a", + content_status: "fetched", + content_url: "/v1/crawls/c_1/contents/ct_9", + }), + ).toBe("page"); + }); + + test("getContent refuses a result whose page was not fetched", async () => { + await expect( + client.getContent("c_1", { url: "https://example.com/a", content_status: "pending" }), + ).rejects.toThrow(/no content_url/); + }); + + test("download yields one parsed object per NDJSON line, across chunk boundaries", async () => { + const lines = [ + { url: "u1", content_status: "fetched", content: "1" }, + { url: "u2", content_status: "failed" }, + { url: "u3" }, + ]; + const text = `${lines.map((l) => JSON.stringify(l)).join("\n")}\n`; + server.use( + http.get(`${BASE}/crawls/c_1/download`, () => { + const bytes = new TextEncoder().encode(text); + const stream = new ReadableStream({ + start(controller) { + for (let i = 0; i < bytes.length; i += 7) controller.enqueue(bytes.slice(i, i + 7)); + controller.close(); + }, + }); + return new HttpResponse(stream, { + headers: { "Content-Type": "application/x-ndjson", "X-Crawl-Status": "completed" }, + }); + }), + ); + const got = []; + for await (const line of client.download("c_1")) got.push(line); + expect(got).toEqual(lines); + }); +}); + +describe("ZenRowsCrawlClient — errors", () => { + let client: ZenRowsCrawlClient; + + beforeEach(() => { + client = new ZenRowsCrawlClient("API_KEY", { retries: 0 }); + }); + + test("404 crawl_not_found", async () => { + server.use( + http.get(`${BASE}/crawls/c_missing`, () => + problem(404, "crawl_not_found", "Crawl not found"), + ), + ); + await expect(client.get("c_missing")).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(ZenRowsCrawlError); + const err = error as ZenRowsCrawlError; + expect(err.status).toBe(404); + expect(err.code).toBe("crawl_not_found"); + return true; + }); + }); + + test("422 invalid_parameter carries the problem detail", async () => { + server.use( + http.post(`${BASE}/crawls`, () => problem(422, "invalid_parameter", "Invalid parameter")), + ); + await expect(client.create({ url: "https://example.com/", depth: 0 })).rejects.toSatisfy( + (error: unknown) => { + const err = error as ZenRowsCrawlError; + expect(err.status).toBe(422); + expect(err.code).toBe("invalid_parameter"); + expect(err.problem?.detail).toBe("Invalid parameter."); + return true; + }, + ); + }); + + test("429 too_many_crawls is not retried without an Idempotency-Key and carries retryAfter", async () => { + let calls = 0; + server.use( + http.post(`${BASE}/crawls`, () => { + calls += 1; + return problem(429, "too_many_crawls", "Too many crawls", { "Retry-After": "30" }); + }), + ); + const retrying = new ZenRowsCrawlClient("API_KEY"); + await expect(retrying.create({ url: "https://example.com/", depth: 1 })).rejects.toSatisfy( + (error: unknown) => { + const err = error as ZenRowsCrawlError; + expect(err.status).toBe(429); + expect(err.code).toBe("too_many_crawls"); + expect(err.retryAfter).toBe(30); + return true; + }, + ); + expect(calls).toBe(1); + }); + + test("403 REQS008 says Crawl is not enabled for this account", async () => { + server.use( + http.get(`${BASE}/crawls`, () => + HttpResponse.json( + { + code: "REQS008", + title: "Crawl is not enabled for this account.", + detail: "Crawl is not enabled for this account.", + status: 403, + type: "https://docs.zenrows.com/api-error-codes#REQS008", + instance: "/v1/crawls", + }, + { status: 403, headers: { "Content-Type": "application/problem+json" } }, + ), + ), + ); + await expect(client.list()).rejects.toSatisfy((error: unknown) => { + const err = error as ZenRowsCrawlError; + expect(err).toBeInstanceOf(ZenRowsCrawlError); + expect(err.status).toBe(403); + expect(err.code).toBe("REQS008"); + expect(err.message).toMatch(/^403 Crawl is not enabled for this account \(REQS008\)/); + return true; + }); + }); +}); + +describe("ZenRowsCrawlClient — wait", () => { + test("polls with limit=1 until the crawl ends, and returns it without results", async () => { + let polls = 0; + const limits: (string | null)[] = []; + server.use( + http.get(`${BASE}/crawls/c_1`, ({ request }) => { + polls += 1; + limits.push(new URL(request.url).searchParams.get("limit")); + const status = polls < 3 ? "running" : "completed"; + return HttpResponse.json( + crawl({ + status, + results: [{ url: "u1" }], + next_cursor: status === "running" ? "c" : null, + }), + ); + }), + ); + const client = new ZenRowsCrawlClient("API_KEY"); + const ended = await client.wait("c_1", { pollInterval: 0.01, maxPollInterval: 0.01 }); + expect(ended.status).toBe("completed"); + expect(ended).not.toHaveProperty("results"); + expect(ended).not.toHaveProperty("next_cursor"); + expect(polls).toBe(3); + expect(limits.every((l) => l === "1")).toBe(true); + }); + + test("resolves on a failed crawl, with its error", async () => { + server.use( + http.get(`${BASE}/crawls/c_1`, () => + HttpResponse.json( + crawl({ status: "failed", error: { code: "seed_unreachable", detail: "x" } }), + ), + ), + ); + const ended = await new ZenRowsCrawlClient("API_KEY").wait("c_1"); + expect(ended.error?.code).toBe("seed_unreachable"); + }); + + test("throws WaiterTimeoutError when the crawl keeps running", async () => { + server.use( + http.get(`${BASE}/crawls/c_1`, () => + HttpResponse.json(crawl({ status: "running", next_cursor: "c" })), + ), + ); + await expect( + new ZenRowsCrawlClient("API_KEY").wait("c_1", { + timeout: 0.05, + pollInterval: 0.01, + maxPollInterval: 0.01, + }), + ).rejects.toBeInstanceOf(WaiterTimeoutError); + }); +}); + +describe("ZenRows.crawl", () => { + test("is a ZenRowsCrawlClient on the instance's key", () => { + const client = new ZenRows("API_KEY"); + expect(client.crawl).toBeInstanceOf(ZenRowsCrawlClient); + expect(client.crawl.apiKey).toBe("API_KEY"); + }); +}); diff --git a/tests/e2e/crawl.e2e.test.ts b/tests/e2e/crawl.e2e.test.ts new file mode 100644 index 0000000..98952ac --- /dev/null +++ b/tests/e2e/crawl.e2e.test.ts @@ -0,0 +1,104 @@ +// End-to-end test against a live Crawl API. Skipped unless ZENROWS_API_KEY, +// ZENROWS_CRAWL_BASE_URL and ZENROWS_E2E_CRAWL_URL are set, so `pnpm test` stays hermetic. +// Run with `pnpm test:e2e`. +import { beforeAll, describe, expect, test } from "vitest"; +import { ZenRowsCrawlClient } from "../../src/crawl/client"; +import { ZenRowsCrawlError } from "../../src/crawl/errors"; +import type { Crawl, CrawlResult } from "../../src/crawl/types"; +import { server } from "../_setup"; + +const apiKey = process.env.ZENROWS_API_KEY; +const baseURL = process.env.ZENROWS_CRAWL_BASE_URL; +const startURL = process.env.ZENROWS_E2E_CRAWL_URL; +const include = process.env.ZENROWS_E2E_CRAWL_INCLUDE || undefined; + +// Other runs on the same account share its active crawl slots; wait for one to free. +const CREATE_RETRY_BUDGET_MS = 5 * 60_000; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +describe.skipIf(!apiKey || !baseURL || !startURL)("Crawl e2e", () => { + const client = new ZenRowsCrawlClient(apiKey ?? "", { baseURL }); + let ended: Crawl; + let results: CrawlResult[]; + + beforeAll(async () => { + // The shared msw server intercepts fetch for the unit tests; this file talks to the real API. + server.close(); + + const deadline = Date.now() + CREATE_RETRY_BUDGET_MS; + let created: Crawl; + while (true) { + try { + created = await client.create({ + url: startURL ?? "", + depth: 1, + maxItems: 3, + maxPages: 5, + includePatterns: include ? [include] : undefined, + outputFormat: "html", + }); + break; + } catch (error) { + const tooMany = error instanceof ZenRowsCrawlError && error.code === "too_many_crawls"; + if (!tooMany || Date.now() >= deadline) throw error; + await sleep((error.retryAfter ?? 30) * 1000); + } + } + console.log(`created ${created.crawl_id} (${created.status})`); + ended = await client.wait(created.crawl_id, { timeout: 600 }); + results = []; + for await (const result of client.iterResults(ended.crawl_id)) results.push(result); + console.log( + `ended ${ended.crawl_id}: ${ended.status}, ${results.length} results, coverage ${JSON.stringify(ended.coverage)}`, + ); + }, 20 * 60_000); + + test("the crawl completes with at least one result, each matching the include pattern", () => { + expect(ended.status).toBe("completed"); + expect(results.length).toBeGreaterThanOrEqual(1); + if (include) for (const result of results) expect(result.url).toContain(include); + }); + + test("a fetched result's content is HTML", async () => { + const fetched = results.find((r) => r.content_status === "fetched"); + expect(fetched).toBeDefined(); + const html = await client.getContent(ended.crawl_id, fetched as CrawlResult); + expect(html.toLowerCase()).toMatch(/ { + const lines = []; + for await (const line of client.download(ended.crawl_id)) lines.push(line); + expect(lines.length).toBe(results.length); + }, 60_000); + + test("the crawl is listed", async () => { + let found = false; + for await (const crawl of client.iterCrawls({ limit: 100 })) { + if (crawl.crawl_id === ended.crawl_id) { + found = true; + break; + } + } + expect(found).toBe(true); + }, 60_000); + + test("stop on an ended crawl answers with its terminal status", async () => { + const stopped = await client.stop(ended.crawl_id); + expect(stopped.crawl_id).toBe(ended.crawl_id); + expect(stopped.status).toBe(ended.status); + }, 60_000); + + test("get on an unknown id throws crawl_not_found", async () => { + await expect(client.get("c_does_not_exist")).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(ZenRowsCrawlError); + const err = error as ZenRowsCrawlError; + expect(err.status).toBe(404); + expect(err.code).toBe("crawl_not_found"); + return true; + }); + }, 60_000); +});