Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ SDK to access [Zenrows](https://www.zenrows.com/) API directly from Node.js. Zen
- [POST Requests](#post-requests)
- [Extract](#extract)
- [Batch](#batch)
- [Extensible values](#extensible-values)
- [Extract in a batch](#extract-in-a-batch)
- [Concurrency](#concurrency)
- [An important note about Promise.allSettled() on TypeScript](#an-important-note-about-promiseallsettled-on-typescript)
Expand Down Expand Up @@ -226,6 +227,10 @@ const apiKey = "YOUR-API-KEY";

`client.batch` also exposes `listJobs()`, `deleteJob()`, `stopRun()`, `rerun()`, `listRuns()`, `getRun()`, `deleteRun()`, and `getTaskContent()` (returns the scraped page's raw content as a string, not JSON — the endpoint can return HTML or plain text depending on what the target page served). Scheduling, webhook config, HMAC key rotation, CSV task uploads, and results exports aren't wrapped yet — call the [Batch API](https://docs.zenrows.com) directly for those.

#### Extensible values

Enum-valued fields on responses (run, job, task and export `status`, `ingest_status`, `failure_reason`) are typed `Extensible<...>`: the known values plus any `string`, because the server may add values. The exported named types (`RunStatus`, `JobStatus`, `TaskStatus`, `ExportStatus`, `IngestStatus`) stay closed, so they still work as request input. Give switches on these fields a default branch; an exhaustive `switch` that assigns the leftover to `never` no longer compiles.

#### Extract in a batch

Set `extract` in the batch params to run tasks through Extract — structured data instead of raw HTML. It works job-wide or per task, and per-task values win on collision.
Expand Down
6 changes: 1 addition & 5 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,7 @@
"format": "biome format --write .",
"check": "biome check ."
},
"keywords": [
"sdk",
"zenrows",
"scraping"
],
"keywords": ["sdk", "zenrows", "scraping"],
"author": "ZenRows",
"repository": {
"type": "git",
Expand Down
3 changes: 2 additions & 1 deletion src/batch/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import type {
AddTasksResponse,
CreateJobInputResponse,
Export,
Extensible,
HMACKeyCreated,
HMACKeyFinalized,
HMACKeyList,
Expand Down Expand Up @@ -164,7 +165,7 @@ export class ZenRowsBatchClient {
const { idempotencyKey, waitForIngest, ...body } = options;
const resp = await this.transport.requestJson<{
job_id: string;
status: JobStatus;
status: Extensible<JobStatus>;
latest_run?: Run;
accepted_tasks: number;
webhook?: WebhookConfig;
Expand Down
11 changes: 6 additions & 5 deletions src/batch/resources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import type { Schedule } from "./schedule.js";
import type {
Export,
ExportStatus,
Extensible,
Job,
JobStatus,
Run,
Expand Down Expand Up @@ -127,7 +128,7 @@ export class RunHandle extends RunRef {
super(client, jobId, runId);
}

get status(): RunStatus {
get status(): Extensible<RunStatus> {
return this.data.status;
}

Expand Down Expand Up @@ -210,7 +211,7 @@ export class ExportHandle extends ExportRef {
super(client, jobId, runId, exportId, startResponse);
}

get status(): ExportStatus {
get status(): Extensible<ExportStatus> {
return this.data.status;
}
}
Expand Down Expand Up @@ -394,7 +395,7 @@ export class JobRef {
}

/** The job status from the submit response — only known on refs from a `submit*` call. */
get status(): JobStatus | undefined {
get status(): Extensible<JobStatus> | undefined {
return this.submitResponse?.status;
}

Expand Down Expand Up @@ -434,7 +435,7 @@ export class JobRef {
options: { lastBatch?: boolean } = {},
): Promise<{
accepted_tasks: number;
job_status: JobStatus;
job_status: Extensible<JobStatus>;
latest_run: Run;
}> {
return this.#client._postTasks(this.jobId, tasks, options);
Expand Down Expand Up @@ -491,7 +492,7 @@ export class JobHandle extends JobRef {
super(client, jobId, submitResponse);
}

get status(): JobStatus {
get status(): Extensible<JobStatus> {
return this.data.status;
}
}
28 changes: 18 additions & 10 deletions src/batch/types.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
/**
* A server-extensible string enum: the listed values autocomplete, but the server may
* return values not listed (the Batch API marks these `x-extensible-enum`). Response fields
* use `Extensible<...>`, so callers must handle an unknown value. The named types stay closed
* so they keep working as request input.
*/
export type Extensible<T extends string> = T | (string & {});

export type JobType = "regular" | "scheduled";
export type JobStatus = "open" | "closed" | "deleted";
export type ScheduleState = "active" | "paused";
Expand Down Expand Up @@ -45,20 +53,20 @@ export interface Run {
run_id: string;
job_id: string;
run_sequence: number;
status: RunStatus;
status: Extensible<RunStatus>;
stats: RunStats;
last_batch_received?: boolean;
pause_state?: PauseState;
ingest_status?: IngestStatus;
failure_reason?: "insufficient_credits" | "subscription_inactive";
ingest_status?: Extensible<IngestStatus>;
failure_reason?: Extensible<"insufficient_credits" | "subscription_inactive">;
created_at?: string;
updated_at?: string;
}

export interface Job {
job_id: string;
type: JobType;
status: JobStatus;
status: Extensible<JobStatus>;
format?: string;
zenrows_params?: Record<string, string>;
external_id?: string;
Expand All @@ -85,21 +93,21 @@ export interface ListJobRunsResponse {

export interface SubmitJobResponse {
job_id: string;
status: JobStatus;
status: Extensible<JobStatus>;
latest_run?: Run;
accepted_tasks: number;
webhook?: WebhookConfig;
}

export interface AddTasksResponse {
accepted_tasks: number;
job_status: JobStatus;
job_status: Extensible<JobStatus>;
latest_run: Run;
}

export interface RerunJobResponse {
job_id: string;
status: JobStatus;
status: Extensible<JobStatus>;
latest_run: Run;
rerun_of?: string;
retried_tasks: number;
Expand All @@ -122,7 +130,7 @@ export interface TaskResult {
url: string;
metadata?: Record<string, string>;
method?: "GET" | "POST";
status: TaskStatus;
status: Extensible<TaskStatus>;
type?: string;
result_url?: string;
error?: ProblemJson;
Expand All @@ -149,14 +157,14 @@ export interface TaskHistoryResponse {

export interface StartExportResponse {
export_id: string;
status: ExportStatus;
status: Extensible<ExportStatus>;
created_at: string;
expires_at: string;
}

export interface Export {
export_id: string;
status: ExportStatus;
status: Extensible<ExportStatus>;
error?: string;
download_url?: string;
created_at: string;
Expand Down
1 change: 1 addition & 0 deletions tests/batch-download.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ describe("downloadToDir / downloadToMemory / single-task download", () => {
test("downloadTaskToFile / downloadTaskToMemory work on a single already-held TaskResult", async () => {
const { results } = await client.getResults("job_download");
const task = results[0];
if (!task) throw new Error("fixture returned no results");
const memory = await client.downloadTaskToMemory(task);
expect(memory.toString("utf-8")).toBe("body A");

Expand Down
41 changes: 41 additions & 0 deletions tests/batch-extensible-enums.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { http, HttpResponse } from "msw";
import { describe, expect, expectTypeOf, test } from "vitest";
import { type ListJobsOptions, ZenRowsBatchClient } from "../src/batch/client";
import type { JobStatus, Run, RunStatus, TaskResult, TaskStatus } from "../src/batch/types";
import { server } from "./_setup";

const BASE = "https://async.api.zenrows.com/v1";

describe("server-extensible enums", () => {
test("response fields list the known values and still accept unknown ones", () => {
expectTypeOf<"failed">().toMatchTypeOf<Run["status"]>();
expectTypeOf<"archived">().toMatchTypeOf<Run["status"]>();
expectTypeOf<"a_reason_added_later">().toMatchTypeOf<NonNullable<Run["failure_reason"]>>();
expectTypeOf<"queued">().toMatchTypeOf<TaskResult["status"]>();
});

test("named types stay closed so they keep working as request input", () => {
expectTypeOf<"archived">().not.toMatchTypeOf<RunStatus>();
expectTypeOf<"queued">().not.toMatchTypeOf<TaskStatus>();
const status: JobStatus = "open";
expectTypeOf<{ status: typeof status }>().toMatchTypeOf<ListJobsOptions>();
});

test("parses a run with an unknown failure_reason and status without throwing", async () => {
server.use(
http.get(`${BASE}/jobs/job_x/runs/run_1`, () =>
HttpResponse.json({
run_id: "run_1",
job_id: "job_x",
run_sequence: 1,
status: "archived",
stats: { total: 1, completed: 0, successful: 0, failed: 0 },
failure_reason: "a_reason_added_later",
}),
),
);
const run = await new ZenRowsBatchClient("API_KEY").getRun("job_x", "run_1");
expect(run.data.failure_reason).toBe("a_reason_added_later");
expect(run.status).toBe("archived");
});
});
4 changes: 2 additions & 2 deletions tests/concurrency.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ describe("ZenRows Client with Concurrency", () => {
const [response1, response2] = responses;

expect(responses.length).toBe(2);
expect(response1.status).toBe(200);
expect(response2.status).toBe(200);
expect(response1?.status).toBe(200);
expect(response2?.status).toBe(200);
});
});
4 changes: 3 additions & 1 deletion tests/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ describe("ZenRows Client Get", () => {
const parsedUrl = new URL(requestUrl);

for (const key in optionalParams) {
expect(parsedUrl.searchParams.get(key)).toBe(optionalParams[key].toString());
expect(parsedUrl.searchParams.get(key)).toBe(
optionalParams[key as keyof typeof optionalParams].toString(),
);
}
});

Expand Down
9 changes: 9 additions & 0 deletions tsconfig.test.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"noEmit": true,
"module": "ESNext",
"moduleResolution": "Bundler"
},
"include": ["src", "tests"]
}
2 changes: 2 additions & 0 deletions vitest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ export default defineConfig({
globals: true,
include: ["tests/**/*.test.ts"],
setupFiles: ["tests/_setup.ts"],
// Type-level assertions (expectTypeOf) only run under typecheck.
typecheck: { enabled: true, tsconfig: "./tsconfig.test.json", include: ["tests/**/*.test.ts"] },
coverage: {
include: ["src/**"],
exclude: ["examples/**"],
Expand Down
Loading