From f51cea7547833e03c22cac5f9726c39f3a8f6d30 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 10 Jan 2025 00:57:37 +0800 Subject: [PATCH] Refactor: migrate to undici fetch --- Build/lib/fetch-retry.ts | 3 +- Build/lib/fetch-text-by-line.ts | 5 +- Build/lib/make-fetch-happen.ts | 148 ++++++++++++++++---------------- Build/lib/parse-dnsmasq.ts | 3 +- Build/validate-gfwlist.ts | 8 +- 5 files changed, 83 insertions(+), 84 deletions(-) diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index bcd233b03..49233f43d 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -105,7 +105,8 @@ setGlobalDispatcher(agent.compose( }), interceptors.cache({ store: new BetterSqlite3CacheStore({ - location: path.resolve(__dirname, '../../.cache/undici-better-sqlite3-cache-store.db') + location: path.resolve(__dirname, '../../.cache/undici-better-sqlite3-cache-store.db'), + maxEntrySize: 1024 * 1024 * 50 // 50 MiB }) }) )); diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index bbef23c12..c40446501 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -8,7 +8,6 @@ import { TextLineStream } from './text-line-transform-stream'; import type { ReadableStream } from 'node:stream/web'; import { TextDecoderStream } from 'node:stream/web'; import { processLine, ProcessLineStream } from './process-line'; -import type { NodeFetchResponse } from './make-fetch-happen'; import { $$fetch } from './fetch-retry'; import type { UndiciResponseData } from './fetch-retry'; import type { Response as UnidiciWebResponse } from 'undici'; @@ -38,7 +37,7 @@ export async function readFileByLineNew(file: string): Promise(resp: T): NonNullable { +function ensureResponseBody(resp: T): NonNullable { if (resp.body == null) { throw new Error('Failed to fetch remote text'); } @@ -48,7 +47,7 @@ function ensureResponseBody ReadableStream) = (resp, processLine = false) => { +export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream) = (resp, processLine = false) => { const stream = ensureResponseBody(resp); const webStream: ReadableStream = 'getReader' in stream diff --git a/Build/lib/make-fetch-happen.ts b/Build/lib/make-fetch-happen.ts index d5529729e..d1c9e1795 100644 --- a/Build/lib/make-fetch-happen.ts +++ b/Build/lib/make-fetch-happen.ts @@ -1,86 +1,86 @@ import path from 'node:path'; -import fs from 'node:fs'; -import makeFetchHappen from 'make-fetch-happen'; -import type { FetchOptions } from 'make-fetch-happen'; -import cacache from 'cacache'; -import picocolors from 'picocolors'; -// eslint-disable-next-line @typescript-eslint/no-restricted-imports -- type only -import type { Response as NodeFetchResponse } from 'node-fetch'; -import { task } from '../trace'; -import { bytes } from 'xbits'; +import fsp from 'node:fs/promises'; +// import makeFetchHappen from 'make-fetch-happen'; +// import type { FetchOptions } from 'make-fetch-happen'; +// import cacache from 'cacache'; +// import picocolors from 'picocolors'; -export type { NodeFetchResponse }; +import { task } from '../trace'; +// import { bytes } from 'xbits'; const cachePath = path.resolve(__dirname, '../../.cache/__make_fetch_happen__'); -fs.mkdirSync(cachePath, { recursive: true }); +// fs.mkdirSync(cachePath, { recursive: true }); -interface CacacheVerifyStats { - startTime: Date, - endTime: Date, - runTime: { - markStartTime: 0, - fixPerms: number, - garbageCollect: number, - rebuildIndex: number, - cleanTmp: number, - writeVerifile: number, - markEndTime: number, - total: number - }, - verifiedContent: number, - reclaimedCount: number, - reclaimedSize: number, - badContentCount: number, - keptSize: number, - missingContent: number, - rejectedEntries: number, - totalEntries: number -} +// interface CacacheVerifyStats { +// startTime: Date, +// endTime: Date, +// runTime: { +// markStartTime: 0, +// fixPerms: number, +// garbageCollect: number, +// rebuildIndex: number, +// cleanTmp: number, +// writeVerifile: number, +// markEndTime: number, +// total: number +// }, +// verifiedContent: number, +// reclaimedCount: number, +// reclaimedSize: number, +// badContentCount: number, +// keptSize: number, +// missingContent: number, +// rejectedEntries: number, +// totalEntries: number +// } export const cacheGc = task(require.main === module, __filename)( - (span) => span - .traceChildAsync('cacache gc', () => cacache.verify(cachePath, { concurrency: 64 })) - .then((stats: CacacheVerifyStats) => { - // console.log({ stats }); - console.log(picocolors.green('[cacheGc] running gc on cache:'), cachePath); - console.log(picocolors.green('[cacheGc] content verified:'), stats.verifiedContent, '(' + bytes(stats.keptSize) + ')'); - console.log(picocolors.green('[cacheGc] reclaimed:'), stats.reclaimedCount, '(' + bytes(stats.reclaimedSize) + ')'); - }) + () => fsp.rm(cachePath, { recursive: true, force: true }) + // span + // .traceChildAsync('cacache gc', () => cacache.verify(cachePath, { concurrency: 64 })) + // .then((stats: CacacheVerifyStats) => { + // // console.log({ stats }); + // console.log(picocolors.green('[cacheGc] running gc on cache:'), cachePath); + // console.log(picocolors.green('[cacheGc] content verified:'), stats.verifiedContent, '(' + bytes(stats.keptSize) + ')'); + // console.log(picocolors.green('[cacheGc] reclaimed:'), stats.reclaimedCount, '(' + bytes(stats.reclaimedSize) + ')'); + // }); ); -const _fetch = makeFetchHappen.defaults({ - cachePath, - maxSockets: 32, /** - * They said 15 is a good default that prevents knocking out others' routers, - * I disagree. 32 is a good number. - */ - headers: { - 'User-Agent': 'curl/8.9.1 (https://github.com/SukkaW/Surge)' - }, - retry: { - retries: 5, - randomize: true - } -}); +// const _fetch = makeFetchHappen.defaults({ +// cachePath, +// maxSockets: 32, /** +// * They said 15 is a good default that prevents knocking out others' routers, +// * I disagree. 32 is a good number. +// */ +// headers: { +// 'User-Agent': 'curl/8.9.1 (https://github.com/SukkaW/Surge)' +// }, +// retry: { +// retries: 5, +// randomize: true +// } +// }); + +// export function $fetch(uriOrRequest: string | Request, opts?: FetchOptions) { +// return _fetch(uriOrRequest, opts).then((resp) => { +// printResponseStatus(resp); +// return resp; +// }); +// } -export function $fetch(uriOrRequest: string | Request, opts?: FetchOptions) { - return _fetch(uriOrRequest, opts).then((resp) => { - printResponseStatus(resp); - return resp; - }); -} +// export async function $delete(resp: NodeFetchResponse) { +// const cacheKey = resp.headers.get('X-Local-Cache-Key'); +// if (cacheKey) { +// await cacache.rm.entry(cachePath, cacheKey); +// await cacache.verify(cachePath, { concurrency: 64 }); +// } +// } -export async function $delete(resp: NodeFetchResponse) { - const cacheKey = resp.headers.get('X-Local-Cache-Key'); - if (cacheKey) { - await cacache.rm.entry(cachePath, cacheKey); - await cacache.verify(cachePath, { concurrency: 64 }); - } -} +// export function printResponseStatus(resp: NodeFetchResponse) { +// const status = resp.headers.get('X-Local-Cache-Status'); +// if (status) { +// console.log('[$fetch cache]', { status }, picocolors.gray(resp.url)); +// } +// } -export function printResponseStatus(resp: NodeFetchResponse) { - const status = resp.headers.get('X-Local-Cache-Status'); - if (status) { - console.log('[$fetch cache]', { status }, picocolors.gray(resp.url)); - } -} +// export { type Response as NodeFetchResponse } from 'node-fetch'; diff --git a/Build/lib/parse-dnsmasq.ts b/Build/lib/parse-dnsmasq.ts index 45e3dc468..b1d493f70 100644 --- a/Build/lib/parse-dnsmasq.ts +++ b/Build/lib/parse-dnsmasq.ts @@ -4,7 +4,6 @@ import { createReadlineInterfaceFromResponse } from './fetch-text-by-line'; // In short, single label domain suffix is ignored due to the size optimization, so no isIcann // import tldts from 'tldts-experimental'; import tldts from 'tldts'; -import type { NodeFetchResponse } from './make-fetch-happen'; import type { UndiciResponseData } from './fetch-retry'; import type { Response } from 'undici'; @@ -20,7 +19,7 @@ export function extractDomainsFromFelixDnsmasq(line: string): string | null { return null; } -export async function parseFelixDnsmasqFromResp(resp: NodeFetchResponse | UndiciResponseData | Response): Promise { +export async function parseFelixDnsmasqFromResp(resp: UndiciResponseData | Response): Promise { const results: string[] = []; for await (const line of createReadlineInterfaceFromResponse(resp, true)) { diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts index 5e5bbf639..8b67fda65 100644 --- a/Build/validate-gfwlist.ts +++ b/Build/validate-gfwlist.ts @@ -6,8 +6,8 @@ import { parse } from 'csv-parse/sync'; import { readFileByLine } from './lib/fetch-text-by-line'; import path from 'node:path'; import { OUTPUT_SURGE_DIR } from './constants/dir'; -import { $fetch } from './lib/make-fetch-happen'; import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; +import { $$fetch } from './lib/fetch-retry'; export async function parseGfwList() { const whiteSet = new Set(); @@ -22,7 +22,7 @@ export async function parseGfwList() { '?' ]); - const text = await (await $fetch('https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt')).text(); + const text = await (await $$fetch('https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt')).text(); for (const l of atob(text).split('\n')) { const line = processLine(l); if (!line) continue; @@ -60,11 +60,11 @@ export async function parseGfwList() { continue; } } - for (const l of (await (await $fetch('https://raw.githubusercontent.com/Loyalsoldier/cn-blocked-domain/release/domains.txt')).text()).split('\n')) { + for (const l of (await (await $$fetch('https://raw.githubusercontent.com/Loyalsoldier/cn-blocked-domain/release/domains.txt')).text()).split('\n')) { trie.add(l); } - const res = await (await $fetch('https://litter.catbox.moe/sqmgyn.csv', { + const res = await (await $$fetch('https://litter.catbox.moe/sqmgyn.csv', { headers: { accept: '*/*', 'user-agent': 'curl/8.9.1'