diff --git a/bun.lock b/bun.lock index 9217d577..86bd0c93 100644 --- a/bun.lock +++ b/bun.lock @@ -9,17 +9,23 @@ "@ast-grep/napi": "^0.40.0", "@clack/prompts": "^0.11.0", "@code-yeongyu/comment-checker": "^0.6.0", + "@mozilla/readability": "^0.6.0", "@openauthjs/openauth": "^0.4.3", "@opencode-ai/plugin": "^1.0.162", + "cheerio": "^1.1.2", "commander": "^14.0.2", "hono": "^4.10.4", + "jq-wasm": "^1.1.0-jq-1.8.1", + "jsdom": "^27.3.0", "picocolors": "^1.1.1", "picomatch": "^4.0.2", + "turndown": "^7.2.2", "xdg-basedir": "^5.1.0", "zod": "^4.1.8", }, "devDependencies": { "@types/picomatch": "^3.0.2", + "@types/turndown": "^5.0.6", "bun-types": "latest", "typescript": "^5.7.3", }, @@ -31,6 +37,14 @@ "@code-yeongyu/comment-checker", ], "packages": { + "@acemir/cssom": ["@acemir/cssom@0.9.30", "", {}, "sha512-9CnlMCI0LmCIq0olalQqdWrJHPzm0/tw3gzOA9zJSgvFX7Xau3D24mAGa4BtwxwY69nsuJW6kQqqCzf/mEcQgg=="], + + "@asamuzakjp/css-color": ["@asamuzakjp/css-color@4.1.1", "", { "dependencies": { "@csstools/css-calc": "^2.1.4", "@csstools/css-color-parser": "^3.1.0", "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4", "lru-cache": "^11.2.4" } }, "sha512-B0Hv6G3gWGMn0xKJ0txEi/jM5iFpT3MfDxmhZFb4W047GvytCf1DHQ1D69W3zHI4yWe2aTZAA0JnbMZ7Xc8DuQ=="], + + "@asamuzakjp/dom-selector": ["@asamuzakjp/dom-selector@6.7.6", "", { "dependencies": { "@asamuzakjp/nwsapi": "^2.3.9", "bidi-js": "^1.0.3", "css-tree": "^3.1.0", "is-potential-custom-element-name": "^1.0.1", "lru-cache": "^11.2.4" } }, "sha512-hBaJER6A9MpdG3WgdlOolHmbOYvSk46y7IQN/1+iqiCuUu6iWdQrs9DGKF8ocqsEqWujWf/V7b7vaDgiUmIvUg=="], + + "@asamuzakjp/nwsapi": ["@asamuzakjp/nwsapi@2.3.9", "", {}, "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q=="], + "@ast-grep/cli": ["@ast-grep/cli@0.40.0", "", { "dependencies": { "detect-libc": "2.1.2" }, "optionalDependencies": { "@ast-grep/cli-darwin-arm64": "0.40.0", "@ast-grep/cli-darwin-x64": "0.40.0", "@ast-grep/cli-linux-arm64-gnu": "0.40.0", "@ast-grep/cli-linux-x64-gnu": "0.40.0", "@ast-grep/cli-win32-arm64-msvc": "0.40.0", "@ast-grep/cli-win32-ia32-msvc": "0.40.0", "@ast-grep/cli-win32-x64-msvc": "0.40.0" }, "bin": { "sg": "sg", "ast-grep": "ast-grep" } }, "sha512-L8AkflsfI2ZP70yIdrwqvjR02ScCuRmM/qNGnJWUkOFck+e6gafNVJ4e4jjGQlEul+dNdBpx36+O2Op629t47A=="], "@ast-grep/cli-darwin-arm64": ["@ast-grep/cli-darwin-arm64@0.40.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-UehY2MMUkdJbsriP7NKc6+uojrqPn7d1Cl0em+WAkee7Eij81VdyIjRsRxtZSLh440ZWQBHI3PALZ9RkOO8pKQ=="], @@ -73,6 +87,22 @@ "@code-yeongyu/comment-checker": ["@code-yeongyu/comment-checker@0.6.0", "", { "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ], "bin": { "comment-checker": "bin/comment-checker" } }, "sha512-VtDPrhbUJcb5BIS18VMcY/N/xSLbMr6dpU9MO1NYQyEDhI4pSIx07K4gOlCutG/nHVCjO+HEarn8rttODP+5UA=="], + "@csstools/color-helpers": ["@csstools/color-helpers@5.1.0", "", {}, "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA=="], + + "@csstools/css-calc": ["@csstools/css-calc@2.1.4", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ=="], + + "@csstools/css-color-parser": ["@csstools/css-color-parser@3.1.0", "", { "dependencies": { "@csstools/color-helpers": "^5.1.0", "@csstools/css-calc": "^2.1.4" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA=="], + + "@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@3.0.5", "", { "peerDependencies": { "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ=="], + + "@csstools/css-syntax-patches-for-csstree": ["@csstools/css-syntax-patches-for-csstree@1.0.22", "", {}, "sha512-qBcx6zYlhleiFfdtzkRgwNC7VVoAwfK76Vmsw5t+PbvtdknO9StgRk7ROvq9so1iqbdW4uLIDAsXRsTfUrIoOw=="], + + "@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="], + + "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="], + + "@mozilla/readability": ["@mozilla/readability@0.6.0", "", {}, "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ=="], + "@openauthjs/openauth": ["@openauthjs/openauth@0.4.3", "", { "dependencies": { "@standard-schema/spec": "1.0.0-beta.3", "aws4fetch": "1.0.20", "jose": "5.9.6" }, "peerDependencies": { "arctic": "^2.2.2", "hono": "^4.0.0" } }, "sha512-RlnjqvHzqcbFVymEwhlUEuac4utA5h4nhSK/i2szZuQmxTIqbGUxZ+nM+avM+VV4Ing+/ZaNLKILoXS3yrkOOw=="], "@opencode-ai/plugin": ["@opencode-ai/plugin@1.0.162", "", { "dependencies": { "@opencode-ai/sdk": "1.0.162", "zod": "4.1.8" } }, "sha512-tiJw7SCfSlG/3tY2O0J2UT06OLuazOzsv1zYlFbLxLy/EVedtW0pzxYalO20a4e//vInvOXFkhd2jLyB5vNEVA=="], @@ -95,34 +125,150 @@ "@types/picomatch": ["@types/picomatch@3.0.2", "", {}, "sha512-n0i8TD3UDB7paoMMxA3Y65vUncFJXjcUf7lQY7YyKGl6031FNjfsLs6pdLFCy2GNFxItPJG8GvvpbZc2skH7WA=="], + "@types/turndown": ["@types/turndown@5.0.6", "", {}, "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg=="], + + "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], + "arctic": ["arctic@2.3.4", "", { "dependencies": { "@oslojs/crypto": "1.0.1", "@oslojs/encoding": "1.1.0", "@oslojs/jwt": "0.2.0" } }, "sha512-+p30BOWsctZp+CVYCt7oAean/hWGW42sH5LAcRQX56ttEkFJWbzXBhmSpibbzwSJkRrotmsA+oAoJoVsU0f5xA=="], "aws4fetch": ["aws4fetch@1.0.20", "", {}, "sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g=="], + "bidi-js": ["bidi-js@1.0.3", "", { "dependencies": { "require-from-string": "^2.0.2" } }, "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw=="], + + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + "bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], + "cheerio": ["cheerio@1.1.2", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.0.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.12.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg=="], + + "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], + "commander": ["commander@14.0.2", "", {}, "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ=="], + "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + + "css-tree": ["css-tree@3.1.0", "", { "dependencies": { "mdn-data": "2.12.2", "source-map-js": "^1.0.1" } }, "sha512-0eW44TGN5SQXU1mWSkKwFstI/22X2bG1nYzZTYMAWjylYURhse752YgbE4Cx46AC+bAvI+/dYTPRk1LqSUnu6w=="], + + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + + "cssstyle": ["cssstyle@5.3.5", "", { "dependencies": { "@asamuzakjp/css-color": "^4.1.1", "@csstools/css-syntax-patches-for-csstree": "^1.0.21", "css-tree": "^3.1.0" } }, "sha512-GlsEptulso7Jg0VaOZ8BXQi3AkYM5BOJKEO/rjMidSCq70FkIC5y0eawrCXeYzxgt3OCf4Ls+eoxN+/05vN0Ag=="], + + "data-urls": ["data-urls@6.0.0", "", { "dependencies": { "whatwg-mimetype": "^4.0.0", "whatwg-url": "^15.0.0" } }, "sha512-BnBS08aLUM+DKamupXs3w2tJJoqU+AkaE/+6vQxi/G/DPmIZFJJp9Dkb1kM03AZx8ADehDUZgsNxju3mPXZYIA=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "decimal.js": ["decimal.js@10.6.0", "", {}, "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + + "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], + + "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + + "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="], + + "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "hono": ["hono@4.10.8", "", {}, "sha512-DDT0A0r6wzhe8zCGoYOmMeuGu3dyTAE40HHjwUsWFTEy5WxK1x2WDSsBPlEXgPbRIFY6miDualuUDbasPogIww=="], + "html-encoding-sniffer": ["html-encoding-sniffer@4.0.0", "", { "dependencies": { "whatwg-encoding": "^3.1.1" } }, "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ=="], + + "htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="], + + "http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="], + + "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "is-potential-custom-element-name": ["is-potential-custom-element-name@1.0.1", "", {}, "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="], + "jose": ["jose@5.9.6", "", {}, "sha512-AMlnetc9+CV9asI19zHmrgS/WYsWUwCn2R7RzlbJWD7F9eWYUTGyBmU9o6PxngtLGOiDGPRu+Uc4fhKzbpteZQ=="], + "jq-wasm": ["jq-wasm@1.1.0-jq-1.8.1", "", {}, "sha512-lWfu34lpDFIygOYcL5TzxhZIApDR9iR5XywcVoyUAZ6jlQrj8HKHOKeCcHgUm2dE9RVdbP3eqNAKGLuj+k4seQ=="], + + "jsdom": ["jsdom@27.3.0", "", { "dependencies": { "@acemir/cssom": "^0.9.28", "@asamuzakjp/dom-selector": "^6.7.6", "cssstyle": "^5.3.4", "data-urls": "^6.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "parse5": "^8.0.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.0", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^15.1.0", "ws": "^8.18.3", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-GtldT42B8+jefDUC4yUKAvsaOrH7PDHmZxZXNgF2xMmymjUbRYJvpAybZAKEmXDGTM0mCsz8duOa4vTm5AY2Kg=="], + + "lru-cache": ["lru-cache@11.2.4", "", {}, "sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg=="], + + "mdn-data": ["mdn-data@2.12.2", "", {}, "sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + + "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + + "parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="], + + "parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="], + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "saxes": ["saxes@6.0.0", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA=="], + "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + + "symbol-tree": ["symbol-tree@3.2.4", "", {}, "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="], + + "tldts": ["tldts@7.0.19", "", { "dependencies": { "tldts-core": "^7.0.19" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-8PWx8tvC4jDB39BQw1m4x8y5MH1BcQ5xHeL2n7UVFulMPH/3Q0uiamahFJ3lXA0zO2SUyRXuVVbWSDmstlt9YA=="], + + "tldts-core": ["tldts-core@7.0.19", "", {}, "sha512-lJX2dEWx0SGH4O6p+7FPwYmJ/bu1JbcGJ8RLaG9b7liIgZ85itUVEPbMtWRVrde/0fnDPEPHW10ZsKW3kVsE9A=="], + + "tough-cookie": ["tough-cookie@6.0.0", "", { "dependencies": { "tldts": "^7.0.5" } }, "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w=="], + + "tr46": ["tr46@6.0.0", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw=="], + + "turndown": ["turndown@7.2.2", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "undici": ["undici@7.16.0", "", {}, "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g=="], + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + "w3c-xmlserializer": ["w3c-xmlserializer@5.0.0", "", { "dependencies": { "xml-name-validator": "^5.0.0" } }, "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA=="], + + "webidl-conversions": ["webidl-conversions@8.0.0", "", {}, "sha512-n4W4YFyz5JzOfQeA8oN7dUYpR+MBP3PIUsn2jLjWXwK5ASUzt0Jc/A5sAUZoCYFJRGF0FBKJ+1JjN43rNdsQzA=="], + + "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="], + + "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], + + "whatwg-url": ["whatwg-url@15.1.0", "", { "dependencies": { "tr46": "^6.0.0", "webidl-conversions": "^8.0.0" } }, "sha512-2ytDk0kiEj/yu90JOAp44PVPUkO9+jVhyf+SybKlRHSDlvOOZhdPIrr7xTH64l4WixO2cP+wQIcgujkGBPPz6g=="], + + "ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="], + "xdg-basedir": ["xdg-basedir@5.1.0", "", {}, "sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ=="], + "xml-name-validator": ["xml-name-validator@5.0.0", "", {}, "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg=="], + + "xmlchars": ["xmlchars@2.2.0", "", {}, "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="], + "zod": ["zod@4.1.8", "", {}, "sha512-5R1P+WwQqmmMIEACyzSvo4JXHY5WiAFHRMg+zBZKgKS+Q1viRa0C1hmUKtHltoIFKtIdki3pRxkmpP74jnNYHQ=="], "@oslojs/jwt/@oslojs/encoding": ["@oslojs/encoding@0.4.1", "", {}, "sha512-hkjo6MuIK/kQR5CrGNdAPZhS01ZCXuWDRJ187zh6qqF2+yMHZpD9fAYpX8q2bOO6Ryhl3XpCT6kUX76N8hhm4Q=="], + + "htmlparser2/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + + "jsdom/parse5": ["parse5@8.0.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA=="], + + "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + + "jsdom/parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], } } diff --git a/package.json b/package.json index ffd45898..c773c5c9 100644 --- a/package.json +++ b/package.json @@ -54,17 +54,23 @@ "@ast-grep/napi": "^0.40.0", "@clack/prompts": "^0.11.0", "@code-yeongyu/comment-checker": "^0.6.0", + "@mozilla/readability": "^0.6.0", "@openauthjs/openauth": "^0.4.3", "@opencode-ai/plugin": "^1.0.162", + "cheerio": "^1.1.2", "commander": "^14.0.2", "hono": "^4.10.4", + "jq-wasm": "^1.1.0-jq-1.8.1", + "jsdom": "^27.3.0", "picocolors": "^1.1.1", "picomatch": "^4.0.2", + "turndown": "^7.2.2", "xdg-basedir": "^5.1.0", "zod": "^4.1.8" }, "devDependencies": { "@types/picomatch": "^3.0.2", + "@types/turndown": "^5.0.6", "bun-types": "latest", "typescript": "^5.7.3" }, diff --git a/src/tools/index.ts b/src/tools/index.ts index ccfda72a..7585d325 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -20,6 +20,7 @@ import { import { grep } from "./grep" import { glob } from "./glob" import { slashcommand } from "./slashcommand" +import { webfetch } from "./webfetch" export { interactive_bash, startBackgroundCheck as startTmuxCheck } from "./interactive-bash" export { getTmuxPath } from "./interactive-bash/utils" @@ -63,4 +64,5 @@ export const builtinTools = { grep, glob, slashcommand, + webfetch, } diff --git a/src/tools/webfetch/constants.ts b/src/tools/webfetch/constants.ts new file mode 100644 index 00000000..a5872abe --- /dev/null +++ b/src/tools/webfetch/constants.ts @@ -0,0 +1,4 @@ +export const DEFAULT_STRATEGY: import("./types").CompactionStrategy = "raw" + +export const MAX_OUTPUT_SIZE = 500_000 +export const TIMEOUT_MS = 30_000 diff --git a/src/tools/webfetch/index.ts b/src/tools/webfetch/index.ts new file mode 100644 index 00000000..e098268e --- /dev/null +++ b/src/tools/webfetch/index.ts @@ -0,0 +1,3 @@ +import { webfetch } from "./tools" + +export { webfetch } diff --git a/src/tools/webfetch/strategies.ts b/src/tools/webfetch/strategies.ts new file mode 100644 index 00000000..63f0f61e --- /dev/null +++ b/src/tools/webfetch/strategies.ts @@ -0,0 +1,322 @@ +import { JSDOM } from "jsdom" +import { Readability } from "@mozilla/readability" +import TurndownService from "turndown" +import * as cheerio from "cheerio" +import type { Element } from "domhandler" +import * as jq from "jq-wasm" + +const turndown = new TurndownService({ + headingStyle: "atx", + codeBlockStyle: "fenced", +}) + +export function applyReadability(html: string, url: string): string { + const dom = new JSDOM(html, { url }) + const reader = new Readability(dom.window.document.cloneNode(true) as Document) + const article = reader.parse() + + if (!article?.content) { + return turndown.turndown(html) + } + + return turndown.turndown(article.content) +} + +export function applyRaw(content: string): string { + return content +} + +function truncateAroundMatch(line: string, pattern: RegExp, contextLength: number = 200): string { + // CRITICAL: Create fresh regex without 'g' flag - RegExp.exec with 'g' flag maintains lastIndex state + const freshPattern = new RegExp(pattern.source, pattern.flags.replace("g", "")) + const match = freshPattern.exec(line) + + if (!match) return line.length > contextLength * 2 ? line.slice(0, contextLength * 2) + "..." : line + + const matchStart = match.index + const matchEnd = matchStart + match[0].length + + const start = Math.max(0, matchStart - contextLength) + const end = Math.min(line.length, matchEnd + contextLength) + + let result = line.slice(start, end) + if (start > 0) result = "..." + result + if (end < line.length) result = result + "..." + + return result +} + +export interface GrepOptions { + limit?: number + offset?: number + before?: number + after?: number +} + +export function applyGrep(content: string, pattern: string, options: GrepOptions = {}): string { + const { limit = 100, offset = 0, before = 0, after = 0 } = options + + let regex: RegExp + try { + regex = new RegExp(pattern, "gi") + } catch { + return `Error: Invalid regex pattern: ${pattern}` + } + + const lines = content.split("\n") + const matchingIndices = new Set() + + for (let i = 0; i < lines.length; i++) { + if (regex.test(lines[i])) { + matchingIndices.add(i) + } + regex.lastIndex = 0 + } + + if (matchingIndices.size === 0) { + return `No matches found for pattern: ${pattern}` + } + + const contextIndices = new Set() + for (const idx of matchingIndices) { + for (let i = Math.max(0, idx - before); i <= Math.min(lines.length - 1, idx + after); i++) { + contextIndices.add(i) + } + } + + const sortedIndices = Array.from(contextIndices).sort((a, b) => a - b) + const paginatedIndices = sortedIndices.slice(offset, offset + limit) + + const resultLines: string[] = [] + let prevIdx = -2 + + for (const idx of paginatedIndices) { + if (prevIdx !== -2 && idx > prevIdx + 1) { + resultLines.push("--") + } + prevIdx = idx + + const line = lines[idx] + const isMatch = matchingIndices.has(idx) + const lineNum = String(idx + 1).padStart(4) + + if (isMatch) { + const truncatedLine = truncateAroundMatch(line, regex, 200) + resultLines.push(`${lineNum}:${truncatedLine}`) + } else { + const truncatedLine = line.length > 450 ? line.slice(0, 450) + "..." : line + resultLines.push(`${lineNum}-${truncatedLine}`) + } + } + + const totalMatches = matchingIndices.size + const totalWithContext = sortedIndices.length + const showing = paginatedIndices.length + + const header = [ + `Pattern: ${pattern}`, + `Matches: ${totalMatches} lines`, + before > 0 || after > 0 ? `Context: ${before} before, ${after} after (${totalWithContext} total lines)` : "", + showing < totalWithContext ? `Showing: ${offset + 1}-${offset + showing} of ${totalWithContext}` : "", + "---", + ] + .filter(Boolean) + .join("\n") + + return `${header}\n${resultLines.join("\n")}` +} + +const SEMANTIC_ELEMENTS: Record = { + h1: "heading1", + h2: "heading2", + h3: "heading3", + h4: "heading4", + h5: "heading5", + h6: "heading6", + a: "link", + button: "button", + input: "input", + select: "combobox", + textarea: "textbox", + form: "form", + nav: "navigation", + main: "main", + header: "banner", + footer: "contentinfo", + aside: "complementary", + article: "article", + section: "region", + img: "image", + table: "table", + ul: "list", + ol: "list", + li: "listitem", +} + +function getAriaRole(el: Element, $: cheerio.CheerioAPI): string | null { + const $el = $(el) + const explicitRole = $el.attr("role") + if (explicitRole) return explicitRole + + const tagName = el.tagName?.toLowerCase() + return SEMANTIC_ELEMENTS[tagName] || null +} + +function getElementLabel(el: Element, $: cheerio.CheerioAPI): string { + const $el = $(el) + const tagName = el.tagName?.toLowerCase() + + const ariaLabel = $el.attr("aria-label") + if (ariaLabel) return ariaLabel + + const title = $el.attr("title") + if (title) return title + + if (tagName === "img") { + const alt = $el.attr("alt") + if (alt) return alt + } + + if (tagName === "input") { + const placeholder = $el.attr("placeholder") + const name = $el.attr("name") + const type = $el.attr("type") || "text" + return placeholder || name || `[${type}]` + } + + const text = $el.clone().children().remove().end().text().trim() + if (text && text.length <= 100) return text + + return "" +} + +function getElementAttrs(el: Element, $: cheerio.CheerioAPI): string { + const $el = $(el) + const tagName = el.tagName?.toLowerCase() + const attrs: string[] = [] + + if (tagName === "a") { + const href = $el.attr("href") + if (href && !href.startsWith("javascript:")) { + attrs.push(`href="${href.length > 80 ? href.slice(0, 80) + "..." : href}"`) + } + } + + if (tagName === "img") { + const src = $el.attr("src") + if (src) attrs.push(`src="${src.length > 80 ? src.slice(0, 80) + "..." : src}"`) + } + + if (tagName === "input") { + const type = $el.attr("type") + if (type) attrs.push(`type="${type}"`) + } + + const id = $el.attr("id") + if (id) attrs.push(`id="${id}"`) + + return attrs.join(" ") +} + +export function applySnapshot(html: string): string { + const $ = cheerio.load(html) + + $("script, style, noscript, svg, path").remove() + + const lines: string[] = [] + + function traverse(el: Element, depth: number): void { + if (el.type !== "tag") return + + const role = getAriaRole(el, $) + if (!role) { + $(el) + .children() + .each((_, child) => traverse(child, depth)) + return + } + + const label = getElementLabel(el, $) + const attrs = getElementAttrs(el, $) + const indent = " ".repeat(depth) + + let line = `${indent}[${role}]` + if (label) line += ` "${label.length > 60 ? label.slice(0, 60) + "..." : label}"` + if (attrs) line += ` (${attrs})` + + lines.push(line) + + $(el) + .children() + .each((_, child) => traverse(child, depth + 1)) + } + + $("body") + .children() + .each((_, el) => traverse(el, 0)) + + if (lines.length === 0) { + return "No semantic elements found in page" + } + + return `Page Snapshot (${lines.length} elements)\n---\n${lines.join("\n")}` +} + +export function applySelector(html: string, selector: string): string { + const $ = cheerio.load(html) + + const elements = $(selector) + if (elements.length === 0) { + return `No elements found matching selector: ${selector}` + } + + const results: string[] = [] + + elements.each((i, el) => { + const $el = $(el) + const tagName = (el as Element).tagName?.toLowerCase() || "unknown" + + const text = $el.text().trim() + const truncatedText = text.length > 200 ? text.slice(0, 200) + "..." : text + + const attrs: string[] = [] + const href = $el.attr("href") + if (href) attrs.push(`href="${href.length > 100 ? href.slice(0, 100) + "..." : href}"`) + + const src = $el.attr("src") + if (src) attrs.push(`src="${src.length > 100 ? src.slice(0, 100) + "..." : src}"`) + + const id = $el.attr("id") + if (id) attrs.push(`id="${id}"`) + + const className = $el.attr("class") + if (className) attrs.push(`class="${className.length > 50 ? className.slice(0, 50) + "..." : className}"`) + + let line = `[${i + 1}] <${tagName}>` + if (attrs.length > 0) line += ` (${attrs.join(", ")})` + if (truncatedText) line += `\n ${truncatedText}` + + results.push(line) + }) + + return `Selector: ${selector}\nMatches: ${elements.length}\n---\n${results.join("\n\n")}` +} + +export async function applyJq(content: string, query: string): Promise { + try { + JSON.parse(content) + } catch { + return "Error: Content is not valid JSON. Use 'jq' strategy only for JSON APIs." + } + + try { + const result = await jq.raw(content, query) + if (result.exitCode !== 0) { + return `Error: jq query failed: ${result.stderr}` + } + return result.stdout.trim() + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + return `Error: Invalid jq query: ${message}` + } +} diff --git a/src/tools/webfetch/tools.ts b/src/tools/webfetch/tools.ts new file mode 100644 index 00000000..ad987c3f --- /dev/null +++ b/src/tools/webfetch/tools.ts @@ -0,0 +1,146 @@ +import { tool } from "@opencode-ai/plugin/tool" +import { DEFAULT_STRATEGY, MAX_OUTPUT_SIZE, TIMEOUT_MS } from "./constants" +import { applyReadability, applyRaw, applyGrep, applySnapshot, applySelector, applyJq, type GrepOptions } from "./strategies" +import type { CompactionStrategy } from "./types" + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes}B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB` + return `${(bytes / (1024 * 1024)).toFixed(1)}MB` +} + +async function fetchWithTimeout(url: string, timeoutMs: number): Promise { + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), timeoutMs) + + try { + const response = await fetch(url, { + signal: controller.signal, + headers: { + "User-Agent": "Mozilla/5.0 (compatible; OpenCode/1.0)", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + }, + }) + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`) + } + + return await response.text() + } finally { + clearTimeout(timeoutId) + } +} + +interface StrategyOptions extends GrepOptions { + pattern?: string + selector?: string + query?: string +} + +async function applyStrategy( + content: string, + url: string, + strategy: CompactionStrategy, + options?: StrategyOptions +): Promise { + switch (strategy) { + case "readability": + return applyReadability(content, url) + case "raw": + return applyRaw(content) + case "grep": + if (!options?.pattern) { + return "Error: 'pattern' is required for grep strategy" + } + return applyGrep(content, options.pattern, options) + case "snapshot": + return applySnapshot(content) + case "selector": + if (!options?.selector) { + return "Error: 'selector' is required for selector strategy" + } + return applySelector(content, options.selector) + case "jq": + if (!options?.query) { + return "Error: 'query' is required for jq strategy" + } + return await applyJq(content, options.query) + default: + return applyReadability(content, url) + } +} + +export const webfetch = tool({ + description: + "Fetch and process web content with compaction strategies.\n\n" + + "STRATEGY SELECTION GUIDE:\n" + + "- 'jq': Query JSON APIs with jq syntax. Best for REST APIs, npm registry, GitHub API.\n" + + "- 'readability': Extracts article content as markdown. Best for blogs, news, documentation pages.\n" + + "- 'snapshot': ARIA-like semantic tree of page structure. Best for understanding layout, forms, navigation.\n" + + "- 'selector': Extract elements matching a CSS selector. Best when you know exact element to target.\n" + + "- 'grep': Filter lines matching a pattern with optional before/after context (like grep -B/-A).\n" + + "- 'raw': No processing. Returns exact content (truncated to 500KB if larger).", + args: { + url: tool.schema.string().describe("The URL to fetch"), + strategy: tool.schema + .enum(["readability", "snapshot", "selector", "grep", "jq", "raw"]) + .optional() + .describe("Compaction strategy (default: raw)."), + pattern: tool.schema.string().optional().describe("Regex pattern for grep strategy"), + selector: tool.schema.string().optional().describe("CSS selector for selector strategy"), + query: tool.schema.string().optional().describe("jq query for JSON APIs (e.g., '.data.items[]', '.name')"), + limit: tool.schema.number().optional().describe("Max lines to return for grep (default: 100)"), + offset: tool.schema.number().optional().describe("Skip first N result lines for grep pagination"), + before: tool.schema.number().optional().describe("Lines of context before each match (like grep -B)"), + after: tool.schema.number().optional().describe("Lines of context after each match (like grep -A)"), + }, + execute: async (args) => { + const strategy = args.strategy ?? DEFAULT_STRATEGY + const url = args.url.startsWith("http") ? args.url : `https://${args.url}` + + try { + const rawContent = await fetchWithTimeout(url, TIMEOUT_MS) + const originalSize = rawContent.length + + let result = await applyStrategy(rawContent, url, strategy, { + pattern: args.pattern, + selector: args.selector, + query: args.query, + limit: args.limit, + offset: args.offset, + before: args.before, + after: args.after, + }) + + let truncated = false + if (result.length > MAX_OUTPUT_SIZE) { + result = result.slice(0, MAX_OUTPUT_SIZE) + truncated = true + } + + const compactedSize = result.length + const reduction = ((1 - compactedSize / originalSize) * 100).toFixed(1) + + const header = [ + `URL: ${url}`, + `Strategy: ${strategy}`, + `Size: ${formatBytes(originalSize)} → ${formatBytes(compactedSize)} (${reduction}% reduction)`, + truncated ? `[Output truncated to ${formatBytes(MAX_OUTPUT_SIZE)}]` : "", + "---", + ] + .filter(Boolean) + .join("\n") + + return `${header}\n\n${result}` + } catch (error) { + if (error instanceof Error) { + if (error.name === "AbortError") { + return `Error: Request timed out after ${TIMEOUT_MS / 1000}s` + } + return `Error: ${error.message}` + } + return `Error: ${String(error)}` + } + }, +}) diff --git a/src/tools/webfetch/types.ts b/src/tools/webfetch/types.ts new file mode 100644 index 00000000..ce8a0261 --- /dev/null +++ b/src/tools/webfetch/types.ts @@ -0,0 +1 @@ +export type CompactionStrategy = "raw" | "readability" | "grep" | "snapshot" | "selector" | "jq"