diff --git a/config.example.toml b/config.example.toml index ab233f3f..85ea236f 100644 --- a/config.example.toml +++ b/config.example.toml @@ -2,6 +2,11 @@ mode = "apps_script" script_id = "YOUR_APPS_SCRIPT_DEPLOYMENT_ID" auth_key = "CHANGE_ME_TO_A_STRONG_SECRET" +# Response header/connect timeout for each relay request. +request_timeout_secs = 30 +# Per-body-chunk idle timeout after headers arrive. Keep this larger than +# request_timeout_secs so slow large downloads can continue streaming. +stream_timeout_secs = 300 [network] google_ip = "216.239.38.120" diff --git a/config.full.example.toml b/config.full.example.toml index e75f0b42..a2427cb3 100644 --- a/config.full.example.toml +++ b/config.full.example.toml @@ -2,6 +2,11 @@ mode = "full" script_id = "YOUR_APPS_SCRIPT_DEPLOYMENT_ID" auth_key = "CHANGE_ME_TO_A_STRONG_SECRET" +# Response header/connect timeout for each relay request. +request_timeout_secs = 30 +# Per-body-chunk idle timeout after headers arrive. Keep this larger than +# request_timeout_secs so slow large downloads can continue streaming. +stream_timeout_secs = 300 [network] google_ip = "216.239.38.120" diff --git a/docs/guide.fa.md b/docs/guide.fa.md index d0247453..154fc56d 100644 --- a/docs/guide.fa.md +++ b/docs/guide.fa.md @@ -230,7 +230,8 @@ HTTP / HTTPS مثل قبل از Apps Script می‌رود (تغییری نمی **محافظ‌های منابع:** - **حداکثر ۵۰ op** در هر بَچ — اگر سشن‌های فعال بیشتر باشند، مالتی‌پلکسر چند بَچ می‌فرستد - **سقف payload ۴ مگابایت** در هر بَچ — خیلی کمتر از ۵۰ مگابایت Apps Script -- **timeout ۳۰ ثانیه** هر بَچ — مقصد کند / مرده نمی‌تواند سایر سشن‌ها را گیر بیاندازد +- **timeout ۳۰ ثانیه برای اتصال و هدرها** در هر بَچ — مقصد کند / مرده نمی‌تواند سایر سشن‌ها را گیر بیاندازد +- **timeout ۳۰۰ ثانیه برای هر chunk بدنه** بعد از رسیدن هدرها — دانلودهای بزرگ و کند با بودجهٔ کوتاه هدر قطع نمی‌شوند ### راه‌اندازی سریع حالت full @@ -253,6 +254,8 @@ HTTP / HTTPS مثل قبل از Apps Script می‌رود (تغییری نمی mode = "full" script_id = ["id1", "id2", "id3", "id4", "id5", "id6"] auth_key = "your-secret" + request_timeout_secs = 30 + stream_timeout_secs = 300 ``` ## Exit node @@ -368,6 +371,8 @@ sni_hosts = ["www.google.com", "drive.google.com", "docs.google.com"] | بیلد musl | OpenWRT / Alpine / محیط‌های بدون libc — باینری استاتیک، با procd init | | **Exit node** | برای سایت‌های پشت Cloudflare (v1.9.4+) | | **Unwrap goog.script.init** | دفاع‌در‌عمق در مقابل Deploymentهایی که پاسخ HtmlService-wrapped می‌فرستند (v1.9.6+) | +| دانلود بزرگ Range-aware | استریم chunk شده و resume برای درخواست‌های `Range: bytes=N-` | +| timeout جدا برای رله | `request_timeout_secs` برای اتصال/هدر و `stream_timeout_secs` برای idle هر chunk بدنه | ### عمداً پیاده نشده @@ -375,7 +380,6 @@ sni_hosts = ["www.google.com", "drive.google.com", "docs.google.com"] |---|---| | HTTP/2 multiplexing | state machine کریت `h2` (stream IDs، flow control، GOAWAY) موارد hang ظریف زیادی دارد؛ coalescing + pool ۲۰-conn بیشتر فایده را می‌گیرد | | Batch (`q:[...]` در apps_script) | connection pool + tokio async از قبل خوب موازی‌سازی می‌کند؛ batch ~۲۰۰ خط مدیریت state اضافه می‌کند با سود نامشخص | -| Range-based parallel download | edge case‌های واقعی (سرورهای بدون Range، chunked وسط stream)؛ ویدیوی یوتیوب از قبل با تونل بازنویسی SNI، Apps Script را دور می‌زند | | حالت‌های `domain_fronting` / `google_fronting` / `custom_domain` | Cloudflare در ۲۰۲۴ domain fronting عمومی را کشت؛ Cloud Run پلن پولی می‌خواهد | ## محدودیت‌های شناخته‌شده @@ -394,6 +398,10 @@ HTML یوتیوب سریع می‌آید (از تونل بازنویسی SNI)، برای مرور متنی خوب است، برای ۱۰۸۰p دردناک. چند `script_id` بچرخان برای هد روم بیشتر، یا VPN واقعی برای ویدیو. +### دانلودهای بزرگ هنوز سهمیه مصرف می‌کنند + +رله می‌تواند پاسخ‌های Range-capable را chunk شده استریم کند و وقتی کلاینت با `Range: bytes=N-` ادامه می‌دهد، resume تمیز داشته باشد. اما هر chunk هنوز یک اجرای `UrlFetchApp` است. `request_timeout_secs` فقط اتصال و رسیدن هدرها را کنترل می‌کند؛ `stream_timeout_secs` سکوت بین chunkهای بدنه را. + ### Brotli حذف می‌شود از هدر `Accept-Encoding` ‏`br` حذف می‌شود. Apps Script gzip را decompress می‌کند ولی Brotli نه؛ forward کردن `br` پاسخ را خراب می‌کند. سربار حجمی جزئی. diff --git a/docs/guide.md b/docs/guide.md index 679a35d0..f21a1740 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -230,7 +230,8 @@ More deployments = more total concurrency = lower per-session latency. Each batc **Resource guards:** - **50 ops max** per batch — if more sessions are active, the mux splits into multiple batches - **4 MB payload cap** per batch — well under Apps Script's 50 MB limit -- **30 s timeout** per batch — slow / dead targets can't block other sessions forever +- **30 s header/connect timeout** per batch — slow / dead targets can't block other sessions forever +- **300 s per-chunk stream timeout** after headers arrive — slow large downloads can keep moving without being killed by the shorter header budget ### Full mode quick start @@ -253,6 +254,8 @@ More deployments = more total concurrency = lower per-session latency. Each batc mode = "full" script_id = ["id1", "id2", "id3", "id4", "id5", "id6"] auth_key = "your-secret" + request_timeout_secs = 30 + stream_timeout_secs = 300 ``` ## Exit node @@ -364,12 +367,13 @@ This port focuses on the **`apps_script` mode** — the only one that reliably w - [x] OpenWRT / Alpine / musl builds — static binaries, procd init script included - [x] **Exit node** support for Cloudflare-fronted sites (v1.9.4+) - [x] **Goog.script.init iframe unwrap** — defense-in-depth against deployments that return HtmlService-wrapped responses (v1.9.6+) +- [x] Range-aware large download streaming with resume support for `Range: bytes=N-` requests +- [x] Separate relay header/connect timeout and per-chunk body idle timeout Intentionally **not** implemented: - **HTTP/2 multiplexing** — `h2` crate state machine has too many subtle hang cases; coalescing + 20-conn pool gets most of the benefit - **Request batching (`q:[...]` mode in apps_script mode)** — connection pool + tokio async already parallelizes well; batching adds ~200 lines of state for unclear gain -- **Range-based parallel download** — edge cases real (non-Range servers, chunked mid-stream); YouTube already bypasses Apps Script via SNI-rewrite tunnel - **Other modes** (`domain_fronting`, `google_fronting`, `custom_domain`) — Cloudflare killed generic domain fronting in 2024; Cloud Run needs a paid plan ## Known limitations @@ -378,6 +382,7 @@ These are inherent to the Apps Script + domain-fronting approach, not bugs in th - **User-Agent fixed to `Google-Apps-Script`** for traffic through the relay. `UrlFetchApp.fetch()` doesn't allow override. Sites that detect bots (Google search, some CAPTCHAs) serve degraded / no-JS pages. Workaround: add the affected domain to the `hosts` map so it's routed through the SNI-rewrite tunnel with your real browser's UA. `google.com`, `youtube.com`, `fonts.googleapis.com` are already there. - **Video playback slow and quota-limited** for anything through the relay. YouTube HTML loads fast (SNI-rewrite tunnel), but `googlevideo.com` chunks go through Apps Script. Free tier: ~20k `UrlFetchApp` calls / day, 50 MB body cap per fetch. Fine for text browsing, painful for 1080p. Rotate multiple `script_id`s for headroom, or use a real VPN for video. +- **Large downloads still consume Apps Script calls.** The relay can stream range-capable responses in chunks and resume cleanly when clients retry with `Range: bytes=N-`, but every chunk is still a `UrlFetchApp` invocation. `request_timeout_secs` controls connection/headers; `stream_timeout_secs` controls idle time between body chunks. - **Brotli stripped** from forwarded `Accept-Encoding`. Apps Script can decompress gzip but not `br`; forwarding `br` would garble responses. Minor size overhead. - **WebSockets don't work** through the relay — it's request / response JSON. Sites that upgrade to WS fail (ChatGPT streaming, Discord voice, etc.). - **HSTS-preloaded / hard-pinned sites** reject the MITM cert. Most sites are fine; a handful aren't. diff --git a/src/bin/ui.rs b/src/bin/ui.rs index c5f9ed63..ccde67fc 100644 --- a/src/bin/ui.rs +++ b/src/bin/ui.rs @@ -286,14 +286,16 @@ struct FormState { /// there is no UI editor for these yet, only file-edited config. /// See config.rs `fronting_groups`. fronting_groups: Vec, - /// Auto-blacklist tuning + per-batch timeout. Config-only knobs (no UI + /// Auto-blacklist tuning + relay timeouts. Config-only knobs (no UI /// fields yet — power-user file edit). Round-tripped through FormState /// so Save preserves the user's hand-edited values. See config.rs - /// `auto_blacklist_*` and `request_timeout_secs`. + /// `auto_blacklist_*`, `request_timeout_secs`, and + /// `stream_timeout_secs`. auto_blacklist_strikes: u32, auto_blacklist_window_secs: u64, auto_blacklist_cooldown_secs: u64, request_timeout_secs: u64, + stream_timeout_secs: u64, /// Optional second-hop exit node for CF-anti-bot bypass (chatgpt.com / /// claude.ai / grok.com / x.com). Config-only — no UI editor yet. /// See `assets/exit_node/` for the generic exit-node handler. @@ -391,6 +393,7 @@ fn load_form() -> (FormState, Option) { auto_blacklist_window_secs: c.auto_blacklist_window_secs, auto_blacklist_cooldown_secs: c.auto_blacklist_cooldown_secs, request_timeout_secs: c.request_timeout_secs, + stream_timeout_secs: c.stream_timeout_secs, exit_node: c.exit_node.clone(), } } else { @@ -427,12 +430,14 @@ fn load_form() -> (FormState, Option) { bypass_doh_hosts: Vec::new(), block_doh: true, fronting_groups: Vec::new(), - // Defaults match `default_auto_blacklist_*` and - // `default_request_timeout_secs` in src/config.rs. + // Defaults match `default_auto_blacklist_*`, + // `default_request_timeout_secs`, and + // `default_stream_timeout_secs` in src/config.rs. auto_blacklist_strikes: 3, auto_blacklist_window_secs: 30, auto_blacklist_cooldown_secs: 120, request_timeout_secs: 30, + stream_timeout_secs: 300, exit_node: mhrv_rs::config::ExitNodeConfig::default(), } }; @@ -610,7 +615,7 @@ impl FormState { // batch alongside the system-proxy toggle (#432). coalesce_step_ms: 0, coalesce_max_ms: 0, - // Auto-blacklist + batch timeout: config-only knobs (#391, + // Auto-blacklist + relay timeouts: config-only knobs (#391, // #444, #430). Round-trip through FormState so Save doesn't // drop hand-edited values. UI editor planned alongside the // v1.8.x desktop UI batch. @@ -618,6 +623,7 @@ impl FormState { auto_blacklist_window_secs: self.auto_blacklist_window_secs, auto_blacklist_cooldown_secs: self.auto_blacklist_cooldown_secs, request_timeout_secs: self.request_timeout_secs, + stream_timeout_secs: self.stream_timeout_secs, // Exit-node config (CF-anti-bot bypass for chatgpt.com / claude.ai // / grok.com / x.com). Round-trip through FormState — config-only // editing for now, UI editor planned for v1.9.x desktop UI batch. diff --git a/src/config.rs b/src/config.rs index cd63b8b8..498d5868 100644 --- a/src/config.rs +++ b/src/config.rs @@ -376,9 +376,22 @@ pub struct Config { /// retry sooner when a deployment hangs. Floor `5`, ceiling `300` /// (anything beyond exceeds Apps Script's hard 6-min cap with /// no benefit). + /// + /// This applies to connection establishment and response header + /// arrival only. Body streaming is governed by `stream_timeout_secs`. #[serde(default = "default_request_timeout_secs")] pub request_timeout_secs: u64, + /// Per-chunk body streaming idle timeout (seconds). Default `300`. + /// Applies to each individual body chunk read after headers arrive — + /// a chunk that goes silent for longer than this is considered a + /// stalled connection and the request is aborted. Distinct from + /// `request_timeout_secs` so large responses through Apps Script + /// (where each 256 KB range chunk can take 30-90s) are not killed + /// mid-transfer. Floor `10`, ceiling `3600`. + #[serde(default = "default_stream_timeout_secs")] + pub stream_timeout_secs: u64, + /// Optional second-hop exit node, for sites that block traffic /// from Google datacenter IPs (Apps Script's outbound IP space). /// Most visibly: Cloudflare-fronted services that flag the GCP IP @@ -531,6 +544,10 @@ fn default_auto_blacklist_cooldown_secs() -> u64 { 120 } /// hard-coded `BATCH_TIMEOUT` and Apps Script's typical response cliff. fn default_request_timeout_secs() -> u64 { 30 } +/// Default for `stream_timeout_secs`: 300s per-chunk idle timeout for +/// body streaming, separate from the header/connect timeout. +fn default_stream_timeout_secs() -> u64 { 300 } + fn default_google_ip() -> String { "216.239.38.120".into() } @@ -766,6 +783,8 @@ pub struct TomlRelay { pub auto_blacklist_cooldown_secs: u64, #[serde(default = "default_request_timeout_secs")] pub request_timeout_secs: u64, + #[serde(default = "default_stream_timeout_secs")] + pub stream_timeout_secs: u64, } /// [network] section of config.toml. @@ -919,6 +938,7 @@ impl From for Config { auto_blacklist_window_secs: t.relay.auto_blacklist_window_secs, auto_blacklist_cooldown_secs: t.relay.auto_blacklist_cooldown_secs, request_timeout_secs: t.relay.request_timeout_secs, + stream_timeout_secs: t.relay.stream_timeout_secs, exit_node: t.exit_node, } } @@ -946,6 +966,7 @@ impl From<&Config> for TomlConfig { auto_blacklist_window_secs: c.auto_blacklist_window_secs, auto_blacklist_cooldown_secs: c.auto_blacklist_cooldown_secs, request_timeout_secs: c.request_timeout_secs, + stream_timeout_secs: c.stream_timeout_secs, }, network: TomlNetwork { google_ip: c.google_ip.clone(), @@ -1211,7 +1232,9 @@ mod rt_tests { "fetch_ips_from_api": true, "max_ips_to_scan": 50, "scan_batch_size": 100, - "google_ip_validation": true + "google_ip_validation": true, + "request_timeout_secs": 45, + "stream_timeout_secs": 600 }"#; let tmp = std::env::temp_dir().join("mhrv-rt-test.json"); std::fs::write(&tmp, json).unwrap(); @@ -1221,6 +1244,8 @@ mod rt_tests { assert_eq!(cfg.listen_port, 8085); assert_eq!(cfg.upstream_socks5.as_deref(), Some("127.0.0.1:50529")); assert_eq!(cfg.parallel_relay, 2); + assert_eq!(cfg.request_timeout_secs, 45); + assert_eq!(cfg.stream_timeout_secs, 600); assert_eq!( cfg.sni_hosts.as_ref().unwrap(), &vec!["www.google.com".to_string(), "drive.google.com".to_string()] @@ -1346,6 +1371,38 @@ hosts = ["claude.ai", "chatgpt.com"] assert_eq!(cfg.exit_node.hosts, vec!["claude.ai", "chatgpt.com"]); } + #[test] + fn toml_parses_separate_header_and_stream_timeouts() { + let s = r#" +[relay] +mode = "apps_script" +auth_key = "SECRET" +script_id = "X" +request_timeout_secs = 45 +stream_timeout_secs = 900 +"#; + let toml_cfg: TomlConfig = toml::from_str(s).unwrap(); + let cfg = Config::from(toml_cfg); + assert_eq!(cfg.request_timeout_secs, 45); + assert_eq!(cfg.stream_timeout_secs, 900); + cfg.validate().unwrap(); + } + + #[test] + fn toml_defaults_stream_timeout_when_omitted() { + let s = r#" +[relay] +mode = "apps_script" +auth_key = "SECRET" +script_id = "X" +"#; + let toml_cfg: TomlConfig = toml::from_str(s).unwrap(); + let cfg = Config::from(toml_cfg); + assert_eq!(cfg.request_timeout_secs, 30); + assert_eq!(cfg.stream_timeout_secs, 300); + cfg.validate().unwrap(); + } + #[test] fn toml_parses_fronting_groups_array_of_tables() { let s = r#" @@ -1450,4 +1507,4 @@ script_id = "ABCDEF" let _ = std::fs::remove_file(&json_path); let _ = std::fs::remove_file(&toml_path); } -} \ No newline at end of file +} diff --git a/src/domain_fronter.rs b/src/domain_fronter.rs index 0e11e764..6d810569 100644 --- a/src/domain_fronter.rs +++ b/src/domain_fronter.rs @@ -423,7 +423,12 @@ pub struct DomainFronter { /// Per-batch HTTP timeout. Mirrors `Config::request_timeout_secs` /// (#430, masterking32 PR #25). Read by `tunnel_client::fire_batch` /// so a single config field tunes the timeout used everywhere. + /// Applies to connection establishment and response header arrival only. batch_timeout: Duration, + /// Per-chunk body streaming idle timeout. Mirrors `Config::stream_timeout_secs`. + /// Applied per-iteration of the body drain loop so large responses + /// through Apps Script are not killed mid-transfer by `batch_timeout`. + stream_timeout: Duration, /// Optional second-hop exit node (Deno Deploy / fly.io / etc.) /// to bypass CF-anti-bot blocks on sites that flag Google datacenter /// IPs (chatgpt.com, claude.ai, grok.com, x.com). Mirrors @@ -642,6 +647,9 @@ impl DomainFronter { batch_timeout: Duration::from_secs( config.request_timeout_secs.clamp(5, 300), ), + stream_timeout: Duration::from_secs( + config.stream_timeout_secs.clamp(10, 3600), + ), exit_node_enabled: config.exit_node.enabled && !config.exit_node.relay_url.is_empty() && !config.exit_node.psk.is_empty(), @@ -697,6 +705,11 @@ impl DomainFronter { self.batch_timeout } + /// Per-chunk body streaming idle timeout. Clamped to `[10s, 3600s]`. + pub(crate) fn stream_timeout(&self) -> Duration { + self.stream_timeout + } + /// Record one relay call toward the daily budget. Called once per /// outbound Apps Script fetch. Rolls over both daily counters at /// 00:00 Pacific Time, matching Apps Script's quota reset cadence @@ -1533,18 +1546,17 @@ impl DomainFronter { })?; } - // Phase 2: response headers + body drain. Bounded by the - // caller's deadline. Errors and timeout here are - // `RequestSent::Maybe` — the request is on the wire and may - // already have side effects. - let response_phase = async { + // Phase 2a: wait for response headers. Bounded by the caller's + // deadline (`batch_timeout` / `request_timeout_secs`). A timeout + // here means the relay never responded — safe to retry. + let header_phase = async { let response = response_fut.await.map_err(|e| { ( FronterError::Relay(format!("h2 response: {}", e)), RequestSent::Maybe, ) })?; - let (parts, mut body) = response.into_parts(); + let (parts, body) = response.into_parts(); let status = parts.status.as_u16(); // Convert headers to the (String, String) Vec the rest of @@ -1557,27 +1569,12 @@ impl DomainFronter { headers.push((name.as_str().to_string(), v.to_string())); } } - - // Drain body. Release flow-control credit per chunk so - // large responses don't stall after the initial 4 MB window. - let mut buf: Vec = Vec::new(); - while let Some(chunk) = body.data().await { - let chunk = chunk.map_err(|e| { - ( - FronterError::Relay(format!("h2 body chunk: {}", e)), - RequestSent::Maybe, - ) - })?; - let n = chunk.len(); - buf.extend_from_slice(&chunk); - let _ = body.flow_control().release_capacity(n); - } - Ok::<_, (FronterError, RequestSent)>((status, headers, buf)) + Ok::<_, (FronterError, RequestSent)>((status, headers, body)) }; - let (status, headers, mut buf) = match tokio::time::timeout( + let (status, headers, mut body) = match tokio::time::timeout( response_deadline, - response_phase, + header_phase, ) .await { @@ -1586,6 +1583,32 @@ impl DomainFronter { Err(_) => return Err((FronterError::Timeout, RequestSent::Maybe)), }; + // Phase 2b: drain body. Each chunk is individually bounded by + // `stream_timeout` (default 300s) so large responses routed + // through Apps Script (where a 256 KB range chunk can take 30-90s + // of wall-clock time) are not killed by the tighter `batch_timeout`. + // Release flow-control credit per chunk so large responses don't + // stall after the initial 4 MB window. + let stream_timeout = self.stream_timeout(); + let mut buf: Vec = Vec::new(); + loop { + match tokio::time::timeout(stream_timeout, body.data()).await { + Ok(None) => break, + Ok(Some(Ok(chunk))) => { + let n = chunk.len(); + buf.extend_from_slice(&chunk); + let _ = body.flow_control().release_capacity(n); + } + Ok(Some(Err(e))) => { + return Err(( + FronterError::Relay(format!("h2 body chunk: {}", e)), + RequestSent::Maybe, + )); + } + Err(_) => return Err((FronterError::Timeout, RequestSent::Maybe)), + } + } + // Mirror `read_http_response`: if the server gzipped the body // (we asked for it via accept-encoding), decompress before // handing back so downstream JSON / envelope parsers see plain @@ -1785,20 +1808,34 @@ impl DomainFronter { return bytes; } Err(e) if !e.is_retryable() => { - // The exit node may have already processed this - // request (h2 post-send failure on a POST etc.). - // Don't fall through to the direct path — that - // would re-send to the same destination via Apps - // Script and duplicate the side effect. - tracing::warn!( - "exit node failed for {} and request was already sent ({}); not falling back to direct Apps Script", - url, - e, - ); - self.relay_failures.fetch_add(1, Ordering::Relaxed); - let inner = e.into_inner(); - self.record_site(url, false, 0, t0.elapsed().as_nanos() as u64); - return error_response(502, &format!("Relay error: {}", inner)); + // The NonRetryable guard exists to prevent duplicate + // side-effects on POST/PUT/PATCH/DELETE: if the h2 + // outer call reached Apps Script and timed out, the + // inner request may have already been executed by the + // exit node. Falling through would re-send it. + // + // For idempotent methods (GET/HEAD/OPTIONS) there are + // no side-effects, so re-sending via direct Apps Script + // is always safe. Range downloads are GET — if a script + // ID hits its 6-minute cap and times out, falling back + // to direct Apps Script (round-robining to a fresh ID) + // is the correct behaviour rather than returning 502. + if is_method_safe_for_fanout(method) { + tracing::warn!( + "exit node non-retryable timeout for {} {} — method is idempotent, falling back to direct Apps Script", + method, url, + ); + // fall through to the regular relay path below + } else { + tracing::warn!( + "exit node failed for {} {} and request was already sent ({}); not falling back to direct Apps Script", + method, url, e, + ); + self.relay_failures.fetch_add(1, Ordering::Relaxed); + let inner = e.into_inner(); + self.record_site(url, false, 0, t0.elapsed().as_nanos() as u64); + return error_response(502, &format!("Relay error: {}", inner)); + } } Err(e) => { tracing::warn!( @@ -1983,10 +2020,53 @@ impl DomainFronter { let raw = self.relay(method, url, headers, body).await; return write_response_with_head_transform(writer, &raw, &transform_head).await; } - // If the client already sent a Range header, honour it as-is — - // don't second-guess a caller that knows what bytes they want. - if headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("range")) { + // If the client already sent a Range header, inspect it: + // + // • bytes=N- or bytes=N-M with N>0 (resume / mid-file seek): route + // through the parallel chunk path starting at offset N. Passing the + // raw header to relay() would ask Apps Script to return everything + // from byte N to EOF in one call — for a 3 GiB file that's well + // over Apps Script's 50 MiB response cap, guaranteed 504 every try. + // + // • bytes=0-M (small specific range from the start): pass through + // to relay() as-is. On relay failure close cleanly so the client + // retries with its Range intact rather than restarting from byte 0. + if let Some(range_val) = headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("range")) + .map(|(_, v)| v.clone()) + { + if let Some(start) = parse_range_start(&range_val).filter(|&s| s > 0) { + tracing::debug!( + "range-parallel-resume: client Range {} for {}; probing from offset {}", + range_val, url, start, + ); + return self + .stream_range_from_offset( + writer, + method, + url, + headers, + body, + start, + chunk, + transform_head, + ) + .await; + } + // start == 0 or unparseable — honour as-is with clean-close on failure. let raw = self.relay(method, url, headers, body).await; + let status = split_response(&raw).map(|(s, _, _)| s).unwrap_or(0); + if status >= 400 || status == 0 { + tracing::warn!( + "range relay returned status {} for request {}; closing cleanly so client retries with Range", + status, url, + ); + return Err(std::io::Error::other(format!( + "range relay status {} — closing for clean resume", + status + ))); + } return write_response_with_head_transform(writer, &raw, &transform_head).await; } @@ -2193,6 +2273,119 @@ impl DomainFronter { write_response_with_head_transform(writer, &raw, &transform_head).await } + /// Resume a large download from a byte offset by probing at + /// `[start, start+chunk-1]` and streaming the remaining chunks in + /// parallel — exactly like the initial download path but starting + /// mid-file. Called when the client sends `Range: bytes=N-` with + /// N > 0 (wget `-c`, browser resume). Responds with `206 Partial + /// Content` so the client appends to its existing partial file. + async fn stream_range_from_offset( + &self, + writer: &mut W, + method: &str, + url: &str, + client_headers: &[(String, String)], + body: &[u8], + start: u64, + chunk: u64, + transform_head: &F, + ) -> std::io::Result<()> + where + W: tokio::io::AsyncWrite + Unpin, + F: Fn(&[u8]) -> Vec, + { + const MAX_PARALLEL: usize = 16; + + // Strip client's Range header; add our probe range [start, start+chunk-1]. + let mut probe_headers: Vec<(String, String)> = client_headers + .iter() + .filter(|(k, _)| !k.eq_ignore_ascii_case("range")) + .cloned() + .collect(); + probe_headers.push(( + "Range".into(), + format!("bytes={}-{}", start, start + chunk - 1), + )); + + let first = self.relay(method, url, &probe_headers, body).await; + let (status, resp_headers, resp_body) = match split_response(&first) { + Some(v) => v, + None => { + tracing::warn!( + "range-parallel-resume: malformed probe response for {}; closing cleanly", + url + ); + return Err(std::io::Error::other( + "range-parallel-resume: malformed probe — closing for clean resume", + )); + } + }; + + if status != 206 { + if status >= 400 { + tracing::warn!( + "range-parallel-resume: probe returned {} for {}; closing cleanly", + status, url, + ); + return Err(std::io::Error::other(format!( + "range-parallel-resume: probe status {} — closing for clean resume", + status, + ))); + } + // Non-206 success (origin sent 200 for the full body) — forward as-is. + return write_response_with_head_transform(writer, &first, transform_head).await; + } + + let probe_range = + match validate_probe_range_at_offset(status, &resp_headers, resp_body, start, start + chunk - 1) + { + Some(r) => r, + None => { + tracing::warn!( + "range-parallel-resume: invalid 206 for {}; closing cleanly", + url, + ); + return Err(std::io::Error::other( + "range-parallel-resume: invalid 206 — closing for clean resume", + )); + } + }; + let total = probe_range.total; + + // Probe covered the rest of the file — forward this 206 as-is. + if (probe_range.end + 1) >= total { + return write_response_with_head_transform(writer, &first, transform_head).await; + } + + let probe_end = probe_range.end; + let body_total = total - start; + let expected_chunks = (total - probe_end - 1).div_ceil(chunk); + tracing::info!( + "range-parallel-resume: {} total, resuming from byte {}, {} more chunks after probe, up to {} in flight for {}", + total, start, expected_chunks, MAX_PARALLEL, url, + ); + + // base_headers for fetch_chunks_stream must not include Range + // (fetch_chunks_stream adds its own per-chunk Range header). + let base_headers: Vec<(String, String)> = client_headers + .iter() + .filter(|(k, _)| !k.eq_ignore_ascii_case("range")) + .cloned() + .collect(); + + let fetches = self.fetch_chunks_stream( + url, + &base_headers, + plan_remaining_ranges(probe_end, total, chunk), + total, + MAX_PARALLEL, + ); + + let head = assemble_206_head(&resp_headers, start, total); + let head = transform_head(&head); + stream_chunks_to_writer(writer, &head, resp_body, body_total, fetches, url).await + } + /// Backward-compatible wrapper around `relay_parallel_range_to` /// that buffers the full response into a `Vec` before /// returning. Retained so downstream callers (and external @@ -3408,6 +3601,34 @@ fn validate_probe_range( None } +/// Parse the start byte from a `Range: bytes=N-` or `Range: bytes=N-M` header value. +fn parse_range_start(range_header: &str) -> Option { + let s = range_header.trim().strip_prefix("bytes=")?; + s.split('-').next()?.trim().parse::().ok() +} + +/// Variant of `validate_probe_range` for mid-file resume probes where +/// `Content-Range: bytes N-M/total` has a non-zero start. +fn validate_probe_range_at_offset( + status: u16, + headers: &[(String, String)], + body: &[u8], + req_start: u64, + req_end: u64, +) -> Option { + if status != 206 { + return None; + } + let range = parse_content_range(headers)?; + if range.start != req_start || range.end > req_end { + return None; + } + if content_range_matches_body(range, body.len()) { + return Some(range); + } + None +} + fn probe_range_covers_complete_entity(range: ContentRange, requested_end: u64) -> bool { // Apps Script may decode a gzip body while preserving the origin's // compressed Content-Range. For the synthetic first probe only, a @@ -3497,6 +3718,46 @@ fn assemble_200_head(src_headers: &[(String, String)], declared_length: u64) -> out } +/// Build a `HTTP/1.1 206 Partial Content` head for the resume streaming +/// path. `start` is the first byte the client requested; `total` is the +/// full file size reported by the origin's `Content-Range`. Mirrors +/// `assemble_200_head`'s header-skip rules. +fn assemble_206_head(src_headers: &[(String, String)], start: u64, total: u64) -> Vec { + let skip = |k: &str| { + matches!( + k.to_ascii_lowercase().as_str(), + "content-length" + | "content-range" + | "content-encoding" + | "transfer-encoding" + | "connection" + | "keep-alive", + ) + }; + let length = total.saturating_sub(start); + let mut out: Vec = b"HTTP/1.1 206 Partial Content\r\n".to_vec(); + for (k, v) in src_headers { + if skip(k) { + continue; + } + out.extend_from_slice(k.as_bytes()); + out.extend_from_slice(b": "); + out.extend_from_slice(v.as_bytes()); + out.extend_from_slice(b"\r\n"); + } + out.extend_from_slice( + format!( + "Content-Range: bytes {}-{}/{}\r\nContent-Length: {}\r\n\r\n", + start, + total - 1, + total, + length, + ) + .as_bytes(), + ); + out +} + /// Apply `transform_head` to the head block of an HTTP/1.x response /// (everything up to and including the first `\r\n\r\n` terminator), /// then write the transformed head followed by the unchanged body to @@ -5542,6 +5803,47 @@ Content-Length: 45812\r\n\r\n" assert!(validate_probe_range(206, &headers, b"hey", 4).is_none()); } + #[test] + fn parse_range_start_accepts_resume_ranges() { + assert_eq!(parse_range_start("bytes=123-"), Some(123)); + assert_eq!(parse_range_start("bytes=123-456"), Some(123)); + assert_eq!(parse_range_start(" bytes=0-999 "), Some(0)); + assert_eq!(parse_range_start("items=123-456"), None); + assert_eq!(parse_range_start("bytes=-500"), None); + } + + #[test] + fn validate_probe_range_at_offset_accepts_exact_resume_probe() { + let headers = vec![( + "Content-Range".to_string(), + "bytes 262144-524287/1048576".to_string(), + )]; + let body = vec![0u8; 262144]; + let range = validate_probe_range_at_offset(206, &headers, &body, 262144, 524287) + .expect("exact resume probe must validate"); + + assert_eq!( + range, + ContentRange { + start: 262144, + end: 524287, + total: 1048576, + } + ); + } + + #[test] + fn validate_probe_range_at_offset_rejects_wrong_start_or_body_len() { + let headers = vec![( + "Content-Range".to_string(), + "bytes 262144-524287/1048576".to_string(), + )]; + let body = vec![0u8; 262144]; + + assert!(validate_probe_range_at_offset(206, &headers, &body, 0, 524287).is_none()); + assert!(validate_probe_range_at_offset(206, &headers, b"short", 262144, 524287).is_none()); + } + #[test] fn extract_exact_range_body_rejects_body_length_mismatch() { let raw = b"HTTP/1.1 206 Partial Content\r\n\ diff --git a/src/proxy_server.rs b/src/proxy_server.rs index 209bbc58..26b07194 100644 --- a/src/proxy_server.rs +++ b/src/proxy_server.rs @@ -2185,6 +2185,12 @@ async fn run_mitm_then_relay( } } } + // Always send TLS close_notify so the client gets a clean EOF. + // Without this, dropping `tls` mid-stream (e.g. after a partial + // range-parallel response) causes wget/curl to report + // "TLS connection was non-properly terminated" rather than a + // clean truncation they can resume from. + let _ = tls.shutdown().await; } /// True if `s` parses as an IPv4 or IPv6 literal. Used to decide whether