From 12c98e2f46486c1de46a2eaf9578b2a2510b6d79 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 6 Nov 2026 19:24:39 +0800 Subject: [PATCH 1/3] fix: keep the remainder of a Google Drive upload chunk Encrypted dumps arrive in 16 MiB pieces while the resumable window is 8 MiB, and the unread tail was discarded. Shared Drive writes also need supportsAllDrives on the request URL, which Google does not read from the JSON body. --- .../storage/providers/google_drive/helpers.rs | 123 ++++++++++++++---- 1 file changed, 100 insertions(+), 23 deletions(-) diff --git a/src/services/storage/providers/google_drive/helpers.rs b/src/services/storage/providers/google_drive/helpers.rs index 688bfa43..2e0df882 100644 --- a/src/services/storage/providers/google_drive/helpers.rs +++ b/src/services/storage/providers/google_drive/helpers.rs @@ -1,15 +1,66 @@ use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; -use anyhow::{Context, Result, anyhow}; +use anyhow::{anyhow, Context, Result}; use bytes::Bytes; use futures::Stream; use futures::StreamExt; use oauth2::{ - AuthUrl, ClientId, ClientSecret, RefreshToken, TokenResponse, TokenUrl, basic::BasicClient, - reqwest::Client as OAuth2ReqwestClient, + basic::BasicClient, reqwest::Client as OAuth2ReqwestClient, AuthUrl, ClientId, ClientSecret, + RefreshToken, TokenResponse, TokenUrl, }; +use reqwest::{header, Client}; use reqwest::{Client as ReqwestClient, StatusCode}; -use reqwest::{Client, header}; -use serde_json::{Value, json}; +use serde_json::{json, Value}; + +pub const DRIVE_FILES_URL: &str = + "https://www.googleapis.com/drive/v3/files?supportsAllDrives=true"; +pub const DRIVE_RESUMABLE_UPLOAD_URL: &str = + "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true"; + +pub fn append_from_pending(buf: &mut Vec, pending: &mut Bytes, remaining: &mut u64) { + if pending.is_empty() || *remaining == 0 { + return; + } + let to_take = (*remaining as usize).min(pending.len()); + buf.extend_from_slice(&pending[..to_take]); + *pending = pending.slice(to_take..); + *remaining -= to_take as u64; +} + +pub fn collect_windows(chunks: Vec, total: u64, window: u64) -> Result>> { + let mut pending = Bytes::new(); + let mut iter = chunks.into_iter(); + let mut uploaded = 0u64; + let mut windows = Vec::new(); + + while uploaded < total { + let chunk_size = (total - uploaded).min(window); + let mut buf = Vec::with_capacity(chunk_size as usize); + let mut remaining = chunk_size; + + while remaining > 0 { + if pending.is_empty() { + match iter.next() { + Some(next) => pending = next, + None => { + if uploaded + buf.len() as u64 != total { + return Err(anyhow!("Stream ended early")); + } + break; + } + } + } + append_from_pending(&mut buf, &mut pending, &mut remaining); + } + + if buf.is_empty() && uploaded < total { + return Err(anyhow!("Unexpected end of stream")); + } + uploaded += buf.len() as u64; + windows.push(buf); + } + + Ok(windows) +} pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Result { let http_client = OAuth2ReqwestClient::new(); @@ -80,11 +131,10 @@ pub async fn ensure_folder_path( "name": name, "mimeType": "application/vnd.google-apps.folder", "parents": [parent_id], - "supportsAllDrives": true, }); let folder = client - .post("https://www.googleapis.com/drive/v3/files") + .post(DRIVE_FILES_URL) .bearer_auth(&token) .json(&create_payload) .send() @@ -171,11 +221,10 @@ pub async fn upload_stream_to_google_drive( "name": file_name, "parents": [folder_id], "mimeType": mime, - "supportsAllDrives": true, }); let session_res = client - .post("https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable") + .post(DRIVE_RESUMABLE_UPLOAD_URL) .bearer_auth(&token) .header("X-Upload-Content-Type", mime) .header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot @@ -199,6 +248,7 @@ pub async fn upload_stream_to_google_drive( const CHUNK_SIZE: u64 = 8 * 1024 * 1024; let mut uploaded: u64 = 0; + let mut pending = Bytes::new(); while uploaded < total_size { let chunk_size = (total_size - uploaded).min(CHUNK_SIZE); @@ -207,23 +257,19 @@ pub async fn upload_stream_to_google_drive( let mut remaining = chunk_size; while remaining > 0 { - match content_stream.next().await { - Some(Ok(bytes)) => { - let to_take = remaining.min(bytes.len() as u64) as usize; - chunk_bytes.extend_from_slice(&bytes[..to_take]); - remaining -= to_take as u64; - - if to_take < bytes.len() { - // TODO : Put remainder back - } - } - Some(Err(e)) => return Err(e).context("Stream error during chunk"), - None => { - if uploaded + chunk_bytes.len() as u64 != total_size { - return Err(anyhow::anyhow!("Stream ended early")); + if pending.is_empty() { + match content_stream.next().await { + Some(Ok(bytes)) => pending = bytes, + Some(Err(e)) => return Err(e).context("Stream error during chunk"), + None => { + if uploaded + chunk_bytes.len() as u64 != total_size { + return Err(anyhow::anyhow!("Stream ended early")); + } + break; } } } + append_from_pending(&mut chunk_bytes, &mut pending, &mut remaining); } if chunk_bytes.is_empty() && uploaded < total_size { @@ -279,3 +325,34 @@ pub async fn upload_stream_to_google_drive( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sixteen_mib_chunk_survives_eight_mib_windows() { + let window = 8 * 1024 * 1024; + let payload = vec![7u8; 16 * 1024 * 1024]; + let windows = collect_windows( + vec![Bytes::from(payload.clone())], + payload.len() as u64, + window, + ) + .expect("windows"); + + assert_eq!(windows.len(), 2); + assert_eq!(windows[0].len(), window as usize); + assert_eq!(windows[1].len(), window as usize); + let mut joined = windows[0].clone(); + joined.extend_from_slice(&windows[1]); + assert_eq!(joined, payload); + } + + #[test] + fn drive_write_urls_carry_supports_all_drives() { + assert!(DRIVE_RESUMABLE_UPLOAD_URL.contains("supportsAllDrives=true")); + assert!(DRIVE_RESUMABLE_UPLOAD_URL.contains("uploadType=resumable")); + assert!(DRIVE_FILES_URL.contains("supportsAllDrives=true")); + } +} From f71a47bc4bacd2bb545d6791f4e443a01c2074ae Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Fri, 6 Nov 2026 17:55:05 +0100 Subject: [PATCH 2/3] fix(google-drive): upload the whole encrypted stream and test the real chunk loop --- .../storage/providers/google_drive/helpers.rs | 144 +++++------------ .../storage/providers/google_drive/mod.rs | 2 +- src/tests/storage/google_drive.rs | 151 ++++++++++++++++++ src/tests/storage/mod.rs | 1 + 4 files changed, 191 insertions(+), 107 deletions(-) create mode 100644 src/tests/storage/google_drive.rs diff --git a/src/services/storage/providers/google_drive/helpers.rs b/src/services/storage/providers/google_drive/helpers.rs index 2e0df882..66eddfde 100644 --- a/src/services/storage/providers/google_drive/helpers.rs +++ b/src/services/storage/providers/google_drive/helpers.rs @@ -1,67 +1,21 @@ use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result, anyhow}; use bytes::Bytes; use futures::Stream; use futures::StreamExt; use oauth2::{ - basic::BasicClient, reqwest::Client as OAuth2ReqwestClient, AuthUrl, ClientId, ClientSecret, - RefreshToken, TokenResponse, TokenUrl, + AuthUrl, ClientId, ClientSecret, RefreshToken, TokenResponse, TokenUrl, basic::BasicClient, + reqwest::Client as OAuth2ReqwestClient, }; -use reqwest::{header, Client}; use reqwest::{Client as ReqwestClient, StatusCode}; -use serde_json::{json, Value}; +use reqwest::{Client, header}; +use serde_json::{Value, json}; pub const DRIVE_FILES_URL: &str = "https://www.googleapis.com/drive/v3/files?supportsAllDrives=true"; pub const DRIVE_RESUMABLE_UPLOAD_URL: &str = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true"; -pub fn append_from_pending(buf: &mut Vec, pending: &mut Bytes, remaining: &mut u64) { - if pending.is_empty() || *remaining == 0 { - return; - } - let to_take = (*remaining as usize).min(pending.len()); - buf.extend_from_slice(&pending[..to_take]); - *pending = pending.slice(to_take..); - *remaining -= to_take as u64; -} - -pub fn collect_windows(chunks: Vec, total: u64, window: u64) -> Result>> { - let mut pending = Bytes::new(); - let mut iter = chunks.into_iter(); - let mut uploaded = 0u64; - let mut windows = Vec::new(); - - while uploaded < total { - let chunk_size = (total - uploaded).min(window); - let mut buf = Vec::with_capacity(chunk_size as usize); - let mut remaining = chunk_size; - - while remaining > 0 { - if pending.is_empty() { - match iter.next() { - Some(next) => pending = next, - None => { - if uploaded + buf.len() as u64 != total { - return Err(anyhow!("Stream ended early")); - } - break; - } - } - } - append_from_pending(&mut buf, &mut pending, &mut remaining); - } - - if buf.is_empty() && uploaded < total { - return Err(anyhow!("Unexpected end of stream")); - } - uploaded += buf.len() as u64; - windows.push(buf); - } - - Ok(windows) -} - pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Result { let http_client = OAuth2ReqwestClient::new(); @@ -191,7 +145,7 @@ pub async fn find_file_by_name( pub async fn upload_stream_to_google_drive( config: &GoogleDriveProviderConfig, full_path: &str, - mut content_stream: impl Stream> + Send + Unpin + 'static, + content_stream: impl Stream> + Send + Unpin + 'static, total_size: u64, mime_type: Option<&str>, ) -> Result<()> { @@ -227,7 +181,6 @@ pub async fn upload_stream_to_google_drive( .post(DRIVE_RESUMABLE_UPLOAD_URL) .bearer_auth(&token) .header("X-Upload-Content-Type", mime) - .header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot .json(&metadata) .send() .await @@ -245,51 +198,61 @@ pub async fn upload_stream_to_google_drive( .to_str()? .to_string(); - const CHUNK_SIZE: u64 = 8 * 1024 * 1024; + upload_chunks(&client, &upload_url, content_stream, total_size).await +} + +pub async fn upload_chunks( + client: &Client, + upload_url: &str, + mut content_stream: impl Stream> + Unpin, + source_size: u64, +) -> Result<()> { + const CHUNK_SIZE: usize = 8 * 1024 * 1024; let mut uploaded: u64 = 0; let mut pending = Bytes::new(); + let mut stream_ended = false; - while uploaded < total_size { - let chunk_size = (total_size - uploaded).min(CHUNK_SIZE); - - let mut chunk_bytes = Vec::with_capacity(chunk_size as usize); - let mut remaining = chunk_size; + while !stream_ended { + let mut chunk_bytes = Vec::with_capacity(CHUNK_SIZE); - while remaining > 0 { + loop { if pending.is_empty() { match content_stream.next().await { Some(Ok(bytes)) => pending = bytes, Some(Err(e)) => return Err(e).context("Stream error during chunk"), None => { - if uploaded + chunk_bytes.len() as u64 != total_size { - return Err(anyhow::anyhow!("Stream ended early")); - } + stream_ended = true; break; } } + } else if chunk_bytes.len() < CHUNK_SIZE { + let to_take = (CHUNK_SIZE - chunk_bytes.len()).min(pending.len()); + chunk_bytes.extend_from_slice(&pending.split_to(to_take)); + } else { + break; } - append_from_pending(&mut chunk_bytes, &mut pending, &mut remaining); } - if chunk_bytes.is_empty() && uploaded < total_size { - return Err(anyhow::anyhow!("Unexpected end of stream")); + let chunk_end = uploaded + chunk_bytes.len() as u64; + if stream_ended && chunk_end < source_size { + return Err(anyhow::anyhow!("Stream ended early")); } - let range_end = uploaded + chunk_bytes.len() as u64 - 1; - let content_range = if uploaded + chunk_bytes.len() as u64 == total_size { - format!("bytes {}-{}/{}", uploaded, range_end, total_size) - } else { - format!("bytes {}-{}/*", uploaded, range_end) + let content_range = match (chunk_bytes.is_empty(), stream_ended) { + (true, _) => format!("bytes */{chunk_end}"), + (false, true) => format!("bytes {uploaded}-{}/{chunk_end}", chunk_end - 1), + (false, false) => format!("bytes {uploaded}-{}/*", chunk_end - 1), }; + let chunk_bytes = Bytes::from(chunk_bytes); let mut retries = 0; loop { let res = client - .put(&upload_url) + .put(upload_url) .header("Content-Range", &content_range) .header("Content-Length", chunk_bytes.len().to_string()) - .body(chunk_bytes.clone()) // clone is cheap if small; optimize later if needed + .body(chunk_bytes.clone()) .send() .await; @@ -299,8 +262,8 @@ pub async fn upload_stream_to_google_drive( || resp.status() == StatusCode::PERMANENT_REDIRECT => { // 200 or 308 = good - uploaded += chunk_bytes.len() as u64; - tracing::info!("Uploaded {}/{} bytes", uploaded, total_size); + uploaded = chunk_end; + tracing::info!("Uploaded {} bytes", uploaded); break; } Ok(resp) if resp.status() == StatusCode::TOO_MANY_REQUESTS => { @@ -325,34 +288,3 @@ pub async fn upload_stream_to_google_drive( Ok(()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn sixteen_mib_chunk_survives_eight_mib_windows() { - let window = 8 * 1024 * 1024; - let payload = vec![7u8; 16 * 1024 * 1024]; - let windows = collect_windows( - vec![Bytes::from(payload.clone())], - payload.len() as u64, - window, - ) - .expect("windows"); - - assert_eq!(windows.len(), 2); - assert_eq!(windows[0].len(), window as usize); - assert_eq!(windows[1].len(), window as usize); - let mut joined = windows[0].clone(); - joined.extend_from_slice(&windows[1]); - assert_eq!(joined, payload); - } - - #[test] - fn drive_write_urls_carry_supports_all_drives() { - assert!(DRIVE_RESUMABLE_UPLOAD_URL.contains("supportsAllDrives=true")); - assert!(DRIVE_RESUMABLE_UPLOAD_URL.contains("uploadType=resumable")); - assert!(DRIVE_FILES_URL.contains("supportsAllDrives=true")); - } -} diff --git a/src/services/storage/providers/google_drive/mod.rs b/src/services/storage/providers/google_drive/mod.rs index c8089e64..319832bd 100644 --- a/src/services/storage/providers/google_drive/mod.rs +++ b/src/services/storage/providers/google_drive/mod.rs @@ -1,4 +1,4 @@ -mod helpers; +pub mod helpers; pub mod models; use crate::core::context::Context; diff --git a/src/tests/storage/google_drive.rs b/src/tests/storage/google_drive.rs new file mode 100644 index 00000000..1c45b66d --- /dev/null +++ b/src/tests/storage/google_drive.rs @@ -0,0 +1,151 @@ +use crate::services::storage::providers::google_drive::helpers::{ + DRIVE_FILES_URL, DRIVE_RESUMABLE_UPLOAD_URL, upload_chunks, +}; +use crate::tests::services::local_storage_tests::MASTER_KEY_B64; +use crate::utils::file::decrypt_file_stream_gcm; +use crate::utils::stream::build_stream; + +use bytes::Bytes; +use futures::stream; +use reqwest::Client; +use tempfile::TempDir; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const WINDOW: usize = 8 * 1024 * 1024; + +async fn drive_session() -> MockServer { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path("/upload")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + server +} + +async fn received_puts(server: &MockServer) -> Vec<(String, Vec)> { + server + .received_requests() + .await + .unwrap() + .into_iter() + .map(|request| { + let content_range = request.headers["content-range"] + .to_str() + .unwrap() + .to_string(); + (content_range, request.body) + }) + .collect() +} + +#[tokio::test] +async fn sixteen_mib_chunk_is_sent_as_two_eight_mib_windows() { + let server = drive_session().await; + let payload: Vec = (0..2 * WINDOW).map(|index| (index % 251) as u8).collect(); + let content_stream = stream::iter(vec![Ok(Bytes::from(payload.clone()))]); + + upload_chunks( + &Client::new(), + &format!("{}/upload", server.uri()), + content_stream, + payload.len() as u64, + ) + .await + .unwrap(); + + let puts = received_puts(&server).await; + assert_eq!(puts.len(), 2); + assert_eq!(puts[0].0, "bytes 0-8388607/*"); + assert_eq!(puts[1].0, "bytes 8388608-16777215/16777216"); + assert_eq!(puts[0].1.len(), WINDOW); + assert_eq!(puts[1].1.len(), WINDOW); + assert_eq!( + [puts[0].1.as_slice(), puts[1].1.as_slice()].concat(), + payload + ); +} + +#[tokio::test] +async fn encrypted_dump_uploaded_to_drive_decrypts_back() { + let server = drive_session().await; + let work = TempDir::new().unwrap(); + let dump_path = work.path().join("dump.sql"); + let plaintext: Vec = (0..WINDOW + 1024 * 1024) + .map(|index| (index % 253) as u8) + .collect(); + std::fs::write(&dump_path, &plaintext).unwrap(); + let upload = build_stream(&dump_path, true, &MASTER_KEY_B64.to_string()) + .await + .unwrap(); + + upload_chunks( + &Client::new(), + &format!("{}/upload", server.uri()), + upload.stream, + plaintext.len() as u64, + ) + .await + .unwrap(); + + let puts = received_puts(&server).await; + let uploaded: Vec = puts.iter().flat_map(|(_, body)| body.clone()).collect(); + let (last_range, _) = puts.last().unwrap(); + assert!(last_range.ends_with(&format!("/{}", uploaded.len()))); + let encrypted_path = work.path().join("dump.sql.enc"); + let decrypted_path = work.path().join("dump.restored.sql"); + std::fs::write(&encrypted_path, &uploaded).unwrap(); + decrypt_file_stream_gcm( + encrypted_path, + decrypted_path.clone(), + MASTER_KEY_B64.to_string(), + ) + .await + .unwrap(); + assert_eq!(std::fs::read(decrypted_path).unwrap(), plaintext); +} + +#[tokio::test] +async fn stream_shorter_than_the_dump_never_finalizes_the_upload() { + let server = drive_session().await; + let content_stream = stream::iter(vec![Ok(Bytes::from(vec![1u8; 1024]))]); + + let error = upload_chunks( + &Client::new(), + &format!("{}/upload", server.uri()), + content_stream, + 2048, + ) + .await + .unwrap_err(); + + assert_eq!(error.to_string(), "Stream ended early"); + assert!(received_puts(&server).await.is_empty()); +} + +#[tokio::test] +async fn empty_dump_finalizes_a_zero_byte_file() { + let server = drive_session().await; + + upload_chunks( + &Client::new(), + &format!("{}/upload", server.uri()), + stream::iter(Vec::>::new()), + 0, + ) + .await + .unwrap(); + + let puts = received_puts(&server).await; + assert_eq!(puts.len(), 1); + assert_eq!(puts[0].0, "bytes */0"); + assert!(puts[0].1.is_empty()); +} + +#[test] +fn drive_write_urls_carry_supports_all_drives() { + assert!(DRIVE_RESUMABLE_UPLOAD_URL.contains("supportsAllDrives=true")); + assert!(DRIVE_RESUMABLE_UPLOAD_URL.contains("uploadType=resumable")); + assert!(DRIVE_FILES_URL.contains("supportsAllDrives=true")); +} diff --git a/src/tests/storage/mod.rs b/src/tests/storage/mod.rs index 96840a00..e1b86632 100644 --- a/src/tests/storage/mod.rs +++ b/src/tests/storage/mod.rs @@ -1,5 +1,6 @@ mod azure_blob; mod google_cloud_storage; +mod google_drive; mod rclone; mod rclone_target; mod sftp; From 2b4d89ce3b6ca20b68c5bd0ec751ce191a7feaf7 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Fri, 6 Nov 2026 18:31:26 +0100 Subject: [PATCH 3/3] fix: refactoring --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4020e3ee..64c044c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMTM1ZmFiM2QtZGQ2Yy00MjVjLWI1YTMtMzRmMTBhYTBmNDU0IiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGYzOTAyYjgtZmU0Yy00NmY2LWE0ODktNzE5MTNhZTgxMjg3IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" #CHUNK_SIZE_MB: "1" #POLLING: 1 #RETRY_ATTEMPTS: 3