diff --git a/core/core/src/types/capability.rs b/core/core/src/types/capability.rs index 9994f6e5cd58..e4d15c270e9d 100644 --- a/core/core/src/types/capability.rs +++ b/core/core/src/types/capability.rs @@ -118,8 +118,16 @@ pub struct Capability { /// Indicates if Cache-Control can be specified during write operations. pub write_with_cache_control: bool, /// Indicates if conditional write operations using If-Match are supported. + /// + /// The value passed to `if_match` is normally a literal ETag. GCS is an exception: + /// it has no ETag-based conditional write API, so it repurposes this field to carry + /// the object's generation number (`Metadata::version()`) instead. pub write_with_if_match: bool, /// Indicates if conditional write operations using If-None-Match are supported. + /// + /// The value passed to `if_none_match` is normally a literal ETag. GCS is an exception: + /// it has no ETag-based conditional write API, so it repurposes this field to carry + /// the object's generation number (`Metadata::version()`) instead. pub write_with_if_none_match: bool, /// Indicates if write operations can be conditional on object non-existence. pub write_with_if_not_exists: bool, diff --git a/core/services/azblob/src/backend.rs b/core/services/azblob/src/backend.rs index 62fb8f51ab04..45a9e50ae5e8 100644 --- a/core/services/azblob/src/backend.rs +++ b/core/services/azblob/src/backend.rs @@ -407,6 +407,7 @@ impl Builder for AzblobBuilder { write_can_multi: true, write_with_cache_control: true, write_with_content_type: true, + write_with_if_match: true, write_with_if_not_exists: true, write_with_if_none_match: true, write_with_user_metadata: true, diff --git a/core/services/azblob/src/core.rs b/core/services/azblob/src/core.rs index f714e9af9cc0..5eac3e40f6b9 100644 --- a/core/services/azblob/src/core.rs +++ b/core/services/azblob/src/core.rs @@ -306,6 +306,10 @@ impl AzblobCore { req = req.header(IF_NONE_MATCH, v); } + if let Some(v) = args.if_match() { + req = req.header(IF_MATCH, v); + } + if let Some(cache_control) = args.cache_control() { req = req.header(constants::X_MS_BLOB_CACHE_CONTROL, cache_control); } diff --git a/core/services/gcs/src/backend.rs b/core/services/gcs/src/backend.rs index c038fb5de58c..d0bc0234bbc1 100644 --- a/core/services/gcs/src/backend.rs +++ b/core/services/gcs/src/backend.rs @@ -357,6 +357,8 @@ impl Builder for GcsBuilder { write_with_content_type: true, write_with_content_encoding: true, write_with_user_metadata: true, + write_with_if_match: true, + write_with_if_none_match: true, write_with_if_not_exists: true, // The min multipart size of Gcs is 5 MiB. diff --git a/core/services/gcs/src/core.rs b/core/services/gcs/src/core.rs index 80eab2090725..8c8b9f15c873 100644 --- a/core/services/gcs/src/core.rs +++ b/core/services/gcs/src/core.rs @@ -278,8 +278,32 @@ impl GcsCore { // Makes the operation conditional on whether the object's current generation // matches the given value. Setting to 0 makes the operation succeed only if // there are no live versions of the object. + // + // GCS has no ETag-based conditional write, so `if_match`/`if_none_match` are + // repurposed to carry the object's generation number (as exposed via + // `Metadata::version()`) instead of a literal ETag. if op.if_not_exists() { write!(&mut url, "&ifGenerationMatch=0").unwrap(); + } else if let Some(v) = op.if_match() { + let generation: i64 = v.parse().map_err(|e| { + Error::new( + ErrorKind::Unsupported, + "if_match value must be a valid GCS object generation number, see Metadata::version()", + ) + .set_source(e) + })?; + write!(&mut url, "&ifGenerationMatch={generation}").unwrap(); + } + + if let Some(v) = op.if_none_match() { + let generation: i64 = v.parse().map_err(|e| { + Error::new( + ErrorKind::Unsupported, + "if_none_match value must be a valid GCS object generation number, see Metadata::version()", + ) + .set_source(e) + })?; + write!(&mut url, "&ifGenerationNotMatch={generation}").unwrap(); } let mut req = Request::post(&url); diff --git a/core/services/gcs/src/docs.md b/core/services/gcs/src/docs.md index 12d6c0342fa3..2e799330914e 100644 --- a/core/services/gcs/src/docs.md +++ b/core/services/gcs/src/docs.md @@ -12,6 +12,14 @@ This service can be used to: - [ ] rename - [x] presign +## Conditional Writes + +GCS has no ETag-based conditional write API. `write_with_if_match` and `write_with_if_none_match` +are implemented on top of GCS's generation-number conditional parameters +(`ifGenerationMatch` / `ifGenerationNotMatch`) instead. Pass the object's generation number, +obtained via `Metadata::version()` from a prior `stat`/`read`/`list`, rather than a literal ETag +(`Metadata::etag()` is not usable here). + ## Configuration - `root`: Set the work directory for backend diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs index d3b5b05e788c..707804abe96d 100644 --- a/core/tests/behavior/async_write.rs +++ b/core/tests/behavior/async_write.rs @@ -724,6 +724,16 @@ pub async fn test_writer_write_with_overwrite(op: Operator) -> Result<()> { Ok(()) } +/// GCS has no ETag-based conditional write; `if_match`/`if_none_match` are repurposed +/// to carry the object's generation number (`Metadata::version()`) instead of an ETag. +fn condition_token(op: &Operator, meta: &Metadata) -> String { + if op.info().scheme() == "gcs" { + meta.version().expect("generation must exist").to_string() + } else { + meta.etag().expect("etag must exist").to_string() + } +} + /// Write an exists file with if_none_match should match, else get a ConditionNotMatch error. pub async fn test_write_with_if_none_match(op: Operator) -> Result<()> { if !op.info().capability().write_with_if_none_match { @@ -737,10 +747,11 @@ pub async fn test_write_with_if_none_match(op: Operator) -> Result<()> { .expect("write must succeed"); let meta = op.stat(&path).await?; + let token = condition_token(&op, &meta); let res = op .write_with(&path, content.clone()) - .if_none_match(meta.etag().expect("etag must exist")) + .if_none_match(&token) .await; assert!(res.is_err()); assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); @@ -786,23 +797,23 @@ pub async fn test_write_with_if_match(op: Operator) -> Result<()> { op.write(&path_a, content_a.clone()).await?; op.write(&path_b, content_b.clone()).await?; - // Get etags for both files + // Get condition tokens for both files let meta_a = op.stat(&path_a).await?; - let etag_a = meta_a.etag().expect("etag must exist"); + let token_a = condition_token(&op, &meta_a); let meta_b = op.stat(&path_b).await?; - let etag_b = meta_b.etag().expect("etag must exist"); + let token_b = condition_token(&op, &meta_b); - // Should succeed: Writing to path_a with its own etag + // Should succeed: Writing to path_a with its own token let res = op .write_with(&path_a, content_a.clone()) - .if_match(etag_a) + .if_match(&token_a) .await; assert!(res.is_ok()); - // Should fail: Writing to path_a with path_b's etag + // Should fail: Writing to path_a with path_b's token let res = op .write_with(&path_a, content_a.clone()) - .if_match(etag_b) + .if_match(&token_b) .await; assert!(res.is_err()); assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);