Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions core/core/src/types/capability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,16 @@ pub struct Capability {
/// Indicates if Cache-Control can be specified during write operations.
pub write_with_cache_control: bool,
/// Indicates if conditional write operations using If-Match are supported.
///
/// The value passed to `if_match` is normally a literal ETag. GCS is an exception:

@Xuanwo Xuanwo Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, we don't do this. It's very clear that if-match accepts an etag.


Think about this, we now have a user want to support all of s3, azblob and gcs.

The user is using APIs like:

let expected_etag = op.stat(pathx).await?.etag();
let _ = op.write_with(pathx, data).if_match(expected_etag).await?;

Then this user will go crazy to find that it doesn't work on gcs, because if_match always didn't match the give etag. After reading docs, they realized that they need to do:

let expected_etag = if op.scheme() == "gcs" {
    op.stat(pathx).await?.version()
} else {
    op.stat(pathx).await?.etag()
};

let _ = op.write_with(pathx, data).if_match(expected_etag).await?;

I don't think they will love opendal again.

/// it has no ETag-based conditional write API, so it repurposes this field to carry
/// the object's generation number (`Metadata::version()`) instead.
pub write_with_if_match: bool,
/// Indicates if conditional write operations using If-None-Match are supported.
///
/// The value passed to `if_none_match` is normally a literal ETag. GCS is an exception:
/// it has no ETag-based conditional write API, so it repurposes this field to carry

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The same.

/// the object's generation number (`Metadata::version()`) instead.
pub write_with_if_none_match: bool,
/// Indicates if write operations can be conditional on object non-existence.
pub write_with_if_not_exists: bool,
Expand Down
1 change: 1 addition & 0 deletions core/services/azblob/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,7 @@ impl Builder for AzblobBuilder {
write_can_multi: true,
write_with_cache_control: true,
write_with_content_type: true,
write_with_if_match: true,
write_with_if_not_exists: true,
write_with_if_none_match: true,
write_with_user_metadata: true,
Expand Down
4 changes: 4 additions & 0 deletions core/services/azblob/src/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,10 @@ impl AzblobCore {
req = req.header(IF_NONE_MATCH, v);
}

if let Some(v) = args.if_match() {
req = req.header(IF_MATCH, v);
}

if let Some(cache_control) = args.cache_control() {
req = req.header(constants::X_MS_BLOB_CACHE_CONTROL, cache_control);
}
Expand Down
2 changes: 2 additions & 0 deletions core/services/gcs/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,8 @@ impl Builder for GcsBuilder {
write_with_content_type: true,
write_with_content_encoding: true,
write_with_user_metadata: true,
write_with_if_match: true,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please don't declare a feature that services didn't support.

write_with_if_none_match: true,
write_with_if_not_exists: true,

// The min multipart size of Gcs is 5 MiB.
Expand Down
24 changes: 24 additions & 0 deletions core/services/gcs/src/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -278,8 +278,32 @@ impl GcsCore {
// Makes the operation conditional on whether the object's current generation
// matches the given value. Setting to 0 makes the operation succeed only if
// there are no live versions of the object.
//
// GCS has no ETag-based conditional write, so `if_match`/`if_none_match` are
// repurposed to carry the object's generation number (as exposed via
// `Metadata::version()`) instead of a literal ETag.
if op.if_not_exists() {
write!(&mut url, "&ifGenerationMatch=0").unwrap();
} else if let Some(v) = op.if_match() {
let generation: i64 = v.parse().map_err(|e| {
Error::new(
ErrorKind::Unsupported,
"if_match value must be a valid GCS object generation number, see Metadata::version()",
)
.set_source(e)
})?;
write!(&mut url, "&ifGenerationMatch={generation}").unwrap();
}

if let Some(v) = op.if_none_match() {
let generation: i64 = v.parse().map_err(|e| {
Error::new(
ErrorKind::Unsupported,
"if_none_match value must be a valid GCS object generation number, see Metadata::version()",
)
.set_source(e)
})?;
write!(&mut url, "&ifGenerationNotMatch={generation}").unwrap();
}

let mut req = Request::post(&url);
Expand Down
8 changes: 8 additions & 0 deletions core/services/gcs/src/docs.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@ This service can be used to:
- [ ] rename
- [x] presign

## Conditional Writes

GCS has no ETag-based conditional write API. `write_with_if_match` and `write_with_if_none_match`
are implemented on top of GCS's generation-number conditional parameters
(`ifGenerationMatch` / `ifGenerationNotMatch`) instead. Pass the object's generation number,
obtained via `Metadata::version()` from a prior `stat`/`read`/`list`, rather than a literal ETag
(`Metadata::etag()` is not usable here).

## Configuration

- `root`: Set the work directory for backend
Expand Down
27 changes: 19 additions & 8 deletions core/tests/behavior/async_write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,16 @@ pub async fn test_writer_write_with_overwrite(op: Operator) -> Result<()> {
Ok(())
}

/// GCS has no ETag-based conditional write; `if_match`/`if_none_match` are repurposed
/// to carry the object's generation number (`Metadata::version()`) instead of an ETag.
fn condition_token(op: &Operator, meta: &Metadata) -> String {
if op.info().scheme() == "gcs" {
meta.version().expect("generation must exist").to_string()
} else {
meta.etag().expect("etag must exist").to_string()
}
}

/// Write an exists file with if_none_match should match, else get a ConditionNotMatch error.
pub async fn test_write_with_if_none_match(op: Operator) -> Result<()> {
if !op.info().capability().write_with_if_none_match {
Expand All @@ -737,10 +747,11 @@ pub async fn test_write_with_if_none_match(op: Operator) -> Result<()> {
.expect("write must succeed");

let meta = op.stat(&path).await?;
let token = condition_token(&op, &meta);

let res = op
.write_with(&path, content.clone())
.if_none_match(meta.etag().expect("etag must exist"))
.if_none_match(&token)
.await;
assert!(res.is_err());
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
Expand Down Expand Up @@ -786,23 +797,23 @@ pub async fn test_write_with_if_match(op: Operator) -> Result<()> {
op.write(&path_a, content_a.clone()).await?;
op.write(&path_b, content_b.clone()).await?;

// Get etags for both files
// Get condition tokens for both files
let meta_a = op.stat(&path_a).await?;
let etag_a = meta_a.etag().expect("etag must exist");
let token_a = condition_token(&op, &meta_a);
let meta_b = op.stat(&path_b).await?;
let etag_b = meta_b.etag().expect("etag must exist");
let token_b = condition_token(&op, &meta_b);

// Should succeed: Writing to path_a with its own etag
// Should succeed: Writing to path_a with its own token
let res = op
.write_with(&path_a, content_a.clone())
.if_match(etag_a)
.if_match(&token_a)
.await;
assert!(res.is_ok());

// Should fail: Writing to path_a with path_b's etag
// Should fail: Writing to path_a with path_b's token
let res = op
.write_with(&path_a, content_a.clone())
.if_match(etag_b)
.if_match(&token_b)
.await;
assert!(res.is_err());
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
Expand Down
Loading