From 50444260750f88b8eb3037e548b5b3acaaa3ed55 Mon Sep 17 00:00:00 2001 From: elenagaljak-db Date: Wed, 8 Apr 2026 16:57:50 +0200 Subject: [PATCH 1/6] [Rust] Release 1.1.0 Signed-off-by: elenagaljak-db --- rust/CHANGELOG.md | 21 +++++++++++++++++++++ rust/Cargo.lock | 24 ++++++++++++------------ rust/NEXT_CHANGELOG.md | 6 +----- rust/sdk/Cargo.toml | 2 +- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index 51fd6f1..8d2e330 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -1,5 +1,26 @@ # Version changelog +### Major Changes + +### New Features and Improvements + +- **[Experimental Arrow Flight] Zero-copy IPC ingestion via `ingest_ipc_batch`**: Added `ZerobusArrowStream::ingest_ipc_batch(Bytes)` for FFI callers (Go, Python, Java, TypeScript) that already hold Arrow IPC stream bytes. Raw bytes are forwarded directly to the Flight wire format without deserialising to a `RecordBatch` and re-serialising, eliminating one IPC round-trip per batch compared to `ingest_batch`. The existing `ingest_batch` API is unchanged. + +### Bug Fixes + +- Fixed proto generation tool to skip reserved field numbers 19000-19999 for tables with more than 19000 columns + +### Documentation + +### Internal Changes + +### Breaking Changes + +### Deprecations + +### API Changes + + ## Release v1.0.1 ### Bug Fixes diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9d49235..b4de1c8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -563,13 +563,14 @@ dependencies = [ [[package]] name = "databricks-zerobus-ingest-sdk" version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f0e6f7302cc7df55f016c4faedb4f9294a288cdc4dabdc15105660f0565c256" dependencies = [ "arrow-array", "arrow-flight", "arrow-ipc", "arrow-schema", "async-trait", - "bytes", "futures", "hyper-http-proxy", "hyper-util", @@ -588,20 +589,18 @@ dependencies = [ "tonic 0.13.1", "tonic-build 0.13.1", "tracing", - "tracing-subscriber", ] [[package]] name = "databricks-zerobus-ingest-sdk" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f0e6f7302cc7df55f016c4faedb4f9294a288cdc4dabdc15105660f0565c256" +version = "1.1.0" dependencies = [ "arrow-array", "arrow-flight", "arrow-ipc", "arrow-schema", "async-trait", + "bytes", "futures", "hyper-http-proxy", "hyper-util", @@ -620,6 +619,7 @@ dependencies = [ "tonic 0.13.1", "tonic-build 0.13.1", "tracing", + "tracing-subscriber", ] [[package]] @@ -670,7 +670,7 @@ name = "example_json_batch" version = "0.1.0" dependencies = [ "chrono", - "databricks-zerobus-ingest-sdk 1.0.1", + "databricks-zerobus-ingest-sdk 1.1.0", "serde", "tokio", ] @@ -680,7 +680,7 @@ name = "example_json_single" version = "0.1.0" dependencies = [ "chrono", - "databricks-zerobus-ingest-sdk 1.0.1", + "databricks-zerobus-ingest-sdk 1.1.0", "serde", "tokio", ] @@ -690,7 +690,7 @@ name = "example_proto_batch" version = "0.1.0" dependencies = [ "chrono", - "databricks-zerobus-ingest-sdk 1.0.1", + "databricks-zerobus-ingest-sdk 1.1.0", "prost 0.13.5", "prost-build 0.12.6", "prost-reflect", @@ -706,7 +706,7 @@ name = "example_proto_single" version = "0.1.0" dependencies = [ "chrono", - "databricks-zerobus-ingest-sdk 1.0.1", + "databricks-zerobus-ingest-sdk 1.1.0", "prost 0.13.5", "prost-build 0.12.6", "prost-reflect", @@ -2537,7 +2537,7 @@ dependencies = [ "arrow-schema", "async-trait", "bytes", - "databricks-zerobus-ingest-sdk 1.0.1", + "databricks-zerobus-ingest-sdk 1.1.0", "futures", "prost 0.13.5", "prost-reflect", @@ -3658,7 +3658,7 @@ dependencies = [ "arrow-ipc", "async-trait", "cbindgen", - "databricks-zerobus-ingest-sdk 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "databricks-zerobus-ingest-sdk 1.0.1", "libc", "once_cell", "prost 0.13.5", @@ -3674,7 +3674,7 @@ dependencies = [ "arrow-array", "arrow-ipc", "arrow-schema", - "databricks-zerobus-ingest-sdk 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "databricks-zerobus-ingest-sdk 1.0.1", "jni", "prost 0.13.5", "prost-types 0.13.5", diff --git a/rust/NEXT_CHANGELOG.md b/rust/NEXT_CHANGELOG.md index 67aff93..d5e2f40 100644 --- a/rust/NEXT_CHANGELOG.md +++ b/rust/NEXT_CHANGELOG.md @@ -1,17 +1,13 @@ # NEXT CHANGELOG -## Release v1.1.0 +## Release v1.2.0 ### Major Changes ### New Features and Improvements -- **[Experimental Arrow Flight] Zero-copy IPC ingestion via `ingest_ipc_batch`**: Added `ZerobusArrowStream::ingest_ipc_batch(Bytes)` for FFI callers (Go, Python, Java, TypeScript) that already hold Arrow IPC stream bytes. Raw bytes are forwarded directly to the Flight wire format without deserialising to a `RecordBatch` and re-serialising, eliminating one IPC round-trip per batch compared to `ingest_batch`. The existing `ingest_batch` API is unchanged. - ### Bug Fixes -- Fixed proto generation tool to skip reserved field numbers 19000-19999 for tables with more than 19000 columns - ### Documentation ### Internal Changes diff --git a/rust/sdk/Cargo.toml b/rust/sdk/Cargo.toml index b2b6927..40654dc 100644 --- a/rust/sdk/Cargo.toml +++ b/rust/sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "databricks-zerobus-ingest-sdk" -version = "1.0.1" +version = "1.1.0" authors = ["Databricks"] edition = "2021" rust-version = "1.70" From 1b6206c5badd865ffb630b0118de91b0e341cbc3 Mon Sep 17 00:00:00 2001 From: elenagaljak-db Date: Wed, 8 Apr 2026 16:58:36 +0200 Subject: [PATCH 2/6] fix Signed-off-by: elenagaljak-db --- rust/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index 8d2e330..aaa8c3a 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v1.0.1 + ### Major Changes ### New Features and Improvements From 4579c96fe50840320946faf59da438758ba7e36d Mon Sep 17 00:00:00 2001 From: elenagaljak-db Date: Wed, 8 Apr 2026 17:01:24 +0200 Subject: [PATCH 3/6] try out windows runner Signed-off-by: elenagaljak-db --- .github/workflows/ci-rust.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index b4ac929..93d6941 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -124,6 +124,8 @@ jobs: include: - os: linux-ubuntu-latest runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } + - os: windows-server-latest + runner: { group: databricks-protected-runner-group, labels: windows-server-latest } runs-on: ${{ matrix.runner }} defaults: run: From 66e1a4c6dfc0359a32bb44a52f0ac4de901b03d8 Mon Sep 17 00:00:00 2001 From: elenagaljak-db Date: Wed, 8 Apr 2026 17:23:55 +0200 Subject: [PATCH 4/6] try to fix windows runner Signed-off-by: elenagaljak-db --- .github/workflows/ci-rust.yml | 57 +++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 93d6941..88ac1c2 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -132,7 +132,8 @@ jobs: working-directory: rust steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Get JFrog OIDC token + - name: Get JFrog OIDC token (Linux) + if: matrix.os == 'linux-ubuntu-latest' shell: bash run: | set -euo pipefail @@ -151,7 +152,37 @@ jobs: fi echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" echo "JFrog OIDC token obtained successfully" - - name: Configure Cargo registry + - name: Get JFrog OIDC token (Windows) + if: matrix.os == 'windows-server-latest' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $IdToken = (Invoke-RestMethod ` + -Uri "$env:ACTIONS_ID_TOKEN_REQUEST_URL&audience=jfrog-github" ` + -Headers @{ 'User-Agent' = 'actions/oidc-client'; 'Authorization' = "Bearer $env:ACTIONS_ID_TOKEN_REQUEST_TOKEN" } ` + ).value + Write-Output "::add-mask::$IdToken" + $Body = @{ + grant_type = 'urn:ietf:params:oauth:grant-type:token-exchange' + subject_token_type = 'urn:ietf:params:oauth:token-type:id_token' + subject_token = $IdToken + provider_name = 'github-actions' + } | ConvertTo-Json + $AccessToken = (Invoke-RestMethod ` + -Method POST ` + -Uri 'https://databricks.jfrog.io/access/api/v1/oidc/token' ` + -ContentType 'application/json' ` + -Body $Body ` + ).access_token + Write-Output "::add-mask::$AccessToken" + if (-not $AccessToken -or $AccessToken -eq 'null') { + Write-Error 'FAIL: Could not extract JFrog access token' + exit 1 + } + "JFROG_ACCESS_TOKEN=$AccessToken" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + Write-Output 'JFrog OIDC token obtained successfully' + - name: Configure Cargo registry (Linux) + if: matrix.os == 'linux-ubuntu-latest' shell: bash run: | set -euo pipefail @@ -171,6 +202,28 @@ jobs: EOF echo "CARGO_REGISTRIES_JFROG_TOKEN=Bearer ${JFROG_ACCESS_TOKEN}" >> "$GITHUB_ENV" echo "Cargo configured to use JFrog registry" + - name: Configure Cargo registry (Windows) + if: matrix.os == 'windows-server-latest' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $CargoHome = "$env:USERPROFILE\.cargo" + New-Item -ItemType Directory -Force -Path $CargoHome | Out-Null + @" + [source.crates-io] + replace-with = "jfrog" + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + "@ | Set-Content -Path "$CargoHome\config.toml" -Encoding utf8 + @" + [registries.jfrog] + token = "Bearer $env:JFROG_ACCESS_TOKEN" + "@ | Set-Content -Path "$CargoHome\credentials.toml" -Encoding utf8 + "CARGO_REGISTRIES_JFROG_TOKEN=Bearer $env:JFROG_ACCESS_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + Write-Output 'Cargo configured to use JFrog registry' - name: Unshallow run: git fetch --prune --unshallow - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 From 1d16ecee6d18cc1a551b3e940b50f145b806e010 Mon Sep 17 00:00:00 2001 From: elenagaljak-db Date: Wed, 8 Apr 2026 17:27:06 +0200 Subject: [PATCH 5/6] try to fix windows runner 2 Signed-off-by: elenagaljak-db --- .github/workflows/ci-rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 88ac1c2..2f5a206 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -154,7 +154,7 @@ jobs: echo "JFrog OIDC token obtained successfully" - name: Get JFrog OIDC token (Windows) if: matrix.os == 'windows-server-latest' - shell: pwsh + shell: powershell run: | $ErrorActionPreference = 'Stop' $IdToken = (Invoke-RestMethod ` @@ -204,7 +204,7 @@ jobs: echo "Cargo configured to use JFrog registry" - name: Configure Cargo registry (Windows) if: matrix.os == 'windows-server-latest' - shell: pwsh + shell: powershell run: | $ErrorActionPreference = 'Stop' $CargoHome = "$env:USERPROFILE\.cargo" From 33aa97cabba1f29b85b506e2c37ab970ba05897b Mon Sep 17 00:00:00 2001 From: elenagaljak-db Date: Wed, 8 Apr 2026 18:30:58 +0200 Subject: [PATCH 6/6] revert windows changes Signed-off-by: elenagaljak-db --- .github/workflows/ci-rust.yml | 59 ++--------------------------------- 1 file changed, 2 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 2f5a206..b4ac929 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -124,16 +124,13 @@ jobs: include: - os: linux-ubuntu-latest runner: { group: databricks-protected-runner-group, labels: linux-ubuntu-latest } - - os: windows-server-latest - runner: { group: databricks-protected-runner-group, labels: windows-server-latest } runs-on: ${{ matrix.runner }} defaults: run: working-directory: rust steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Get JFrog OIDC token (Linux) - if: matrix.os == 'linux-ubuntu-latest' + - name: Get JFrog OIDC token shell: bash run: | set -euo pipefail @@ -152,37 +149,7 @@ jobs: fi echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" echo "JFrog OIDC token obtained successfully" - - name: Get JFrog OIDC token (Windows) - if: matrix.os == 'windows-server-latest' - shell: powershell - run: | - $ErrorActionPreference = 'Stop' - $IdToken = (Invoke-RestMethod ` - -Uri "$env:ACTIONS_ID_TOKEN_REQUEST_URL&audience=jfrog-github" ` - -Headers @{ 'User-Agent' = 'actions/oidc-client'; 'Authorization' = "Bearer $env:ACTIONS_ID_TOKEN_REQUEST_TOKEN" } ` - ).value - Write-Output "::add-mask::$IdToken" - $Body = @{ - grant_type = 'urn:ietf:params:oauth:grant-type:token-exchange' - subject_token_type = 'urn:ietf:params:oauth:token-type:id_token' - subject_token = $IdToken - provider_name = 'github-actions' - } | ConvertTo-Json - $AccessToken = (Invoke-RestMethod ` - -Method POST ` - -Uri 'https://databricks.jfrog.io/access/api/v1/oidc/token' ` - -ContentType 'application/json' ` - -Body $Body ` - ).access_token - Write-Output "::add-mask::$AccessToken" - if (-not $AccessToken -or $AccessToken -eq 'null') { - Write-Error 'FAIL: Could not extract JFrog access token' - exit 1 - } - "JFROG_ACCESS_TOKEN=$AccessToken" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - Write-Output 'JFrog OIDC token obtained successfully' - - name: Configure Cargo registry (Linux) - if: matrix.os == 'linux-ubuntu-latest' + - name: Configure Cargo registry shell: bash run: | set -euo pipefail @@ -202,28 +169,6 @@ jobs: EOF echo "CARGO_REGISTRIES_JFROG_TOKEN=Bearer ${JFROG_ACCESS_TOKEN}" >> "$GITHUB_ENV" echo "Cargo configured to use JFrog registry" - - name: Configure Cargo registry (Windows) - if: matrix.os == 'windows-server-latest' - shell: powershell - run: | - $ErrorActionPreference = 'Stop' - $CargoHome = "$env:USERPROFILE\.cargo" - New-Item -ItemType Directory -Force -Path $CargoHome | Out-Null - @" - [source.crates-io] - replace-with = "jfrog" - [source.jfrog] - registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" - [registries.jfrog] - index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" - credential-provider = ["cargo:token"] - "@ | Set-Content -Path "$CargoHome\config.toml" -Encoding utf8 - @" - [registries.jfrog] - token = "Bearer $env:JFROG_ACCESS_TOKEN" - "@ | Set-Content -Path "$CargoHome\credentials.toml" -Encoding utf8 - "CARGO_REGISTRIES_JFROG_TOKEN=Bearer $env:JFROG_ACCESS_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - Write-Output 'Cargo configured to use JFrog registry' - name: Unshallow run: git fetch --prune --unshallow - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3