From be435d85d7a948c4a284267a056814d96ef14736 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Thu, 23 Oct 2025 09:55:19 -0700 Subject: [PATCH 1/9] Chore: Upgrade Datafusion and Arrow dependencies --- Cargo.lock | 737 +++++++++++--------- Cargo.toml | 46 +- c/sedona-proj/src/st_transform.rs | 1 + python/sedonadb/Cargo.toml | 3 +- python/sedonadb/src/lib.rs | 22 +- r/sedonadb/src/rust/Cargo.toml | 3 +- r/sedonadb/src/rust/src/lib.rs | 33 +- rust/sedona-expr/src/aggregate_udf.rs | 14 + rust/sedona-expr/src/scalar_udf.rs | 14 + rust/sedona-expr/src/spatial_filter.rs | 45 +- rust/sedona-functions/src/st_setsrid.rs | 1 + rust/sedona-geoparquet/Cargo.toml | 1 + rust/sedona-geoparquet/src/file_opener.rs | 8 +- rust/sedona-geoparquet/src/format.rs | 15 +- rust/sedona-geoparquet/src/writer.rs | 3 +- rust/sedona-schema/src/schema.rs | 9 +- rust/sedona-spatial-join/src/optimizer.rs | 4 +- rust/sedona-testing/src/testers.rs | 3 +- rust/sedona/src/context.rs | 2 +- rust/sedona/src/ffi.rs | 31 +- rust/sedona/src/random_geometry_provider.rs | 2 +- rust/sedona/src/show.rs | 6 +- 22 files changed, 578 insertions(+), 425 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94550ec3..889ea043 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,19 +56,8 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b891479797b5588e320f7fd3caf15faba311cf8f8a76911195b6a3d55304eb" dependencies = [ - "arrow-array", - "arrow-schema", -] - -[[package]] -name = "adbc_ffi" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30e0f28a8363a76a8ec802934223ea5811ea658308c7403c8beb9aa86fa808f" -dependencies = [ - "adbc_core", - "arrow-array", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-schema 55.2.0", ] [[package]] @@ -86,12 +75,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - [[package]] name = "ahash" version = "0.8.12" @@ -209,28 +192,28 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "apache-avro" -version = "0.17.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" +checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" dependencies = [ "bigdecimal", - "bzip2 0.4.4", + "bon", + "bzip2 0.6.0", "crc32fast", "digest", - "libflate", "log", + "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.8.5", + "rand 0.9.2", "regex-lite", "serde", "serde_bytes", "serde_json", "snap", - "strum", - "strum_macros", - "thiserror 1.0.69", - "typed-builder", + "strum 0.27.2", + "strum_macros 0.27.2", + "thiserror 2.0.17", "uuid", "xz2", "zstd", @@ -259,35 +242,35 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ "arrow-arith", - "arrow-array", - "arrow-buffer", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", "arrow-cast", "arrow-csv", - "arrow-data", + "arrow-data 56.2.0", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema", + "arrow-schema 56.2.0", "arrow-select", "arrow-string", ] [[package]] name = "arrow-arith" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "num", ] @@ -299,16 +282,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", "chrono", - "chrono-tz", "half", "hashbrown 0.15.5", "num", ] +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.0", + "num", +] + [[package]] name = "arrow-buffer" version = "55.2.0" @@ -320,16 +319,27 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-cast" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "arrow-select", "atoi", "base64", @@ -343,13 +353,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "012c9fef3f4a11573b2c74aec53712ff9fdae4a95f4ce452d1bbf088ee00f06b" +checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" dependencies = [ - "arrow-array", + "arrow-array 56.2.0", "arrow-cast", - "arrow-schema", + "arrow-schema 56.2.0", "chrono", "csv", "csv-core", @@ -362,22 +372,35 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 55.2.0", + "arrow-schema 55.2.0", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] [[package]] name = "arrow-ipc" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select", "flatbuffers", "lz4_flex", "zstd", @@ -385,15 +408,15 @@ dependencies = [ [[package]] name = "arrow-json" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" +checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" dependencies = [ - "arrow-array", - "arrow-buffer", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "half", "indexmap", @@ -407,27 +430,27 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "arrow-select", ] [[package]] name = "arrow-row" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "half", ] @@ -436,6 +459,15 @@ name = "arrow-schema" version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +dependencies = [ + "bitflags", +] + +[[package]] +name = "arrow-schema" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" dependencies = [ "bitflags", "serde", @@ -444,28 +476,28 @@ dependencies = [ [[package]] name = "arrow-select" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] [[package]] name = "arrow-string" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "arrow-select", "memchr", "num", @@ -1005,6 +1037,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bon" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +dependencies = [ + "darling 0.21.3", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.106", +] + [[package]] name = "brotli" version = "8.0.2" @@ -1060,16 +1117,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - [[package]] name = "bzip2" version = "0.5.2" @@ -1269,12 +1316,14 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.2.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ - "crossterm", - "unicode-segmentation", + "crossterm 0.27.0", + "crossterm 0.28.1", + "strum 0.26.3", + "strum_macros 0.26.4", "unicode-width", ] @@ -1329,15 +1378,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "core_extensions" version = "1.5.4" @@ -1443,18 +1483,28 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" -version = "0.29.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ "bitflags", "crossterm_winapi", - "document-features", + "libc", "parking_lot", - "rustix", "winapi", ] +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "parking_lot", + "rustix 0.38.44", +] + [[package]] name = "crossterm_winapi" version = "0.9.1" @@ -1507,8 +1557,18 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.13.4", + "darling_macro 0.13.4", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -1525,22 +1585,41 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.106", +] + [[package]] name = "darling_macro" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core", + "darling_core 0.13.4", "quote", "syn 1.0.109", ] [[package]] -name = "dary_heap" -version = "0.3.8" +name = "darling_macro" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.106", +] [[package]] name = "dashmap" @@ -1558,12 +1637,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6759cf9ef57c5c469e4027ac4b4cfa746e06a0f5472c2b922b6a403c2a64c4" dependencies = [ "arrow", "arrow-ipc", - "arrow-schema", + "arrow-schema 56.2.0", "async-trait", "bytes", "bzip2 0.6.0", @@ -1587,6 +1667,7 @@ dependencies = [ "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", @@ -1594,7 +1675,6 @@ dependencies = [ "datafusion-sql", "flate2", "futures", - "hex", "itertools 0.14.0", "log", "object_store", @@ -1613,8 +1693,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a1c48fc7e6d62590d45f7be7c531980b8ff091d1ab113a9ddf465bef41e4093" dependencies = [ "arrow", "async-trait", @@ -1638,8 +1719,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db1266da115de3ab0b2669fc027d96cf0ff777deb3216d52c74b528446ccdd6" dependencies = [ "arrow", "async-trait", @@ -1660,8 +1742,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4eb2a48ca10fa1e1a487a28a5bf080e31efac2d4bf12bb7e92c2d9ea4f35e5" dependencies = [ "ahash", "apache-avro", @@ -1671,7 +1754,6 @@ dependencies = [ "chrono", "half", "hashbrown 0.14.5", - "hex", "indexmap", "libc", "log", @@ -1686,8 +1768,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0422ee64d5791599c46b786063e695f7699fadd3a12ad25038cb3094d05886a" dependencies = [ "futures", "log", @@ -1696,8 +1779,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904c2e1089b3ccf10786f2dae12bc560fda278e4194a8917c5844d2e8c212818" dependencies = [ "arrow", "async-compression", @@ -1710,6 +1794,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", @@ -1731,8 +1816,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1c87828da05c4115935af3394b27499cd2bd91bcb846a928209650627bf7f93" dependencies = [ "apache-avro", "arrow", @@ -1755,8 +1841,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8336a805c42ef4e359daaad142ddc53649f23c7e934c117d8516816afe6b7a3d" dependencies = [ "arrow", "async-trait", @@ -1779,8 +1866,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c691b1565e245ea369bc8418b472a75ea84c2ad2deb61b1521cfa38319a9cd47" dependencies = [ "arrow", "async-trait", @@ -1803,8 +1891,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f7576ceb5974c5f6874d7f2a5ebfeb58960a920da64017def849e0352fe2d8" dependencies = [ "arrow", "async-trait", @@ -1817,13 +1906,13 @@ dependencies = [ "datafusion-expr", "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", "futures", - "hex", "itertools 0.14.0", "log", "object_store", @@ -1835,15 +1924,18 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dde7c10244f3657fc01eef8247c0b2b20eae4cf6439a0ebb27322f32026d6b8" [[package]] name = "datafusion-execution" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5143fc795cef959b6d5271b2e8f1120382fe929fc4bd027c7d7b993f5352ef7e" dependencies = [ "arrow", + "async-trait", "dashmap", "datafusion-common", "datafusion-expr", @@ -1858,8 +1950,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e826296bc5f5d0af3e39c1af473d4091ac6a152a5be2f80c256f0182938428" dependencies = [ "arrow", "async-trait", @@ -1879,8 +1972,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9096732d0d8862d1950ca70324fe91f9dee3799eeb0db53ef452bdb573484db6" dependencies = [ "arrow", "datafusion-common", @@ -1891,12 +1985,13 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4bef25e2b86d9921f7a98b1a86bfb50cebe2fd97f3a9b96c85ce475e9ef78b0" dependencies = [ "abi_stable", "arrow", - "arrow-schema", + "arrow-schema 56.2.0", "async-ffi", "async-trait", "datafusion", @@ -1912,11 +2007,12 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f362c78ac283e64fd3976e060c1a8a57d5f4dcf844a6b6bd2eb320640a1572e" dependencies = [ "arrow", - "arrow-buffer", + "arrow-buffer 56.2.0", "base64", "blake2", "blake3", @@ -1940,8 +2036,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e2a80a80145a796ae3f02eb724ac516178556aec864fe89f6ab3741a4cd249" dependencies = [ "ahash", "arrow", @@ -1960,8 +2057,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7dcca2fe7c33409e9ab3f950366aa4cba5db6175a09599fdb658ad9f2cc4296" dependencies = [ "ahash", "arrow", @@ -1972,8 +2070,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1b298733377f3ec8c2868c75b5555b15396d9c13e36c5fda28e80feee34e3ed" dependencies = [ "arrow", "arrow-ord", @@ -1993,8 +2092,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fa4a380ca362eb0fbd33093e8ca6b7a31057616c7e6ee999b87a4ad3c7c0b3f" dependencies = [ "arrow", "async-trait", @@ -2008,8 +2108,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9068fc85b8e187c706427794d79bb7ee91132b6b192cb7b18e650a5f7c5c1340" dependencies = [ "arrow", "datafusion-common", @@ -2025,8 +2126,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f80ec56e177d166269556649be817a382a374642872df4ca48cf9be3d09b3a" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2034,18 +2136,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4868fe261ba01e462033eff141e90453b7630722cad6420fddd81ebb786f6e2" dependencies = [ - "datafusion-doc", + "datafusion-expr", "quote", "syn 2.0.106", ] [[package]] name = "datafusion-optimizer" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ed8c51b5c37c057e5c7d5945ed807f1cecfba003bdb1a4c3036595dda287c7" dependencies = [ "arrow", "chrono", @@ -2063,8 +2167,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678f5734147446e1adbee63be4b244c8f0e9cbd5c41525004ace3730190d03e" dependencies = [ "ahash", "arrow", @@ -2078,14 +2183,31 @@ dependencies = [ "indexmap", "itertools 0.14.0", "log", + "parking_lot", "paste", "petgraph", ] +[[package]] +name = "datafusion-physical-expr-adapter" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086877d4eca538e9cd1f28b917db0036efe0ad8b4fb7c702f520510672032c8d" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", +] + [[package]] name = "datafusion-physical-expr-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5c5d17f6a4f28f9849ee3449bb9b83406a718e4275c218bf37ca247ee123779" dependencies = [ "ahash", "arrow", @@ -2097,8 +2219,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab9fb8b3fba2634d444e0177862797dc1231e0e20bc4db291a15d39c0d4136c3" dependencies = [ "arrow", "datafusion-common", @@ -2116,19 +2239,21 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5086cb2e579270173ff0eb38d60ba2a081f1d422a743fa673f6096920950eb5" dependencies = [ "ahash", "arrow", "arrow-ord", - "arrow-schema", + "arrow-schema 56.2.0", "async-trait", "chrono", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -2145,8 +2270,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87c686bfd29ec5362fe229247ef03a0beb063b50e307bf72d0f1a80b9d90f8b8" dependencies = [ "arrow", "chrono", @@ -2160,8 +2286,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1972d37680d48d4f6167b535e0a23ea9f814a21e1359d0bd5c30d1345b95aef9" dependencies = [ "arrow", "datafusion-common", @@ -2170,11 +2297,12 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f84b866d906118c320459f30385048aeedbe36ac06973d3e4fa0cc5d60d722c" dependencies = [ "arrow", - "arrow-schema", + "arrow-schema 56.2.0", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -2187,8 +2315,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3820062b9dd2846954eeb844ff9fe3662977b7d2d74947647c779fabfa502508" dependencies = [ "arrow", "async-trait", @@ -2210,8 +2339,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375232baa851b2e9d09fcbe8906141a0ec6e0e058addc5565e0d3d790bb9d51d" dependencies = [ "arrow", "bigdecimal", @@ -2276,15 +2406,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "document-features" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" -dependencies = [ - "litrs", -] - [[package]] name = "dunce" version = "1.0.5" @@ -2384,7 +2505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix", + "rustix 1.1.2", "windows-sys 0.59.0", ] @@ -3370,30 +3491,6 @@ version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" -[[package]] -name = "libflate" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" -dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" -dependencies = [ - "core2", - "hashbrown 0.14.5", - "rle-decode-fast", -] - [[package]] name = "libloading" version = "0.7.4" @@ -3469,6 +3566,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -3481,12 +3584,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" -[[package]] -name = "litrs" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" - [[package]] name = "lock_api" version = "0.4.14" @@ -3848,17 +3945,17 @@ dependencies = [ [[package]] name = "parquet" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", "arrow-cast", - "arrow-data", + "arrow-data 56.2.0", "arrow-ipc", - "arrow-schema", + "arrow-schema 56.2.0", "arrow-select", "base64", "brotli", @@ -3867,7 +3964,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "lz4_flex", "num", "num-bigint", @@ -4493,12 +4590,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "robust" version = "1.2.0" @@ -4567,6 +4658,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.2" @@ -4576,7 +4680,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.11.0", "windows-sys 0.61.2", ] @@ -4765,8 +4869,8 @@ name = "sedona" version = "0.2.0" dependencies = [ "abi_stable", - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "async-trait", "aws-config", "aws-credential-types", @@ -4802,20 +4906,6 @@ dependencies = [ "url", ] -[[package]] -name = "sedona-adbc" -version = "0.2.0" -dependencies = [ - "adbc_core", - "adbc_ffi", - "arrow-array", - "arrow-schema", - "datafusion", - "futures", - "sedona", - "tokio", -] - [[package]] name = "sedona-cli" version = "0.2.0" @@ -4848,8 +4938,8 @@ dependencies = [ name = "sedona-expr" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", @@ -4867,9 +4957,9 @@ dependencies = [ name = "sedona-functions" version = "0.2.0" dependencies = [ - "arrow-array", + "arrow-array 56.2.0", "arrow-json", - "arrow-schema", + "arrow-schema 56.2.0", "criterion", "datafusion", "datafusion-common", @@ -4891,8 +4981,8 @@ dependencies = [ name = "sedona-geo" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "criterion", "datafusion-common", "datafusion-expr", @@ -4951,8 +5041,8 @@ dependencies = [ name = "sedona-geoarrow-c" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "cc", "criterion", "datafusion-common", @@ -4987,14 +5077,15 @@ dependencies = [ name = "sedona-geoparquet" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "async-trait", "bytes", "chrono", "datafusion", "datafusion-catalog", "datafusion-common", + "datafusion-datasource-parquet", "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", @@ -5023,8 +5114,8 @@ dependencies = [ name = "sedona-geos" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "byteorder", "criterion", "datafusion-common", @@ -5047,8 +5138,8 @@ name = "sedona-proj" version = "0.2.0" dependencies = [ "approx", - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "cc", "criterion", "datafusion-common", @@ -5071,8 +5162,8 @@ dependencies = [ name = "sedona-s2geography" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "cmake", "criterion", "datafusion-common", @@ -5093,8 +5184,8 @@ dependencies = [ name = "sedona-schema" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "datafusion-common", "sedona-common", "serde_json", @@ -5105,8 +5196,8 @@ name = "sedona-spatial-join" version = "0.2.0" dependencies = [ "arrow", - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "criterion", "datafusion", "datafusion-common", @@ -5146,9 +5237,9 @@ dependencies = [ name = "sedona-testing" version = "0.2.0" dependencies = [ - "arrow-array", + "arrow-array 56.2.0", "arrow-cast", - "arrow-schema", + "arrow-schema 56.2.0", "criterion", "datafusion-common", "datafusion-expr", @@ -5171,8 +5262,8 @@ dependencies = [ name = "sedona-tg" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "cc", "criterion", "datafusion-common", @@ -5193,8 +5284,8 @@ name = "sedonadb" version = "0.2.0" dependencies = [ "adbc_core", - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "async-trait", "datafusion", "datafusion-common", @@ -5205,7 +5296,6 @@ dependencies = [ "mimalloc", "pyo3", "sedona", - "sedona-adbc", "sedona-geoparquet", "sedona-proj", "sedona-schema", @@ -5218,8 +5308,8 @@ dependencies = [ name = "sedonadbr" version = "0.2.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-schema 56.2.0", "datafusion", "datafusion-common", "datafusion-expr", @@ -5227,7 +5317,6 @@ dependencies = [ "savvy", "savvy-ffi", "sedona", - "sedona-adbc", "sedona-expr", "sedona-geoparquet", "sedona-proj", @@ -5329,7 +5418,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro2", "quote", "syn 1.0.109", @@ -5421,9 +5510,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.55.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" +checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" dependencies = [ "log", "recursive", @@ -5478,6 +5567,12 @@ version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + [[package]] name = "strum_macros" version = "0.26.4" @@ -5491,6 +5586,18 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "subtle" version = "2.6.1" @@ -5565,7 +5672,7 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix", + "rustix 1.1.2", "windows-sys 0.61.2", ] @@ -5896,26 +6003,6 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" -[[package]] -name = "typed-builder" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" -dependencies = [ - "typed-builder-macro", -] - -[[package]] -name = "typed-builder-macro" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "typenum" version = "1.19.0" @@ -6485,7 +6572,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f314e539..73333d4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,8 @@ members = [ "r/sedonadb/src/rust", "rust/sedona-geo-traits-ext", "rust/sedona-geo-generic-alg", - "rust/sedona-adbc", + # Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x + # "rust/sedona-adbc", "rust/sedona-expr", "rust/sedona-functions", "rust/sedona-geo", @@ -59,27 +60,27 @@ abi_stable = "0.11.3" adbc_core = "0.20.0" adbc_ffi = "0.20.0" lru = "0.12" -arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } -arrow-array = { version = "55.1.0" } -arrow-cast = { version = "55.1.0" } -arrow-data = { version = "55.1.0" } -arrow-json = { version = "55.1.0" } -arrow-schema = { version = "55.1.0" } +arrow = { version = "56.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } +arrow-array = { version = "56.0.0" } +arrow-cast = { version = "56.0.0" } +arrow-data = { version = "56.0.0" } +arrow-json = { version = "56.0.0" } +arrow-schema = { version = "56.0.0" } async-trait = { version = "0.1.87" } bytes = "1.10" byteorder = "1" chrono = { version = "0.4.38", default-features = false } comfy-table = { version = "7.0" } criterion = { version = "0.5", features = ["html_reports"] } -datafusion = { version = "49.0.0", default-features = false } -datafusion-catalog = { version = "49.0.0" } -datafusion-common = { version = "49.0.0", default-features = false } -datafusion-common-runtime = { version = "49.0.0", default-features = false } -datafusion-execution = { version = "49.0.0", default-features = false } -datafusion-expr = { version = "49.0.0" } -datafusion-ffi = { version = "49.0.0" } -datafusion-physical-expr = { version = "49.0.0" } -datafusion-physical-plan = { version = "49.0.0" } +datafusion = { version = "50.2.0", default-features = false } +datafusion-catalog = { version = "50.2.0" } +datafusion-common = { version = "50.2.0", default-features = false } +datafusion-common-runtime = { version = "50.2.0", default-features = false } +datafusion-execution = { version = "50.2.0", default-features = false } +datafusion-expr = { version = "50.2.0" } +datafusion-ffi = { version = "50.2.0" } +datafusion-physical-expr = { version = "50.2.0" } +datafusion-physical-plan = { version = "50.2.0" } dirs = "6.0.0" env_logger = "0.11" futures = { version = "0.3" } @@ -102,7 +103,7 @@ wkb = "0.9.1" wkt = "0.14.0" parking_lot = "0.12" -parquet = { version = "55.1.0", default-features = false, features = [ +parquet = { version = "56.0.0", default-features = false, features = [ "arrow", "async", "object_store", @@ -119,17 +120,6 @@ tokio = { version = "1.44" } url = "2.5.4" [patch.crates-io] -# For R package build on Windows (can be removed when we upgrade to DataFusion main/51.0.0) -datafusion = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion" } -datafusion-catalog = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-catalog" } -datafusion-common = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common" } -datafusion-common-runtime = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common-runtime" } -datafusion-execution = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-execution" } -datafusion-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-expr" } -datafusion-ffi = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-ffi" } -datafusion-physical-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-expr" } -datafusion-physical-plan = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-plan" } - # TODO: remove them once changes we made to geo-index and wkb crates are merged to upstream and released geo-index = { git = "https://github.com/wherobots/geo-index.git", branch = "main" } wkb = { git = "https://github.com/georust/wkb.git", rev = "130eb0c2b343bc9299aeafba6d34c2a6e53f3b6a" } diff --git a/c/sedona-proj/src/st_transform.rs b/c/sedona-proj/src/st_transform.rs index 51a26824..4833a7e1 100644 --- a/c/sedona-proj/src/st_transform.rs +++ b/c/sedona-proj/src/st_transform.rs @@ -608,6 +608,7 @@ mod tests { arg_fields: arg_fields.to_vec(), number_rows: row_count, return_field, + config_options: Arc::new(ConfigOptions::default()), }; let value = udf.invoke_with_args(args)?; diff --git a/python/sedonadb/Cargo.toml b/python/sedonadb/Cargo.toml index 98379bde..34450257 100644 --- a/python/sedonadb/Cargo.toml +++ b/python/sedonadb/Cargo.toml @@ -41,7 +41,8 @@ datafusion-ffi = { workspace = true } futures = { workspace = true } pyo3 = { version = "0.25.1" } sedona = { path = "../../rust/sedona" } -sedona-adbc = { path = "../../rust/sedona-adbc" } +# Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x +# sedona-adbc = { path = "../../rust/sedona-adbc" } sedona-geoparquet = { path = "../../rust/sedona-geoparquet" } sedona-schema = { path = "../../rust/sedona-schema" } sedona-proj = { path = "../../c/sedona-proj", default-features = false } diff --git a/python/sedonadb/src/lib.rs b/python/sedonadb/src/lib.rs index ca09d875..7c8e0a33 100644 --- a/python/sedonadb/src/lib.rs +++ b/python/sedonadb/src/lib.rs @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. use crate::error::PySedonaError; -use pyo3::{ffi::Py_uintptr_t, prelude::*}; -use sedona_adbc::AdbcSedonadbDriverInit; +use pyo3::prelude::*; +// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x +// use pyo3::ffi::Py_uintptr_t; +// use sedona_adbc::AdbcSedonadbDriverInit; +// use std::ffi::c_void; use sedona_proj::register::{configure_global_proj_engine, ProjCrsEngineBuilder}; -use std::ffi::c_void; mod context; mod dataframe; @@ -51,11 +53,12 @@ fn sedona_python_version() -> PyResult { Ok(VERSION.to_string()) } -#[pyfunction] -fn sedona_adbc_driver_init() -> PyResult { - let driver_init_void = AdbcSedonadbDriverInit as *const c_void; - Ok(driver_init_void as Py_uintptr_t) -} +// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x +// #[pyfunction] +// fn sedona_adbc_driver_init() -> PyResult { +// let driver_init_void = AdbcSedonadbDriverInit as *const c_void; +// Ok(driver_init_void as Py_uintptr_t) +// } #[pyfunction] fn configure_proj_shared( @@ -87,7 +90,8 @@ fn _lib(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { configure_tg_allocator(); m.add_function(wrap_pyfunction!(configure_proj_shared, m)?)?; - m.add_function(wrap_pyfunction!(sedona_adbc_driver_init, m)?)?; + // Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x + // m.add_function(wrap_pyfunction!(sedona_adbc_driver_init, m)?)?; m.add_function(wrap_pyfunction!(sedona_python_version, m)?)?; m.add_class::()?; diff --git a/r/sedonadb/src/rust/Cargo.toml b/r/sedonadb/src/rust/Cargo.toml index ced8c24c..9a6b90c6 100644 --- a/r/sedonadb/src/rust/Cargo.toml +++ b/r/sedonadb/src/rust/Cargo.toml @@ -33,7 +33,8 @@ datafusion-ffi = { workspace = true } savvy = "*" savvy-ffi = "*" sedona = { path = "../../../../rust/sedona" } -sedona-adbc = { path = "../../../../rust/sedona-adbc" } +# Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x +# sedona-adbc = { path = "../../../../rust/sedona-adbc" } sedona-expr = { path = "../../../../rust/sedona-expr" } sedona-geoparquet = { path = "../../../../rust/sedona-geoparquet" } sedona-proj = { path = "../../../../c/sedona-proj", default-features = false } diff --git a/r/sedonadb/src/rust/src/lib.rs b/r/sedonadb/src/rust/src/lib.rs index 07c6f311..8c2723ef 100644 --- a/r/sedonadb/src/rust/src/lib.rs +++ b/r/sedonadb/src/rust/src/lib.rs @@ -16,12 +16,12 @@ // under the License. // Example functions -use std::ffi::c_void; - use savvy::savvy; -use savvy_ffi::R_NilValue; -use sedona_adbc::AdbcSedonadbDriverInit; +// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x +// use std::ffi::c_void; +// use savvy_ffi::R_NilValue; +// use sedona_adbc::AdbcSedonadbDriverInit; use sedona_proj::register::{configure_global_proj_engine, ProjCrsEngineBuilder}; mod context; @@ -30,18 +30,19 @@ mod error; mod ffi; mod runtime; -#[savvy] -fn sedonadb_adbc_init_func() -> savvy::Result { - let driver_init_void = AdbcSedonadbDriverInit as *mut c_void; - - unsafe { - Ok(savvy::Sexp(savvy_ffi::R_MakeExternalPtr( - driver_init_void, - R_NilValue, - R_NilValue, - ))) - } -} +// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x +// #[savvy] +// fn sedonadb_adbc_init_func() -> savvy::Result { +// let driver_init_void = AdbcSedonadbDriverInit as *mut c_void; +// +// unsafe { +// Ok(savvy::Sexp(savvy_ffi::R_MakeExternalPtr( +// driver_init_void, +// R_NilValue, +// R_NilValue, +// ))) +// } +// } #[savvy] fn configure_proj_shared( diff --git a/rust/sedona-expr/src/aggregate_udf.rs b/rust/sedona-expr/src/aggregate_udf.rs index 9d04b5f7..3ec999d0 100644 --- a/rust/sedona-expr/src/aggregate_udf.rs +++ b/rust/sedona-expr/src/aggregate_udf.rs @@ -41,6 +41,20 @@ pub struct SedonaAggregateUDF { documentation: Option, } +impl PartialEq for SedonaAggregateUDF { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for SedonaAggregateUDF {} + +impl std::hash::Hash for SedonaAggregateUDF { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + impl SedonaAggregateUDF { /// Create a new SedonaAggregateUDF pub fn new( diff --git a/rust/sedona-expr/src/scalar_udf.rs b/rust/sedona-expr/src/scalar_udf.rs index 7f4c187d..225d0d92 100644 --- a/rust/sedona-expr/src/scalar_udf.rs +++ b/rust/sedona-expr/src/scalar_udf.rs @@ -41,6 +41,20 @@ pub struct SedonaScalarUDF { aliases: Vec, } +impl PartialEq for SedonaScalarUDF { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for SedonaScalarUDF {} + +impl std::hash::Hash for SedonaScalarUDF { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + /// User-defined function implementation /// /// A `SedonaScalarUdf` is comprised of one or more kernels, to which it dispatches diff --git a/rust/sedona-expr/src/spatial_filter.rs b/rust/sedona-expr/src/spatial_filter.rs index 246362b4..8f15a9d3 100644 --- a/rust/sedona-expr/src/spatial_filter.rs +++ b/rust/sedona-expr/src/spatial_filter.rs @@ -589,6 +589,7 @@ mod test { Arc::new(unrelated), vec![], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_no_args).unwrap(), @@ -613,7 +614,7 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -626,7 +627,7 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -650,7 +651,7 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -663,7 +664,7 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -689,7 +690,7 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -702,7 +703,7 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -729,7 +730,7 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -743,7 +744,7 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -769,7 +770,7 @@ mod test { "st_dwithin", Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -782,7 +783,7 @@ mod test { "st_dwithin", Arc::new(st_dwithin), vec![literal.clone(), column.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap(); assert!( @@ -796,7 +797,7 @@ mod test { "st_distance", Arc::new(st_distance.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let comparison_expr: Arc = Arc::new(BinaryExpr::new( distance_expr.clone(), @@ -829,7 +830,7 @@ mod test { "st_dwithin", Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), negative_distance], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -844,7 +845,7 @@ mod test { "st_dwithin", Arc::new(st_dwithin), vec![column.clone(), literal.clone(), nan_distance], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate_nan = SpatialFilter::try_from_expr(&dwithin_expr_nan).unwrap(); assert!( @@ -877,7 +878,7 @@ mod test { "intersects", Arc::new(st_intersects.clone()), vec![], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); assert!(SpatialFilter::try_from_expr(&expr_no_args) .unwrap_err() @@ -889,7 +890,7 @@ mod test { "intersects", Arc::new(st_intersects.clone()), vec![literal.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(), @@ -925,7 +926,7 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -951,7 +952,7 @@ mod test { "st_dwithin", Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -964,7 +965,7 @@ mod test { "st_dwithin", Arc::new(st_dwithin), vec![literal.clone(), column.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap(); assert!( @@ -978,7 +979,7 @@ mod test { "st_distance", Arc::new(st_distance.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let comparison_expr: Arc = Arc::new(BinaryExpr::new( distance_expr.clone(), @@ -1013,7 +1014,7 @@ mod test { "has_z", Arc::new(has_z.clone()), vec![column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!(matches!(predicate, SpatialFilter::HasZ(_))); @@ -1028,7 +1029,7 @@ mod test { "has_z", Arc::new(has_z.clone()), vec![], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); assert!(SpatialFilter::try_from_expr(&expr_no_args) .unwrap_err() @@ -1040,7 +1041,7 @@ mod test { "intersects", Arc::new(has_z.clone()), vec![literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(), diff --git a/rust/sedona-functions/src/st_setsrid.rs b/rust/sedona-functions/src/st_setsrid.rs index 007b21b8..5fed35b7 100644 --- a/rust/sedona-functions/src/st_setsrid.rs +++ b/rust/sedona-functions/src/st_setsrid.rs @@ -374,6 +374,7 @@ mod test { arg_fields, number_rows: 1, return_field, + config_options: Arc::new(ConfigOptions::default()), }; let value = udf.invoke_with_args(args)?; diff --git a/rust/sedona-geoparquet/Cargo.toml b/rust/sedona-geoparquet/Cargo.toml index 0a5110f3..d5acaeec 100644 --- a/rust/sedona-geoparquet/Cargo.toml +++ b/rust/sedona-geoparquet/Cargo.toml @@ -46,6 +46,7 @@ chrono = { workspace = true } datafusion = { workspace = true, features = ["parquet"] } datafusion-catalog = { workspace = true } datafusion-common = { workspace = true } +datafusion-datasource-parquet = "50.2.0" datafusion-execution = { workspace = true } datafusion-expr = { workspace = true } datafusion-physical-expr = { workspace = true } diff --git a/rust/sedona-geoparquet/src/file_opener.rs b/rust/sedona-geoparquet/src/file_opener.rs index a71c9545..35d30487 100644 --- a/rust/sedona-geoparquet/src/file_opener.rs +++ b/rust/sedona-geoparquet/src/file_opener.rs @@ -18,10 +18,10 @@ use std::{collections::HashMap, sync::Arc}; use arrow_schema::SchemaRef; use datafusion::datasource::{ - file_format::parquet::fetch_parquet_metadata, listing::PartitionedFile, physical_plan::{parquet::ParquetAccessPlan, FileMeta, FileOpenFuture, FileOpener}, }; +use datafusion_datasource_parquet::metadata::DFParquetMetadata; use datafusion_common::Result; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_plan::metrics::{Count, ExecutionPlanMetricsSet, MetricBuilder}; @@ -113,12 +113,12 @@ impl FileOpener for GeoParquetFileOpener { let self_clone = self.clone(); Ok(Box::pin(async move { - let parquet_metadata = fetch_parquet_metadata( + let parquet_metadata = DFParquetMetadata::new( &self_clone.object_store, &file_meta.object_meta, - self_clone.metadata_size_hint, - None, ) + .with_metadata_size_hint(self_clone.metadata_size_hint) + .fetch_metadata() .await?; let mut access_plan = ParquetAccessPlan::new_all(parquet_metadata.num_row_groups()); diff --git a/rust/sedona-geoparquet/src/format.rs b/rust/sedona-geoparquet/src/format.rs index 8ff87177..75eafb9d 100644 --- a/rust/sedona-geoparquet/src/format.rs +++ b/rust/sedona-geoparquet/src/format.rs @@ -24,7 +24,7 @@ use datafusion::{ datasource::{ file_format::{ file_compression_type::FileCompressionType, - parquet::{fetch_parquet_metadata, ParquetFormat, ParquetFormatFactory}, + parquet::{ParquetFormat, ParquetFormatFactory}, FileFormat, FileFormatFactory, }, physical_plan::{ @@ -34,6 +34,7 @@ use datafusion::{ }; use datafusion_catalog::{memory::DataSourceExec, Session}; use datafusion_common::{plan_err, GetExt, Result, Statistics}; +use datafusion_datasource_parquet::metadata::DFParquetMetadata; use datafusion_physical_expr::{LexRequirement, PhysicalExpr}; use datafusion_physical_plan::{ filter_pushdown::FilterPushdownPropagation, metrics::ExecutionPlanMetricsSet, ExecutionPlan, @@ -182,13 +183,11 @@ impl FileFormat for GeoParquetFormat { // copy more ParquetFormat code. It may be that caching at the object // store level is the way to go here. let metadatas: Vec<_> = futures::stream::iter(objects) - .map(|object| { - fetch_parquet_metadata( - store.as_ref(), - object, - self.inner().metadata_size_hint(), - None, - ) + .map(|object| async move { + DFParquetMetadata::new(store.as_ref(), object) + .with_metadata_size_hint(self.inner().metadata_size_hint()) + .fetch_metadata() + .await }) .boxed() // Workaround https://github.com/rust-lang/rust/issues/64552 .buffered(state.config_options().execution.meta_fetch_concurrency) diff --git a/rust/sedona-geoparquet/src/writer.rs b/rust/sedona-geoparquet/src/writer.rs index ef741e1d..c50e57d0 100644 --- a/rust/sedona-geoparquet/src/writer.rs +++ b/rust/sedona-geoparquet/src/writer.rs @@ -28,7 +28,7 @@ use datafusion::{ file_format::parquet::ParquetSink, physical_plan::FileSinkConfig, sink::DataSinkExec, }, }; -use datafusion_common::{exec_datafusion_err, exec_err, not_impl_err, DataFusionError, Result}; +use datafusion_common::{config::ConfigOptions, exec_datafusion_err, exec_err, not_impl_err, DataFusionError, Result}; use datafusion_expr::{dml::InsertOp, ColumnarValue, ScalarUDF, Volatility}; use datafusion_physical_expr::{ expressions::Column, LexRequirement, PhysicalExpr, ScalarFunctionExpr, @@ -230,6 +230,7 @@ fn project_bboxes( bbox_udf.clone(), vec![column], Arc::new(Field::new("", bbox_type(), true)), + Arc::new(ConfigOptions::default()), )); bbox_exprs.insert(i, (expr, bbox_field_name.clone())); diff --git a/rust/sedona-schema/src/schema.rs b/rust/sedona-schema/src/schema.rs index c3732c26..9ce1d3a4 100644 --- a/rust/sedona-schema/src/schema.rs +++ b/rust/sedona-schema/src/schema.rs @@ -41,15 +41,18 @@ pub trait SedonaSchema { impl SedonaSchema for DFSchema { fn sedona_types(&self) -> impl ExactSizeIterator> { - self.as_arrow().sedona_types() + let arrow_schema = self.as_arrow(); + ::sedona_types(arrow_schema) } fn geometry_column_indices(&self) -> Result> { - self.as_arrow().geometry_column_indices() + let arrow_schema = self.as_arrow(); + ::geometry_column_indices(arrow_schema) } fn primary_geometry_column_index(&self) -> Result> { - self.as_arrow().primary_geometry_column_index() + let arrow_schema = self.as_arrow(); + ::primary_geometry_column_index(arrow_schema) } } diff --git a/rust/sedona-spatial-join/src/optimizer.rs b/rust/sedona-spatial-join/src/optimizer.rs index e996906c..52571ce9 100644 --- a/rust/sedona-spatial-join/src/optimizer.rs +++ b/rust/sedona-spatial-join/src/optimizer.rs @@ -133,7 +133,8 @@ impl OptimizerRule for SpatialJoinOptimizer { plan: LogicalPlan, config: &dyn OptimizerConfig, ) -> Result> { - let Some(extension) = config.options().extensions.get::() else { + let options = config.options(); + let Some(extension) = options.extensions.get::() else { return Ok(Transformed::no(plan)); }; if !extension.spatial_join.enable { @@ -1166,6 +1167,7 @@ mod tests { Arc::clone(&udf), args, field, + Arc::new(ConfigOptions::default()), )) } diff --git a/rust/sedona-testing/src/testers.rs b/rust/sedona-testing/src/testers.rs index a97bbeee..a53cf855 100644 --- a/rust/sedona-testing/src/testers.rs +++ b/rust/sedona-testing/src/testers.rs @@ -18,7 +18,7 @@ use std::{iter::zip, sync::Arc}; use arrow_array::{ArrayRef, RecordBatch}; use arrow_schema::{FieldRef, Schema}; -use datafusion_common::{Result, ScalarValue}; +use datafusion_common::{config::ConfigOptions, Result, ScalarValue}; use datafusion_expr::{ function::{AccumulatorArgs, StateFieldsArgs}, Accumulator, AggregateUDF, ColumnarValue, Expr, Literal, ReturnFieldArgs, ScalarFunctionArgs, @@ -404,6 +404,7 @@ impl ScalarUdfTester { arg_fields: self.arg_fields(), number_rows, return_field: self.return_type()?.to_storage_field("", true)?.into(), + config_options: Arc::new(ConfigOptions::default()), }; self.udf.invoke_with_args(args) diff --git a/rust/sedona/src/context.rs b/rust/sedona/src/context.rs index ded65aa4..744dca22 100644 --- a/rust/sedona/src/context.rs +++ b/rust/sedona/src/context.rs @@ -539,7 +539,7 @@ mod tests { "| plan_type | plan |", "+---------------+---------------------------------+", "| logical_plan | Projection: Int64(1) AS one |", - "| | EmptyRelation |", + "| | EmptyRelation: rows=1 |", "| physical_plan | ProjectionExec: expr=[1 as one] |", "| | PlaceholderRowExec |", "| | |", diff --git a/rust/sedona/src/ffi.rs b/rust/sedona/src/ffi.rs index c4b0e1b1..e1cb1f7b 100644 --- a/rust/sedona/src/ffi.rs +++ b/rust/sedona/src/ffi.rs @@ -19,7 +19,7 @@ use std::{any::Any, sync::Arc}; use abi_stable::StableAbi; use arrow_schema::{DataType, Field, FieldRef, Schema}; use datafusion::physical_plan::{expressions::Column, PhysicalExpr}; -use datafusion_common::{DataFusionError, Result, ScalarValue}; +use datafusion_common::{config::ConfigOptions, DataFusionError, Result, ScalarValue}; use datafusion_expr::{ function::{AccumulatorArgs, StateFieldsArgs}, Accumulator, AggregateUDF, AggregateUDFImpl, ColumnarValue, ReturnFieldArgs, @@ -89,6 +89,20 @@ struct ExportedScalarKernel { sedona_impl: ScalarKernelRef, } +impl PartialEq for ExportedScalarKernel { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for ExportedScalarKernel {} + +impl std::hash::Hash for ExportedScalarKernel { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + impl From for ExportedScalarKernel { fn from(value: ScalarKernelRef) -> Self { Self { @@ -196,6 +210,7 @@ impl SedonaScalarKernel for ImportedScalarKernel { number_rows: arg_rows.unwrap_or(1), // Wrapper code on the other side of this doesn't use this value return_field: Field::new("", DataType::Null, true).into(), + config_options: Arc::new(ConfigOptions::default()), }; // DataFusion's FFI_ScalarUDF always returns array output but @@ -273,6 +288,20 @@ struct ExportedSedonaAccumulator { sedona_impl: SedonaAccumulatorRef, } +impl PartialEq for ExportedSedonaAccumulator { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for ExportedSedonaAccumulator {} + +impl std::hash::Hash for ExportedSedonaAccumulator { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + impl From for ExportedSedonaAccumulator { fn from(value: SedonaAccumulatorRef) -> Self { Self { diff --git a/rust/sedona/src/random_geometry_provider.rs b/rust/sedona/src/random_geometry_provider.rs index c98f415d..00b67dc9 100644 --- a/rust/sedona/src/random_geometry_provider.rs +++ b/rust/sedona/src/random_geometry_provider.rs @@ -199,7 +199,7 @@ impl TableProvider for RandomGeometryProvider { // We're required to handle the projection or we'll get an execution error if let Some(projection) = projection { let schema = self.schema(); - let exprs = projection + let exprs: Vec<_> = projection .iter() .map(|index| -> (Arc, String) { let name = schema.field(*index).name(); diff --git a/rust/sedona/src/show.rs b/rust/sedona/src/show.rs index f1ac3621..63089dfa 100644 --- a/rust/sedona/src/show.rs +++ b/rust/sedona/src/show.rs @@ -21,7 +21,7 @@ use comfy_table::{Cell, CellAlignment, ColumnConstraint, ContentArrangement, Row use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; use datafusion::error::Result; use datafusion_common::format::DEFAULT_FORMAT_OPTIONS; -use datafusion_common::{DataFusionError, ScalarValue}; +use datafusion_common::{config::ConfigOptions, DataFusionError, ScalarValue}; use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF}; use sedona_expr::scalar_udf::SedonaScalarUDF; use sedona_schema::{datatypes::SedonaType, matchers::ArgMatcher}; @@ -252,7 +252,8 @@ impl<'a> DisplayTable<'a> { } } - table.set_truncation_indicator(self.truncation_indicator()); + // TODO: set_truncation_indicator method was removed in comfy_table 7.x + // table.set_truncation_indicator(self.truncation_indicator()); let mut columns = self .columns @@ -495,6 +496,7 @@ impl DisplayColumn { arg_fields, number_rows: array.len(), return_field, + config_options: Arc::new(ConfigOptions::default()), }; let format_proxy_value = format_udf.invoke_with_args(args)?; From 819e6a07ae6eb4a0b42aeb56563114daf6c00521 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Thu, 23 Oct 2025 12:03:57 -0700 Subject: [PATCH 2/9] patch adbc_core to use release 0.21.0 --- Cargo.lock | 249 +++++++++++++++------------------ Cargo.toml | 11 +- python/sedonadb/Cargo.toml | 3 +- python/sedonadb/src/lib.rs | 22 ++- r/sedonadb/src/rust/Cargo.toml | 3 +- r/sedonadb/src/rust/src/lib.rs | 33 +++-- 6 files changed, 147 insertions(+), 174 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 889ea043..9e1c3282 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,12 +52,21 @@ dependencies = [ [[package]] name = "adbc_core" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b891479797b5588e320f7fd3caf15faba311cf8f8a76911195b6a3d55304eb" +version = "0.21.0" +source = "git+https://github.com/apache/arrow-adbc.git#d57cc5013f9e6a38f97c77c05149d9ee5669d46a" dependencies = [ - "arrow-array 55.2.0", - "arrow-schema 55.2.0", + "arrow-array", + "arrow-schema", +] + +[[package]] +name = "adbc_ffi" +version = "0.21.0" +source = "git+https://github.com/apache/arrow-adbc.git#d57cc5013f9e6a38f97c77c05149d9ee5669d46a" +dependencies = [ + "adbc_core", + "arrow-array", + "arrow-schema", ] [[package]] @@ -247,16 +256,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ "arrow-arith", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", + "arrow-array", + "arrow-buffer", "arrow-cast", "arrow-csv", - "arrow-data 56.2.0", + "arrow-data", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema 56.2.0", + "arrow-schema", "arrow-select", "arrow-string", ] @@ -267,30 +276,14 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num", ] -[[package]] -name = "arrow-array" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" -dependencies = [ - "ahash", - "arrow-buffer 55.2.0", - "arrow-data 55.2.0", - "arrow-schema 55.2.0", - "chrono", - "half", - "hashbrown 0.15.5", - "num", -] - [[package]] name = "arrow-array" version = "56.2.0" @@ -298,9 +291,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -308,17 +301,6 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-buffer" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" -dependencies = [ - "bytes", - "half", - "num", -] - [[package]] name = "arrow-buffer" version = "56.2.0" @@ -336,10 +318,10 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "atoi", "base64", @@ -357,35 +339,23 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" dependencies = [ - "arrow-array 56.2.0", + "arrow-array", "arrow-cast", - "arrow-schema 56.2.0", + "arrow-schema", "chrono", "csv", "csv-core", "regex", ] -[[package]] -name = "arrow-data" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" -dependencies = [ - "arrow-buffer 55.2.0", - "arrow-schema 55.2.0", - "half", - "num", -] - [[package]] name = "arrow-data" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ - "arrow-buffer 56.2.0", - "arrow-schema 56.2.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] @@ -396,10 +366,10 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "flatbuffers", "lz4_flex", @@ -412,11 +382,11 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", @@ -434,10 +404,10 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", ] @@ -447,22 +417,13 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" -dependencies = [ - "bitflags", -] - [[package]] name = "arrow-schema" version = "56.2.0" @@ -481,10 +442,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] @@ -494,10 +455,10 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "memchr", "num", @@ -1643,7 +1604,7 @@ checksum = "fc6759cf9ef57c5c469e4027ac4b4cfa746e06a0f5472c2b922b6a403c2a64c4" dependencies = [ "arrow", "arrow-ipc", - "arrow-schema 56.2.0", + "arrow-schema", "async-trait", "bytes", "bzip2 0.6.0", @@ -1991,7 +1952,7 @@ checksum = "a4bef25e2b86d9921f7a98b1a86bfb50cebe2fd97f3a9b96c85ce475e9ef78b0" dependencies = [ "abi_stable", "arrow", - "arrow-schema 56.2.0", + "arrow-schema", "async-ffi", "async-trait", "datafusion", @@ -2012,7 +1973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f362c78ac283e64fd3976e060c1a8a57d5f4dcf844a6b6bd2eb320640a1572e" dependencies = [ "arrow", - "arrow-buffer 56.2.0", + "arrow-buffer", "base64", "blake2", "blake3", @@ -2246,7 +2207,7 @@ dependencies = [ "ahash", "arrow", "arrow-ord", - "arrow-schema 56.2.0", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2302,7 +2263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f84b866d906118c320459f30385048aeedbe36ac06973d3e4fa0cc5d60d722c" dependencies = [ "arrow", - "arrow-schema 56.2.0", + "arrow-schema", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -3277,7 +3238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.15.5", ] [[package]] @@ -3950,12 +3911,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" dependencies = [ "ahash", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 56.2.0", + "arrow-data", "arrow-ipc", - "arrow-schema 56.2.0", + "arrow-schema", "arrow-select", "base64", "brotli", @@ -4869,8 +4830,8 @@ name = "sedona" version = "0.2.0" dependencies = [ "abi_stable", - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "async-trait", "aws-config", "aws-credential-types", @@ -4906,6 +4867,20 @@ dependencies = [ "url", ] +[[package]] +name = "sedona-adbc" +version = "0.2.0" +dependencies = [ + "adbc_core", + "adbc_ffi", + "arrow-array", + "arrow-schema", + "datafusion", + "futures", + "sedona", + "tokio", +] + [[package]] name = "sedona-cli" version = "0.2.0" @@ -4938,8 +4913,8 @@ dependencies = [ name = "sedona-expr" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", @@ -4957,9 +4932,9 @@ dependencies = [ name = "sedona-functions" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", + "arrow-array", "arrow-json", - "arrow-schema 56.2.0", + "arrow-schema", "criterion", "datafusion", "datafusion-common", @@ -4981,8 +4956,8 @@ dependencies = [ name = "sedona-geo" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "criterion", "datafusion-common", "datafusion-expr", @@ -5041,8 +5016,8 @@ dependencies = [ name = "sedona-geoarrow-c" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "cc", "criterion", "datafusion-common", @@ -5077,8 +5052,8 @@ dependencies = [ name = "sedona-geoparquet" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "async-trait", "bytes", "chrono", @@ -5114,8 +5089,8 @@ dependencies = [ name = "sedona-geos" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "byteorder", "criterion", "datafusion-common", @@ -5138,8 +5113,8 @@ name = "sedona-proj" version = "0.2.0" dependencies = [ "approx", - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "cc", "criterion", "datafusion-common", @@ -5162,8 +5137,8 @@ dependencies = [ name = "sedona-s2geography" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "cmake", "criterion", "datafusion-common", @@ -5184,8 +5159,8 @@ dependencies = [ name = "sedona-schema" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "datafusion-common", "sedona-common", "serde_json", @@ -5196,8 +5171,8 @@ name = "sedona-spatial-join" version = "0.2.0" dependencies = [ "arrow", - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "criterion", "datafusion", "datafusion-common", @@ -5237,9 +5212,9 @@ dependencies = [ name = "sedona-testing" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", + "arrow-array", "arrow-cast", - "arrow-schema 56.2.0", + "arrow-schema", "criterion", "datafusion-common", "datafusion-expr", @@ -5262,8 +5237,8 @@ dependencies = [ name = "sedona-tg" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "cc", "criterion", "datafusion-common", @@ -5284,8 +5259,8 @@ name = "sedonadb" version = "0.2.0" dependencies = [ "adbc_core", - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "async-trait", "datafusion", "datafusion-common", @@ -5296,6 +5271,7 @@ dependencies = [ "mimalloc", "pyo3", "sedona", + "sedona-adbc", "sedona-geoparquet", "sedona-proj", "sedona-schema", @@ -5308,8 +5284,8 @@ dependencies = [ name = "sedonadbr" version = "0.2.0" dependencies = [ - "arrow-array 56.2.0", - "arrow-schema 56.2.0", + "arrow-array", + "arrow-schema", "datafusion", "datafusion-common", "datafusion-expr", @@ -5317,6 +5293,7 @@ dependencies = [ "savvy", "savvy-ffi", "sedona", + "sedona-adbc", "sedona-expr", "sedona-geoparquet", "sedona-proj", diff --git a/Cargo.toml b/Cargo.toml index 73333d4f..62b98ea4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,7 @@ members = [ "r/sedonadb/src/rust", "rust/sedona-geo-traits-ext", "rust/sedona-geo-generic-alg", - # Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x - # "rust/sedona-adbc", + "rust/sedona-adbc", "rust/sedona-expr", "rust/sedona-functions", "rust/sedona-geo", @@ -57,8 +56,8 @@ rust-version = "1.82" [workspace.dependencies] approx = "0.5" abi_stable = "0.11.3" -adbc_core = "0.20.0" -adbc_ffi = "0.20.0" +adbc_core = ">=0.20.0" +adbc_ffi = ">=0.20.0" lru = "0.12" arrow = { version = "56.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } arrow-array = { version = "56.0.0" } @@ -120,6 +119,10 @@ tokio = { version = "1.44" } url = "2.5.4" [patch.crates-io] +# Use main branch of arrow-adbc which supports Arrow 56.x (remove when 0.21.0 is released) +adbc_core = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_core" } +adbc_ffi = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_ffi" } + # TODO: remove them once changes we made to geo-index and wkb crates are merged to upstream and released geo-index = { git = "https://github.com/wherobots/geo-index.git", branch = "main" } wkb = { git = "https://github.com/georust/wkb.git", rev = "130eb0c2b343bc9299aeafba6d34c2a6e53f3b6a" } diff --git a/python/sedonadb/Cargo.toml b/python/sedonadb/Cargo.toml index 34450257..98379bde 100644 --- a/python/sedonadb/Cargo.toml +++ b/python/sedonadb/Cargo.toml @@ -41,8 +41,7 @@ datafusion-ffi = { workspace = true } futures = { workspace = true } pyo3 = { version = "0.25.1" } sedona = { path = "../../rust/sedona" } -# Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x -# sedona-adbc = { path = "../../rust/sedona-adbc" } +sedona-adbc = { path = "../../rust/sedona-adbc" } sedona-geoparquet = { path = "../../rust/sedona-geoparquet" } sedona-schema = { path = "../../rust/sedona-schema" } sedona-proj = { path = "../../c/sedona-proj", default-features = false } diff --git a/python/sedonadb/src/lib.rs b/python/sedonadb/src/lib.rs index 7c8e0a33..ca09d875 100644 --- a/python/sedonadb/src/lib.rs +++ b/python/sedonadb/src/lib.rs @@ -15,12 +15,10 @@ // specific language governing permissions and limitations // under the License. use crate::error::PySedonaError; -use pyo3::prelude::*; -// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x -// use pyo3::ffi::Py_uintptr_t; -// use sedona_adbc::AdbcSedonadbDriverInit; -// use std::ffi::c_void; +use pyo3::{ffi::Py_uintptr_t, prelude::*}; +use sedona_adbc::AdbcSedonadbDriverInit; use sedona_proj::register::{configure_global_proj_engine, ProjCrsEngineBuilder}; +use std::ffi::c_void; mod context; mod dataframe; @@ -53,12 +51,11 @@ fn sedona_python_version() -> PyResult { Ok(VERSION.to_string()) } -// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x -// #[pyfunction] -// fn sedona_adbc_driver_init() -> PyResult { -// let driver_init_void = AdbcSedonadbDriverInit as *const c_void; -// Ok(driver_init_void as Py_uintptr_t) -// } +#[pyfunction] +fn sedona_adbc_driver_init() -> PyResult { + let driver_init_void = AdbcSedonadbDriverInit as *const c_void; + Ok(driver_init_void as Py_uintptr_t) +} #[pyfunction] fn configure_proj_shared( @@ -90,8 +87,7 @@ fn _lib(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { configure_tg_allocator(); m.add_function(wrap_pyfunction!(configure_proj_shared, m)?)?; - // Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x - // m.add_function(wrap_pyfunction!(sedona_adbc_driver_init, m)?)?; + m.add_function(wrap_pyfunction!(sedona_adbc_driver_init, m)?)?; m.add_function(wrap_pyfunction!(sedona_python_version, m)?)?; m.add_class::()?; diff --git a/r/sedonadb/src/rust/Cargo.toml b/r/sedonadb/src/rust/Cargo.toml index 9a6b90c6..ced8c24c 100644 --- a/r/sedonadb/src/rust/Cargo.toml +++ b/r/sedonadb/src/rust/Cargo.toml @@ -33,8 +33,7 @@ datafusion-ffi = { workspace = true } savvy = "*" savvy-ffi = "*" sedona = { path = "../../../../rust/sedona" } -# Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x -# sedona-adbc = { path = "../../../../rust/sedona-adbc" } +sedona-adbc = { path = "../../../../rust/sedona-adbc" } sedona-expr = { path = "../../../../rust/sedona-expr" } sedona-geoparquet = { path = "../../../../rust/sedona-geoparquet" } sedona-proj = { path = "../../../../c/sedona-proj", default-features = false } diff --git a/r/sedonadb/src/rust/src/lib.rs b/r/sedonadb/src/rust/src/lib.rs index 8c2723ef..07c6f311 100644 --- a/r/sedonadb/src/rust/src/lib.rs +++ b/r/sedonadb/src/rust/src/lib.rs @@ -16,12 +16,12 @@ // under the License. // Example functions +use std::ffi::c_void; + use savvy::savvy; -// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x -// use std::ffi::c_void; -// use savvy_ffi::R_NilValue; -// use sedona_adbc::AdbcSedonadbDriverInit; +use savvy_ffi::R_NilValue; +use sedona_adbc::AdbcSedonadbDriverInit; use sedona_proj::register::{configure_global_proj_engine, ProjCrsEngineBuilder}; mod context; @@ -30,19 +30,18 @@ mod error; mod ffi; mod runtime; -// Temporarily excluded due to adbc_core 0.20.0 requiring Arrow 55.x -// #[savvy] -// fn sedonadb_adbc_init_func() -> savvy::Result { -// let driver_init_void = AdbcSedonadbDriverInit as *mut c_void; -// -// unsafe { -// Ok(savvy::Sexp(savvy_ffi::R_MakeExternalPtr( -// driver_init_void, -// R_NilValue, -// R_NilValue, -// ))) -// } -// } +#[savvy] +fn sedonadb_adbc_init_func() -> savvy::Result { + let driver_init_void = AdbcSedonadbDriverInit as *mut c_void; + + unsafe { + Ok(savvy::Sexp(savvy_ffi::R_MakeExternalPtr( + driver_init_void, + R_NilValue, + R_NilValue, + ))) + } +} #[savvy] fn configure_proj_shared( From 62a1cce3947856f65408850f37cd8b66d846a23a Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Thu, 23 Oct 2025 13:14:36 -0700 Subject: [PATCH 3/9] fix pre-commit lint --- rust/sedona-expr/src/spatial_filter.rs | 66 +++++++++++++++-------- rust/sedona-geoparquet/src/file_opener.rs | 14 +++-- rust/sedona-geoparquet/src/writer.rs | 4 +- 3 files changed, 53 insertions(+), 31 deletions(-) diff --git a/rust/sedona-expr/src/spatial_filter.rs b/rust/sedona-expr/src/spatial_filter.rs index 8f15a9d3..82dd1b89 100644 --- a/rust/sedona-expr/src/spatial_filter.rs +++ b/rust/sedona-expr/src/spatial_filter.rs @@ -614,7 +614,8 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -627,7 +628,8 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -651,7 +653,8 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -664,7 +667,8 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -690,7 +694,8 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -703,7 +708,8 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -730,7 +736,8 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -744,7 +751,8 @@ mod test { func_name, Arc::new(func), vec![literal.clone(), column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -770,7 +778,8 @@ mod test { "st_dwithin", Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -783,7 +792,8 @@ mod test { "st_dwithin", Arc::new(st_dwithin), vec![literal.clone(), column.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap(); assert!( @@ -797,7 +807,8 @@ mod test { "st_distance", Arc::new(st_distance.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let comparison_expr: Arc = Arc::new(BinaryExpr::new( distance_expr.clone(), @@ -830,7 +841,8 @@ mod test { "st_dwithin", Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), negative_distance], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -845,7 +857,8 @@ mod test { "st_dwithin", Arc::new(st_dwithin), vec![column.clone(), literal.clone(), nan_distance], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_nan = SpatialFilter::try_from_expr(&dwithin_expr_nan).unwrap(); assert!( @@ -878,7 +891,8 @@ mod test { "intersects", Arc::new(st_intersects.clone()), vec![], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(SpatialFilter::try_from_expr(&expr_no_args) .unwrap_err() @@ -890,7 +904,8 @@ mod test { "intersects", Arc::new(st_intersects.clone()), vec![literal.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(), @@ -926,7 +941,8 @@ mod test { func_name, Arc::new(func.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -952,7 +968,8 @@ mod test { "st_dwithin", Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -965,7 +982,8 @@ mod test { "st_dwithin", Arc::new(st_dwithin), vec![literal.clone(), column.clone(), distance_literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap(); assert!( @@ -979,7 +997,8 @@ mod test { "st_distance", Arc::new(st_distance.clone()), vec![column.clone(), literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let comparison_expr: Arc = Arc::new(BinaryExpr::new( distance_expr.clone(), @@ -1014,7 +1033,8 @@ mod test { "has_z", Arc::new(has_z.clone()), vec![column.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!(matches!(predicate, SpatialFilter::HasZ(_))); @@ -1029,7 +1049,8 @@ mod test { "has_z", Arc::new(has_z.clone()), vec![], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(SpatialFilter::try_from_expr(&expr_no_args) .unwrap_err() @@ -1041,7 +1062,8 @@ mod test { "intersects", Arc::new(has_z.clone()), vec![literal.clone()], - Arc::new(Field::new("", DataType::Boolean, true)), Arc::new(ConfigOptions::default()), + Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(), diff --git a/rust/sedona-geoparquet/src/file_opener.rs b/rust/sedona-geoparquet/src/file_opener.rs index 35d30487..183fe0a8 100644 --- a/rust/sedona-geoparquet/src/file_opener.rs +++ b/rust/sedona-geoparquet/src/file_opener.rs @@ -21,8 +21,8 @@ use datafusion::datasource::{ listing::PartitionedFile, physical_plan::{parquet::ParquetAccessPlan, FileMeta, FileOpenFuture, FileOpener}, }; -use datafusion_datasource_parquet::metadata::DFParquetMetadata; use datafusion_common::Result; +use datafusion_datasource_parquet::metadata::DFParquetMetadata; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_plan::metrics::{Count, ExecutionPlanMetricsSet, MetricBuilder}; use object_store::ObjectStore; @@ -113,13 +113,11 @@ impl FileOpener for GeoParquetFileOpener { let self_clone = self.clone(); Ok(Box::pin(async move { - let parquet_metadata = DFParquetMetadata::new( - &self_clone.object_store, - &file_meta.object_meta, - ) - .with_metadata_size_hint(self_clone.metadata_size_hint) - .fetch_metadata() - .await?; + let parquet_metadata = + DFParquetMetadata::new(&self_clone.object_store, &file_meta.object_meta) + .with_metadata_size_hint(self_clone.metadata_size_hint) + .fetch_metadata() + .await?; let mut access_plan = ParquetAccessPlan::new_all(parquet_metadata.num_row_groups()); diff --git a/rust/sedona-geoparquet/src/writer.rs b/rust/sedona-geoparquet/src/writer.rs index c50e57d0..bf137858 100644 --- a/rust/sedona-geoparquet/src/writer.rs +++ b/rust/sedona-geoparquet/src/writer.rs @@ -28,7 +28,9 @@ use datafusion::{ file_format::parquet::ParquetSink, physical_plan::FileSinkConfig, sink::DataSinkExec, }, }; -use datafusion_common::{config::ConfigOptions, exec_datafusion_err, exec_err, not_impl_err, DataFusionError, Result}; +use datafusion_common::{ + config::ConfigOptions, exec_datafusion_err, exec_err, not_impl_err, DataFusionError, Result, +}; use datafusion_expr::{dml::InsertOp, ColumnarValue, ScalarUDF, Volatility}; use datafusion_physical_expr::{ expressions::Column, LexRequirement, PhysicalExpr, ScalarFunctionExpr, From 78ad3288eb01a7e52b2d9fc6e3d0b41a4b93df58 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Thu, 23 Oct 2025 13:53:18 -0700 Subject: [PATCH 4/9] fix cargo clippy errors --- c/sedona-proj/src/st_transform.rs | 1 + rust/sedona-expr/src/spatial_filter.rs | 1 + rust/sedona-functions/src/st_setsrid.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/c/sedona-proj/src/st_transform.rs b/c/sedona-proj/src/st_transform.rs index 4833a7e1..97f0781c 100644 --- a/c/sedona-proj/src/st_transform.rs +++ b/c/sedona-proj/src/st_transform.rs @@ -331,6 +331,7 @@ mod tests { use super::*; use arrow_array::ArrayRef; use arrow_schema::{DataType, Field}; + use datafusion_common::config::ConfigOptions; use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl}; use rstest::rstest; use sedona_expr::scalar_udf::SedonaScalarUDF; diff --git a/rust/sedona-expr/src/spatial_filter.rs b/rust/sedona-expr/src/spatial_filter.rs index 82dd1b89..83e314eb 100644 --- a/rust/sedona-expr/src/spatial_filter.rs +++ b/rust/sedona-expr/src/spatial_filter.rs @@ -402,6 +402,7 @@ fn parse_args(args: &[Arc]) -> Vec> { #[cfg(test)] mod test { use arrow_schema::{DataType, Field}; + use datafusion_common::config::ConfigOptions; use datafusion_expr::{ScalarUDF, Signature, SimpleScalarUDF, Volatility}; use rstest::rstest; use sedona_geometry::{bounding_box::BoundingBox, interval::Interval}; diff --git a/rust/sedona-functions/src/st_setsrid.rs b/rust/sedona-functions/src/st_setsrid.rs index 5fed35b7..68d916eb 100644 --- a/rust/sedona-functions/src/st_setsrid.rs +++ b/rust/sedona-functions/src/st_setsrid.rs @@ -232,6 +232,7 @@ mod test { use std::rc::Rc; use arrow_schema::Field; + use datafusion_common::config::ConfigOptions; use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF}; use sedona_geometry::{error::SedonaGeometryError, transform::CrsTransform}; use sedona_schema::{ From b99d3d529486abe6511a4cd43654950fb68f2eb9 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Thu, 23 Oct 2025 14:52:02 -0700 Subject: [PATCH 5/9] fix python tests due to df v50 show() changes --- python/sedonadb/tests/test_dataframe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sedonadb/tests/test_dataframe.py b/python/sedonadb/tests/test_dataframe.py index acf27ecd..bb3ca782 100644 --- a/python/sedonadb/tests/test_dataframe.py +++ b/python/sedonadb/tests/test_dataframe.py @@ -444,7 +444,7 @@ def test_show_explained(con, capsys): │ utf8 ┆ utf8 │ ╞═══════════════╪═════════════════════════════════╡ │ logical_plan ┆ Projection: Int64(1) AS one │ -│ ┆ EmptyRelation │ +│ ┆ EmptyRelation: rows=1 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ physical_plan ┆ ProjectionExec: expr=[1 as one] │ │ ┆ PlaceholderRowExec │ @@ -462,7 +462,7 @@ def test_explain(con, capsys): │ utf8 ┆ utf8 │ ╞═══════════════╪═════════════════════════════════╡ │ logical_plan ┆ Projection: Int64(1) AS one │ -│ ┆ EmptyRelation │ +│ ┆ EmptyRelation: rows=1 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ physical_plan ┆ ProjectionExec: expr=[1 as one] │ │ ┆ PlaceholderRowExec │ From bbf4acdbb59ea72d5caab0e120df01b7c310af76 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Thu, 23 Oct 2025 17:43:41 -0700 Subject: [PATCH 6/9] fix doctest --- python/sedonadb/python/sedonadb/dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sedonadb/python/sedonadb/dataframe.py b/python/sedonadb/python/sedonadb/dataframe.py index d1c3288b..f1b641b9 100644 --- a/python/sedonadb/python/sedonadb/dataframe.py +++ b/python/sedonadb/python/sedonadb/dataframe.py @@ -441,7 +441,7 @@ def explain( │ utf8 ┆ utf8 │ ╞═══════════════╪═════════════════════════════════╡ │ logical_plan ┆ Projection: Int64(1) AS one │ - │ ┆ EmptyRelation │ + │ ┆ EmptyRelation: rows=1 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ physical_plan ┆ ProjectionExec: expr=[1 as one] │ │ ┆ PlaceholderRowExec │ From c2cd657ae3df70b1240ba2f2ff3f7f82600e4750 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Mon, 27 Oct 2025 12:56:00 -0700 Subject: [PATCH 7/9] move datafusion-datasource-parquet to parent --- Cargo.toml | 1 + rust/sedona-geoparquet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 62b98ea4..548c46de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ datafusion = { version = "50.2.0", default-features = false } datafusion-catalog = { version = "50.2.0" } datafusion-common = { version = "50.2.0", default-features = false } datafusion-common-runtime = { version = "50.2.0", default-features = false } +datafusion-datasource-parquet = { version = "50.2.0" } datafusion-execution = { version = "50.2.0", default-features = false } datafusion-expr = { version = "50.2.0" } datafusion-ffi = { version = "50.2.0" } diff --git a/rust/sedona-geoparquet/Cargo.toml b/rust/sedona-geoparquet/Cargo.toml index d5acaeec..cba3bd52 100644 --- a/rust/sedona-geoparquet/Cargo.toml +++ b/rust/sedona-geoparquet/Cargo.toml @@ -46,7 +46,7 @@ chrono = { workspace = true } datafusion = { workspace = true, features = ["parquet"] } datafusion-catalog = { workspace = true } datafusion-common = { workspace = true } -datafusion-datasource-parquet = "50.2.0" +datafusion-datasource-parquet = { workspace = true } datafusion-execution = { workspace = true } datafusion-expr = { workspace = true } datafusion-physical-expr = { workspace = true } From a5957787dfb6c80e6727ea07bc9e340a6dab35e4 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Mon, 27 Oct 2025 13:15:29 -0700 Subject: [PATCH 8/9] upgraded comfy-table --- Cargo.lock | 117 ++++++++++++++-------------------------- Cargo.toml | 2 +- rust/sedona/src/show.rs | 3 +- 3 files changed, 43 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e1c3282..d8f5513b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,8 +220,8 @@ dependencies = [ "serde_bytes", "serde_json", "snap", - "strum 0.27.2", - "strum_macros 0.27.2", + "strum", + "strum_macros", "thiserror 2.0.17", "uuid", "xz2", @@ -251,9 +251,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "56.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +checksum = "c26b57282a08ae92f727497805122fec964c6245cfa0e13f0e75452eaf3bc41f" dependencies = [ "arrow-arith", "arrow-array", @@ -314,9 +314,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "56.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +checksum = "eed61d9d73eda8df9e3014843def37af3050b5080a9acbe108f045a316d5a0be" dependencies = [ "arrow-array", "arrow-buffer", @@ -335,9 +335,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "56.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +checksum = "fa95b96ce0c06b4d33ac958370db8c0d31e88e54f9d6e08b0353d18374d9f991" dependencies = [ "arrow-array", "arrow-cast", @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "56.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +checksum = "26d747573390905905a2dc4c5a61a96163fe2750457f90a04ee2a88680758c79" dependencies = [ "arrow-array", "arrow-buffer", @@ -1277,14 +1277,12 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.1.2" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" dependencies = [ - "crossterm 0.27.0", - "crossterm 0.28.1", - "strum 0.26.3", - "strum_macros 0.26.4", + "crossterm", + "unicode-segmentation", "unicode-width", ] @@ -1444,28 +1442,18 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" -version = "0.27.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ "bitflags", "crossterm_winapi", - "libc", + "document-features", "parking_lot", + "rustix", "winapi", ] -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags", - "parking_lot", - "rustix 0.38.44", -] - [[package]] name = "crossterm_winapi" version = "0.9.1" @@ -2367,6 +2355,15 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dunce" version = "1.0.5" @@ -2466,7 +2463,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix 1.1.2", + "rustix", "windows-sys 0.59.0", ] @@ -3238,7 +3235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.0", ] [[package]] @@ -3527,12 +3524,6 @@ dependencies = [ "cc", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -3545,6 +3536,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.14" @@ -3906,9 +3903,9 @@ dependencies = [ [[package]] name = "parquet" -version = "56.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +checksum = "89b56b41d1bd36aae415e42f91cae70ee75cf6cba74416b14dce3e958d5990ec" dependencies = [ "ahash", "arrow-array", @@ -3925,7 +3922,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.0", + "hashbrown 0.15.5", "lz4_flex", "num", "num-bigint", @@ -4619,19 +4616,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.2" @@ -4641,7 +4625,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.11.0", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -5538,31 +5522,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.106", -] - [[package]] name = "strum_macros" version = "0.27.2" @@ -5649,7 +5614,7 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix 1.1.2", + "rustix", "windows-sys 0.61.2", ] @@ -6549,7 +6514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.2", + "rustix", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 548c46de..84b19f58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ async-trait = { version = "0.1.87" } bytes = "1.10" byteorder = "1" chrono = { version = "0.4.38", default-features = false } -comfy-table = { version = "7.0" } +comfy-table = { version = "7.1.4" } criterion = { version = "0.5", features = ["html_reports"] } datafusion = { version = "50.2.0", default-features = false } datafusion-catalog = { version = "50.2.0" } diff --git a/rust/sedona/src/show.rs b/rust/sedona/src/show.rs index 63089dfa..ef301d5c 100644 --- a/rust/sedona/src/show.rs +++ b/rust/sedona/src/show.rs @@ -252,8 +252,7 @@ impl<'a> DisplayTable<'a> { } } - // TODO: set_truncation_indicator method was removed in comfy_table 7.x - // table.set_truncation_indicator(self.truncation_indicator()); + table.set_truncation_indicator(self.truncation_indicator()); let mut columns = self .columns From 81309cdaf71d524d29d50d9ef0e7cdbe7dcdd7c4 Mon Sep 17 00:00:00 2001 From: zhangfengcdt Date: Mon, 27 Oct 2025 13:23:54 -0700 Subject: [PATCH 9/9] add todo for piping the new config object --- rust/sedona-geoparquet/src/writer.rs | 2 ++ rust/sedona-spatial-join/src/optimizer.rs | 2 ++ rust/sedona-testing/src/testers.rs | 2 ++ rust/sedona/src/show.rs | 2 ++ 4 files changed, 8 insertions(+) diff --git a/rust/sedona-geoparquet/src/writer.rs b/rust/sedona-geoparquet/src/writer.rs index bf137858..2b2ec9a4 100644 --- a/rust/sedona-geoparquet/src/writer.rs +++ b/rust/sedona-geoparquet/src/writer.rs @@ -227,6 +227,8 @@ fn project_bboxes( column.return_field(&input_schema)?.as_ref(), )?) { let bbox_field_name = bbox_column_name(f.name()); + // TODO: Pipe actual ConfigOptions from session instead of using defaults + // See: https://github.com/apache/sedona-db/issues/248 let expr = Arc::new(ScalarFunctionExpr::new( bbox_udf_name, bbox_udf.clone(), diff --git a/rust/sedona-spatial-join/src/optimizer.rs b/rust/sedona-spatial-join/src/optimizer.rs index 52571ce9..bd01821b 100644 --- a/rust/sedona-spatial-join/src/optimizer.rs +++ b/rust/sedona-spatial-join/src/optimizer.rs @@ -1162,6 +1162,8 @@ mod tests { ) -> Arc { let return_type = udf.return_type(&[]).unwrap(); let field = Arc::new(arrow::datatypes::Field::new("result", return_type, false)); + // TODO: Pipe actual ConfigOptions from session instead of using defaults + // See: https://github.com/apache/sedona-db/issues/248 Arc::new(ScalarFunctionExpr::new( udf.name(), Arc::clone(&udf), diff --git a/rust/sedona-testing/src/testers.rs b/rust/sedona-testing/src/testers.rs index a53cf855..b6580a0e 100644 --- a/rust/sedona-testing/src/testers.rs +++ b/rust/sedona-testing/src/testers.rs @@ -404,6 +404,8 @@ impl ScalarUdfTester { arg_fields: self.arg_fields(), number_rows, return_field: self.return_type()?.to_storage_field("", true)?.into(), + // TODO: Consider piping actual ConfigOptions for more realistic testing + // See: https://github.com/apache/sedona-db/issues/248 config_options: Arc::new(ConfigOptions::default()), }; diff --git a/rust/sedona/src/show.rs b/rust/sedona/src/show.rs index ef301d5c..4276a119 100644 --- a/rust/sedona/src/show.rs +++ b/rust/sedona/src/show.rs @@ -495,6 +495,8 @@ impl DisplayColumn { arg_fields, number_rows: array.len(), return_field, + // TODO: Pipe actual ConfigOptions from SedonaContext instead of using defaults + // See: https://github.com/apache/sedona-db/issues/248 config_options: Arc::new(ConfigOptions::default()), };