Use approximate cardinality to decide whether to use dict compression

Cargo.toml

@@ -112,6 +112,7 @@ bit-vec = "0.9.0"
 bitvec = "1.0.1"
 bytes = "1.11.1"
 bzip2 = "0.6.0"
+cardinality-estimator = "1.0.3"
 cargo_metadata = "0.23.1"
 cbindgen = "0.29.0"
 cc = "1.2"

encodings/fastlanes/src/rle/array/mod.rs

@@ -473,7 +473,7 @@ mod tests {
             .indices()
             .clone()
             .execute::<PrimitiveArray>(&mut ctx)?
-            .narrow()?;
+            .narrow(&mut ctx)?;
         let re_encoded = RLEData::encode(indices_prim.as_view(), &mut ctx)?;
 
         // Reconstruct the outer RLE with re-encoded indices.

encodings/runend/src/compress.rs

@@ -84,7 +84,9 @@ pub fn runend_encode(
         }
     };
 
-    let ends = ends.narrow().vortex_expect("Ends must succeed downcasting");
+    let ends = ends
+        .narrow(ctx)
+        .vortex_expect("Ends must succeed downcasting");
 
     ends.statistics()
         .set(Stat::IsStrictSorted, Precision::Exact(true.into()));

vortex-array/benches/dict_compare.rs

@@ -4,6 +4,7 @@
 #![expect(clippy::unwrap_used)]
 
 use std::str::from_utf8;
+use std::sync::LazyLock;
 
 use vortex_array::Canonical;
 use vortex_array::IntoArray;
@@ -21,6 +22,7 @@ use vortex_array::expr::eq;
 use vortex_array::expr::lit;
 use vortex_array::expr::root;
 use vortex_array::scalar_fn::fns::operators::Operator;
+use vortex_array::session::ArraySession;
 use vortex_session::VortexSession;
 
 fn main() {
@@ -45,15 +47,21 @@ const LENGTH_AND_UNIQUE_VALUES: &[(usize, usize)] = &[
     (100_000, 2048),
 ];
 
+static SESSION: LazyLock<VortexSession> =
+    LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
+
 #[divan::bench(args = LENGTH_AND_UNIQUE_VALUES)]
 fn bench_compare_primitive(bencher: divan::Bencher, (len, uniqueness): (usize, usize)) {
     let primitive_arr = gen_primitive_for_dict::<i32>(len, uniqueness);
-    let dict = dict_encode(&primitive_arr.clone().into_array()).unwrap();
+    let dict = dict_encode(
+        &primitive_arr.clone().into_array(),
+        &mut SESSION.create_execution_ctx(),
+    )
+    .unwrap();
     let value = primitive_arr.as_slice::<i32>()[0];
-    let session = VortexSession::empty();
 
     bencher
-        .with_inputs(|| (&dict, session.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| {
             dict.clone()
                 .into_array()
@@ -67,13 +75,16 @@ fn bench_compare_primitive(bencher: divan::Bencher, (len, uniqueness): (usize, u
 #[divan::bench(args = LENGTH_AND_UNIQUE_VALUES)]
 fn bench_compare_varbin(bencher: divan::Bencher, (len, uniqueness): (usize, usize)) {
     let varbin_arr = VarBinArray::from(gen_varbin_words(len, uniqueness));
-    let dict = dict_encode(&varbin_arr.clone().into_array()).unwrap();
+    let dict = dict_encode(
+        &varbin_arr.clone().into_array(),
+        &mut SESSION.create_execution_ctx(),
+    )
+    .unwrap();
     let bytes = varbin_arr.with_iterator(|i| i.next().unwrap().unwrap().to_vec());
     let value = from_utf8(bytes.as_slice()).unwrap();
-    let session = VortexSession::empty();
 
     bencher
-        .with_inputs(|| (&dict, session.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| {
             dict.clone()
                 .into_array()
@@ -87,13 +98,16 @@ fn bench_compare_varbin(bencher: divan::Bencher, (len, uniqueness): (usize, usiz
 #[divan::bench(args = LENGTH_AND_UNIQUE_VALUES)]
 fn bench_compare_varbinview(bencher: divan::Bencher, (len, uniqueness): (usize, usize)) {
     let varbinview_arr = VarBinViewArray::from_iter_str(gen_varbin_words(len, uniqueness));
-    let dict = dict_encode(&varbinview_arr.clone().into_array()).unwrap();
+    let dict = dict_encode(
+        &varbinview_arr.clone().into_array(),
+        &mut SESSION.create_execution_ctx(),
+    )
+    .unwrap();
     let bytes = varbinview_arr.with_iterator(|i| i.next().unwrap().unwrap().to_vec());
     let value = from_utf8(bytes.as_slice()).unwrap();
-    let session = VortexSession::empty();
 
     bencher
-        .with_inputs(|| (&dict, session.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| {
             dict.clone()
                 .into_array()
@@ -122,13 +136,16 @@ fn bench_compare_sliced_dict_primitive(
     (codes_len, values_len): (usize, usize),
 ) {
     let primitive_arr = gen_primitive_for_dict::<i32>(codes_len.max(values_len), values_len);
-    let dict = dict_encode(&primitive_arr.clone().into_array()).unwrap();
+    let dict = dict_encode(
+        &primitive_arr.clone().into_array(),
+        &mut SESSION.create_execution_ctx(),
+    )
+    .unwrap();
     let dict = dict.into_array().slice(0..codes_len).unwrap();
     let value = primitive_arr.as_slice::<i32>()[0];
-    let session = VortexSession::empty();
 
     bencher
-        .with_inputs(|| (&dict, session.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| {
             dict.clone()
                 .apply(&eq(root(), lit(value)))
@@ -144,14 +161,17 @@ fn bench_compare_sliced_dict_varbinview(
     (codes_len, values_len): (usize, usize),
 ) {
     let varbin_arr = VarBinArray::from(gen_varbin_words(codes_len.max(values_len), values_len));
-    let dict = dict_encode(&varbin_arr.clone().into_array()).unwrap();
+    let dict = dict_encode(
+        &varbin_arr.clone().into_array(),
+        &mut SESSION.create_execution_ctx(),
+    )
+    .unwrap();
     let dict = dict.into_array().slice(0..codes_len).unwrap();
     let bytes = varbin_arr.with_iterator(|i| i.next().unwrap().unwrap().to_vec());
     let value = from_utf8(bytes.as_slice()).unwrap();
-    let session = VortexSession::empty();
 
     bencher
-        .with_inputs(|| (&dict, session.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| {
             dict.clone()
                 .apply(&eq(root(), lit(value)))

vortex-array/benches/dict_compress.rs

@@ -3,19 +3,22 @@
 
 #![expect(clippy::unwrap_used)]
 
+use std::sync::LazyLock;
+
 use divan::Bencher;
 use rand::distr::Distribution;
 use rand::distr::StandardUniform;
 use vortex_array::Canonical;
 use vortex_array::IntoArray;
-use vortex_array::LEGACY_SESSION;
 use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::VarBinArray;
 use vortex_array::arrays::VarBinViewArray;
 use vortex_array::arrays::dict_test::gen_primitive_for_dict;
 use vortex_array::arrays::dict_test::gen_varbin_words;
 use vortex_array::builders::dict::dict_encode;
 use vortex_array::dtype::NativePType;
+use vortex_array::session::ArraySession;
+use vortex_session::VortexSession;
 
 fn main() {
     divan::main();
@@ -35,35 +38,39 @@ const BENCH_ARGS: &[(usize, usize)] = &[
     (10_000, 512),
 ];
 
+static SESSION: LazyLock<VortexSession> =
+    LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
+
 #[divan::bench(types = [u8, f32, i64], args = BENCH_ARGS)]
 fn encode_primitives<T>(bencher: Bencher, (len, unique_values): (usize, usize))
 where
     T: NativePType,
     StandardUniform: Distribution<T>,
 {
-    let primitive_arr = gen_primitive_for_dict::<T>(len, unique_values);
+    let primitive_arr = gen_primitive_for_dict::<T>(len, unique_values).into_array();
 
     bencher
-        .with_inputs(|| &primitive_arr)
-        .bench_refs(|arr| dict_encode(&arr.clone().into_array()));
+        .with_inputs(|| (&primitive_arr, SESSION.create_execution_ctx()))
+        .bench_refs(|(arr, ctx)| dict_encode(arr, ctx));
 }
 
 #[divan::bench(args = BENCH_ARGS)]
 fn encode_varbin(bencher: Bencher, (len, unique_values): (usize, usize)) {
-    let varbin_arr = VarBinArray::from(gen_varbin_words(len, unique_values));
+    let varbin_arr = VarBinArray::from(gen_varbin_words(len, unique_values)).into_array();
 
     bencher
-        .with_inputs(|| &varbin_arr)
-        .bench_refs(|arr| dict_encode(&arr.clone().into_array()));
+        .with_inputs(|| (&varbin_arr, SESSION.create_execution_ctx()))
+        .bench_refs(|(arr, ctx)| dict_encode(arr, ctx));
 }
 
 #[divan::bench(args = BENCH_ARGS)]
 fn encode_varbinview(bencher: Bencher, (len, unique_values): (usize, usize)) {
-    let varbinview_arr = VarBinViewArray::from_iter_str(gen_varbin_words(len, unique_values));
+    let varbinview_arr =
+        VarBinViewArray::from_iter_str(gen_varbin_words(len, unique_values)).into_array();
 
     bencher
-        .with_inputs(|| &varbinview_arr)
-        .bench_refs(|arr| dict_encode(&arr.clone().into_array()));
+        .with_inputs(|| (&varbinview_arr, SESSION.create_execution_ctx()))
+        .bench_refs(|(arr, ctx)| dict_encode(arr, ctx));
 }
 
 #[divan::bench(types = [u8, f32, i64], args = BENCH_ARGS)]
@@ -72,34 +79,37 @@ where
     T: NativePType,
     StandardUniform: Distribution<T>,
 {
-    let primitive_arr = gen_primitive_for_dict::<T>(len, unique_values);
-    let dict = dict_encode(&primitive_arr.into_array())
+    let primitive_arr = gen_primitive_for_dict::<T>(len, unique_values).into_array();
+    let dict = dict_encode(&primitive_arr, &mut SESSION.create_execution_ctx())
         .unwrap()
         .into_array();
 
     bencher
-        .with_inputs(|| (&dict, LEGACY_SESSION.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| (**dict).clone().execute::<Canonical>(ctx));
 }
 
 #[divan::bench(args = BENCH_ARGS)]
 fn decode_varbin(bencher: Bencher, (len, unique_values): (usize, usize)) {
-    let varbin_arr = VarBinArray::from(gen_varbin_words(len, unique_values));
-    let dict = dict_encode(&varbin_arr.into_array()).unwrap().into_array();
+    let varbin_arr = VarBinArray::from(gen_varbin_words(len, unique_values)).into_array();
+    let dict = dict_encode(&varbin_arr, &mut SESSION.create_execution_ctx())
+        .unwrap()
+        .into_array();
 
     bencher
-        .with_inputs(|| (&dict, LEGACY_SESSION.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| (**dict).clone().execute::<Canonical>(ctx));
 }
 
 #[divan::bench(args = BENCH_ARGS)]
 fn decode_varbinview(bencher: Bencher, (len, unique_values): (usize, usize)) {
-    let varbinview_arr = VarBinViewArray::from_iter_str(gen_varbin_words(len, unique_values));
-    let dict = dict_encode(&varbinview_arr.into_array())
+    let varbinview_arr =
+        VarBinViewArray::from_iter_str(gen_varbin_words(len, unique_values)).into_array();
+    let dict = dict_encode(&varbinview_arr, &mut SESSION.create_execution_ctx())
         .unwrap()
         .into_array();
 
     bencher
-        .with_inputs(|| (&dict, LEGACY_SESSION.create_execution_ctx()))
+        .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| (**dict).clone().execute::<Canonical>(ctx));
 }

vortex-array/public-api.lock

@@ -4038,7 +4038,7 @@ pub trait vortex_array::arrays::primitive::PrimitiveArrayExt: vortex_array::Type
 
 pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle
 
-pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::narrow(&self) -> vortex_error::VortexResult<vortex_array::arrays::PrimitiveArray>
+pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::narrow(&self, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::arrays::PrimitiveArray>
 
 pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::nullability(&self) -> vortex_array::dtype::Nullability
 
@@ -4054,7 +4054,7 @@ impl<T: vortex_array::TypedArrayRef<vortex_array::arrays::Primitive>> vortex_arr
 
 pub fn T::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle
 
-pub fn T::narrow(&self) -> vortex_error::VortexResult<vortex_array::arrays::PrimitiveArray>
+pub fn T::narrow(&self, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::arrays::PrimitiveArray>
 
 pub fn T::nullability(&self) -> vortex_array::dtype::Nullability
 
@@ -7440,13 +7440,13 @@ pub trait vortex_array::builders::dict::DictEncoder: core::marker::Send
 
 pub fn vortex_array::builders::dict::DictEncoder::codes_ptype(&self) -> vortex_array::dtype::PType
 
-pub fn vortex_array::builders::dict::DictEncoder::encode(&mut self, &vortex_array::ArrayRef) -> vortex_array::ArrayRef
+pub fn vortex_array::builders::dict::DictEncoder::encode(&mut self, &vortex_array::ArrayRef, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::arrays::PrimitiveArray>
 
 pub fn vortex_array::builders::dict::DictEncoder::reset(&mut self) -> vortex_array::ArrayRef
 
-pub fn vortex_array::builders::dict::dict_encode(&vortex_array::ArrayRef) -> vortex_error::VortexResult<vortex_array::arrays::dict::DictArray>
+pub fn vortex_array::builders::dict::dict_encode(&vortex_array::ArrayRef, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::arrays::dict::DictArray>
 
-pub fn vortex_array::builders::dict::dict_encode_with_constraints(&vortex_array::ArrayRef, &vortex_array::builders::dict::DictConstraints) -> vortex_error::VortexResult<vortex_array::arrays::dict::DictArray>
+pub fn vortex_array::builders::dict::dict_encode_with_constraints(&vortex_array::ArrayRef, &vortex_array::builders::dict::DictConstraints, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::arrays::dict::DictArray>
 
 pub fn vortex_array::builders::dict::dict_encoder(&vortex_array::ArrayRef, &vortex_array::builders::dict::DictConstraints) -> alloc::boxed::Box<dyn vortex_array::builders::dict::DictEncoder>