Skip to content

Remove metadata from DynArray #7435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gatesn merged 2 commits into from
Apr 14, 2026
Merged

Remove metadata from DynArray #7435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c897b28
remove metadata from `DynArray`
connortsui20 Apr 14, 2026
26f7426
update file sizes
connortsui20 Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions encodings/alp/src/alp/plugin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ mod tests {

let array = alp_encoded.as_array();

let metadata = array.metadata(&SESSION)?.unwrap_or_default();
let metadata = SESSION.array_serialize(array)?.unwrap();
let children = array.children();
let buffers = array
.buffers()
Expand Down Expand Up @@ -182,7 +182,7 @@ mod tests {

let array = alp_encoded.as_array();

let metadata = array.metadata(&SESSION)?.unwrap_or_default();
let metadata = SESSION.array_serialize(array)?.unwrap();
let children = array.children();
let buffers = array
.buffers()
Expand Down Expand Up @@ -213,7 +213,7 @@ mod tests {
fn primitive_array_returns_error() {
let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array();

let metadata = array.metadata(&SESSION).unwrap().unwrap_or_default();
let metadata = SESSION.array_serialize(&array).unwrap().unwrap();
let children = array.children();
let buffers = array
.buffers()
Expand Down
3 changes: 1 addition & 2 deletions encodings/bytebool/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,6 @@ impl From<Vec<Option<bool>>> for ByteBoolData {
mod tests {
use vortex_array::ArrayContext;
use vortex_array::IntoArray;
use vortex_array::LEGACY_SESSION;
use vortex_array::assert_arrays_eq;
use vortex_array::serde::SerializeOptions;
use vortex_array::serde::SerializedArray;
Expand Down Expand Up @@ -407,7 +406,7 @@ mod tests {
let serialized = array
.clone()
.into_array()
.serialize(&ctx, &LEGACY_SESSION, &SerializeOptions::default())
.serialize(&ctx, &session, &SerializeOptions::default())
.unwrap();

let mut concat = ByteBufferMut::empty();
Expand Down
6 changes: 3 additions & 3 deletions encodings/fastlanes/src/bitpacking/plugin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ mod tests {

let array = bitpacked.as_array();

let metadata = array.metadata(&SESSION)?.unwrap_or_default();
let metadata = SESSION.array_serialize(array)?.unwrap();
let children = array.children();
let buffers = array
.buffers()
Expand Down Expand Up @@ -182,7 +182,7 @@ mod tests {

let array = bitpacked.as_array();

let metadata = array.metadata(&SESSION)?.unwrap_or_default();
let metadata = SESSION.array_serialize(array)?.unwrap();
let children = array.children();
let buffers = array
.buffers()
Expand Down Expand Up @@ -212,7 +212,7 @@ mod tests {
fn primitive_array_returns_error() -> VortexResult<()> {
let array = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();

let metadata = array.metadata(&SESSION)?.unwrap_or_default();
let metadata = SESSION.array_serialize(&array)?.unwrap();
let children = array.children();
let buffers = array
.buffers()
Expand Down
9 changes: 5 additions & 4 deletions encodings/parquet-variant/src/vtable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,13 +244,13 @@ mod tests {
use vortex_array::IntoArray;
use vortex_array::Precision;
use vortex_array::arrays::VarBinViewArray;
use vortex_array::arrays::Variant;
use vortex_array::arrays::VariantArray;
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::dtype::PType;
use vortex_array::serde::SerializeOptions;
use vortex_array::serde::SerializedArray;
use vortex_array::session::ArraySession;
use vortex_array::session::ArraySessionExt;
use vortex_array::validity::Validity;
use vortex_buffer::BitBuffer;
Expand All @@ -261,11 +261,14 @@ mod tests {

use crate::ParquetVariant;
use crate::array::ParquetVariantArrayExt;

fn roundtrip(array: ArrayRef) -> ArrayRef {
let dtype = array.dtype().clone();
let len = array.len();

let session = VortexSession::empty().with::<vortex_array::session::ArraySession>();
let session = VortexSession::empty().with::<ArraySession>();
session.arrays().register(ParquetVariant);

let ctx = ArrayContext::empty();
let serialized = array
.serialize(&ctx, &session, &SerializeOptions::default())
Expand All @@ -276,8 +279,6 @@ mod tests {
concat.extend_from_slice(buf.as_ref());
}
let concat = concat.freeze();
session.arrays().register(ParquetVariant);
session.arrays().register(Variant);
Copy link
Copy Markdown
Contributor

@joseph-isaacs joseph-isaacs Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

?

Copy link
Copy Markdown
Contributor

@gatesn gatesn Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ParquetVariant is registered above.
Variant is the canonical array, it's now registrered in default ArraySession


let parts = SerializedArray::try_from(concat).unwrap();
parts
Expand Down
5 changes: 3 additions & 2 deletions encodings/pco/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ mod array;
mod compute;
mod rules;
mod slice;
#[cfg(test)]
mod test;

pub use array::*;

Expand Down Expand Up @@ -35,3 +33,6 @@ pub struct PcoMetadata {
#[prost(message, repeated, tag = "2")]
pub chunks: Vec<PcoChunkInfo>,
}

#[cfg(test)]
mod tests;
2 changes: 1 addition & 1 deletion encodings/pco/src/test.rs → encodings/pco/src/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ fn test_serde() -> VortexResult<()> {
let bytes = pco
.serialize(
&context,
&LEGACY_SESSION,
&SESSION,
&SerializeOptions {
offset: 0,
include_padding: true,
Expand Down
8 changes: 6 additions & 2 deletions encodings/runend/src/arrow.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ mod tests {
use vortex_array::search_sorted::SearchSorted;
use vortex_array::search_sorted::SearchSortedSide;
use vortex_array::session::ArraySession;
use vortex_array::session::ArraySessionExt;
use vortex_array::validity::Validity;
use vortex_buffer::Buffer;
use vortex_buffer::buffer;
Expand All @@ -99,8 +100,11 @@ mod tests {
use crate::RunEnd;
use crate::ops::find_slice_end_index;

static SESSION: LazyLock<VortexSession> =
LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty().with::<ArraySession>();
session.arrays().register(RunEnd);
session
});

fn decode_run_array<R: RunEndIndexType>(
array: &RunArray<R>,
Expand Down
6 changes: 3 additions & 3 deletions encodings/zstd/src/zstd_buffers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ impl ZstdBuffers {
session: &VortexSession,
) -> VortexResult<ZstdBuffersArray> {
let encoding_id = array.encoding_id();
let metadata = array
.metadata(session)?
.ok_or_else(|| vortex_err!("Array does not support serialization"))?;
let metadata = session
.array_serialize(array)?
.ok_or_else(|| vortex_err!("[ZstdBuffers]: Array does not support serialization"))?;
let buffer_handles = array.buffer_handles();
let children = array.children();

Expand Down
2 changes: 0 additions & 2 deletions vortex-array/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22024,8 +22024,6 @@ pub fn vortex_array::ArrayRef::is_valid(&self, index: usize) -> vortex_error::Vo

pub fn vortex_array::ArrayRef::len(&self) -> usize

pub fn vortex_array::ArrayRef::metadata(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>

pub fn vortex_array::ArrayRef::metadata_fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result

pub fn vortex_array::ArrayRef::named_buffers(&self) -> alloc::vec::Vec<(alloc::string::String, vortex_array::buffer::BufferHandle)>
Expand Down
6 changes: 0 additions & 6 deletions vortex-array/src/array/erased.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ use vortex_error::vortex_ensure;
use vortex_error::vortex_err;
use vortex_error::vortex_panic;
use vortex_mask::Mask;
use vortex_session::VortexSession;

use crate::AnyCanonical;
use crate::Array;
Expand Down Expand Up @@ -561,11 +560,6 @@ impl ArrayRef {
self.0.slot_name(self, idx)
}

/// Returns the serialized metadata of the array.
pub fn metadata(&self, session: &VortexSession) -> VortexResult<Option<Vec<u8>>> {
self.0.metadata(self, session)
}

/// Formats a human-readable metadata description.
pub fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.metadata_fmt(f)
Expand Down
10 changes: 0 additions & 10 deletions vortex-array/src/array/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use vortex_error::VortexResult;
use vortex_error::vortex_ensure;
use vortex_error::vortex_err;
use vortex_error::vortex_panic;
use vortex_session::VortexSession;
use vortex_session::registry::Id;

use crate::ExecutionCtx;
Expand Down Expand Up @@ -132,10 +131,6 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug {
/// Returns the name of the slot at the given index.
fn slot_name(&self, this: &ArrayRef, idx: usize) -> String;

/// Returns the serialized metadata of the array, or `None` if the array does not
/// support serialization.
fn metadata(&self, this: &ArrayRef, session: &VortexSession) -> VortexResult<Option<Vec<u8>>>;

/// Formats a human-readable metadata description.
fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result;

Expand Down Expand Up @@ -341,11 +336,6 @@ impl<V: VTable> DynArray for ArrayInner<V> {
V::slot_name(view, idx)
}

fn metadata(&self, this: &ArrayRef, session: &VortexSession) -> VortexResult<Option<Vec<u8>>> {
let view = unsafe { ArrayView::new_unchecked(this, &self.data) };
V::serialize(view, session)
}

fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.data, f)
}
Expand Down
6 changes: 4 additions & 2 deletions vortex-array/src/arrow/executor/run_end.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use crate::IntoArray;
use crate::arrays::Constant;
use crate::arrays::ConstantArray;
use crate::arrow::ArrowArrayExecutor;
use crate::session::ArraySessionExt;

/// The encoding ID used by `vortex-runend`. We match on this string to avoid a crate dependency.
const VORTEX_RUNEND_ID: &str = "vortex.runend";
Expand Down Expand Up @@ -79,8 +80,9 @@ fn run_end_to_arrow(
ctx: &mut ExecutionCtx,
) -> VortexResult<ArrowArrayRef> {
let length = array.len();
let metadata_bytes = array
.metadata(ctx.session())?
let metadata_bytes = ctx
.session()
.array_serialize(&array)?
.ok_or_else(|| vortex_err!("RunEndArray missing metadata"))?;
let metadata = RunEndMetadata::decode(&*metadata_bytes)
.map_err(|e| vortex_err!("Failed to decode RunEndMetadata: {e}"))?;
Expand Down
112 changes: 2 additions & 110 deletions vortex-array/src/serde.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,16 +170,6 @@ impl<'a> ArrayNodeFlatBuffer<'a> {
session: &'a VortexSession,
array: &'a ArrayRef,
) -> VortexResult<Self> {
// Depth-first traversal of the array to ensure it supports serialization.
// FIXME(ngates): this serializes the metadata and throws it away!
for child in array.depth_first_traversal() {
if child.metadata(session)?.is_none() {
vortex_bail!(
"Array {} does not support serialization",
child.encoding_id()
);
}
}
let n_buffers_recursive = array.nbuffers_recursive();
if n_buffers_recursive > u16::MAX as usize {
vortex_bail!(
Expand Down Expand Up @@ -210,13 +200,13 @@ impl<'a> ArrayNodeFlatBuffer<'a> {
)
})?;

let metadata = self.array.metadata(self.session)?.ok_or_else(|| {
let metadata_bytes = self.session.array_serialize(self.array)?.ok_or_else(|| {
vortex_err!(
"Array {} does not support serialization",
self.array.encoding_id()
)
})?;
let metadata = Some(fbb.create_vector(metadata.as_slice()));
let metadata = Some(fbb.create_vector(metadata_bytes.as_slice()));

// Assign buffer indices for all child arrays.
let nbuffers = u16::try_from(self.array.nbuffers())
Expand Down Expand Up @@ -701,101 +691,3 @@ impl TryFrom<BufferHandle> for SerializedArray {
Self::try_from(value.try_to_host_sync()?)
}
}

#[cfg(test)]
mod tests {
use std::sync::LazyLock;

use flatbuffers::FlatBufferBuilder;
use vortex_session::VortexSession;
use vortex_session::registry::ReadContext;

use super::SerializeOptions;
use super::SerializedArray;
use crate::ArrayContext;
use crate::array::ArrayId;
use crate::dtype::DType;
use crate::dtype::Nullability;
use crate::flatbuffers as fba;
use crate::session::ArraySession;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);

#[test]
fn unknown_array_encoding_allow_unknown() {
Comment on lines -724 to -725
Copy link
Copy Markdown
Contributor Author

@connortsui20 connortsui20 Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this test didnt actually test anything, and was broken

let mut fbb = FlatBufferBuilder::new();

let child_metadata = fbb.create_vector(&[9u8]);
let child = fba::ArrayNode::create(
&mut fbb,
&fba::ArrayNodeArgs {
encoding: 1,
metadata: Some(child_metadata),
children: None,
buffers: None,
stats: None,
},
);

let children = fbb.create_vector(&[child]);
let metadata = fbb.create_vector(&[1u8, 2, 3]);
let root = fba::ArrayNode::create(
&mut fbb,
&fba::ArrayNodeArgs {
encoding: 0,
metadata: Some(metadata),
children: Some(children),
buffers: None,
stats: None,
},
);
let array = fba::Array::create(
&mut fbb,
&fba::ArrayArgs {
root: Some(root),
buffers: None,
},
);
fbb.finish_minimal(array);
let (buf, start) = fbb.collapse();
let tree = vortex_buffer::ByteBuffer::from(buf).slice(start..);

let ser = SerializedArray::from_array_tree(tree).unwrap();
let ctx = ReadContext::new([
ArrayId::new("vortex.test.foreign_array"),
ArrayId::new("vortex.test.foreign_child"),
]);
let session = VortexSession::empty()
.with::<ArraySession>()
.allow_unknown();

let decoded = ser
.decode(&DType::Variant(Nullability::Nullable), 5, &ctx, &session)
.unwrap();
assert_eq!(decoded.encoding_id().as_ref(), "vortex.test.foreign_array");
assert_eq!(decoded.nchildren(), 1);
assert_eq!(
decoded.nth_child(0).unwrap().encoding_id().as_ref(),
"vortex.test.foreign_child"
);
assert_eq!(decoded.metadata(&SESSION).unwrap().unwrap(), vec![1, 2, 3]);
assert_eq!(
decoded
.nth_child(0)
.unwrap()
.metadata(&SESSION)
.unwrap()
.unwrap(),
vec![9]
);

let serialized = decoded
.serialize(
&ArrayContext::default(),
&SESSION,
&SerializeOptions::default(),
)
.unwrap();
assert!(!serialized.is_empty());
}
}
Loading
Loading