-
Notifications
You must be signed in to change notification settings - Fork 577
fix(cache): drop schema-id from the manifest-list cache key #3235
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,7 +22,7 @@ use crate::encryption::EncryptionManager; | |
| use crate::io::FileIO; | ||
| use crate::spec::{ | ||
| FormatVersion, Manifest, ManifestFile, ManifestList, ManifestListReader, ManifestReader, | ||
| SchemaId, SnapshotRef, TableMetadataRef, | ||
| SnapshotRef, TableMetadataRef, | ||
| }; | ||
| use crate::{Error, ErrorKind, Result}; | ||
|
|
||
|
|
@@ -36,7 +36,7 @@ pub(crate) enum CachedItem { | |
|
|
||
| #[derive(Clone, Debug, Hash, Eq, PartialEq)] | ||
| pub(crate) enum CachedObjectKey { | ||
| ManifestList((String, FormatVersion, SchemaId)), | ||
| ManifestList((String, FormatVersion)), | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not for this PR, but now that Given xanderbailey's note that Java keys on location alone, I'd either drop it in a follow-up or leave a one-liner on why it stays, so the next reader isn't left guessing. wdyt?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think If we ever move Rust impelementation to a byte-cache like Java's,
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I also think we should we should remove FormatVersion, a manifest list could be uniquely identified by a path. IIRC they were initially added for parsing manifest list.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I looked into this and I think you are right.
So if I'll drop it in a follow-up so this PR stays scoped to the schema-id panic fix.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. #3259 as followup |
||
| // The manifest-level `first_row_id` is part of the key because the parsed | ||
| // manifest inherits it onto its entries: the same physical manifest can be | ||
| // referenced with different offsets across snapshots and branches, so it | ||
|
|
@@ -159,7 +159,6 @@ impl ObjectCache { | |
| let key = CachedObjectKey::ManifestList(( | ||
| snapshot.manifest_list().to_string(), | ||
| table_metadata.format_version, | ||
| snapshot.schema_id().unwrap(), | ||
| )); | ||
| let cache_entry = self | ||
| .cache | ||
|
|
@@ -214,6 +213,7 @@ impl ObjectCache { | |
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| use std::collections::HashMap; | ||
| use std::fs; | ||
|
|
||
| use minijinja::value::Value; | ||
|
|
@@ -226,7 +226,8 @@ mod tests { | |
| use crate::io::{FileIO, OutputFile}; | ||
| use crate::spec::{ | ||
| DataContentType, DataFileBuilder, DataFileFormat, Literal, ManifestEntry, | ||
| ManifestListWriter, ManifestStatus, ManifestWriterBuilder, Struct, TableMetadata, | ||
| ManifestListWriter, ManifestStatus, ManifestWriterBuilder, Operation, Snapshot, Struct, | ||
| Summary, TableMetadata, | ||
| }; | ||
| use crate::table::Table; | ||
| use crate::test_utils::test_runtime; | ||
|
|
@@ -444,6 +445,51 @@ mod tests { | |
| ); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn test_get_manifest_list_with_no_schema_id() { | ||
| let mut fixture = TableTestFixture::new(); | ||
| fixture.setup_manifest_files().await; | ||
|
|
||
| let current_snapshot = fixture.table.metadata().current_snapshot().unwrap(); | ||
|
|
||
| // The spec marks `schema-id` optional in every version (v1-v3), so a | ||
| // snapshot may omit it; fetching its manifest list must not depend on the | ||
| // schema-id being present. | ||
| let snapshot_without_schema_id: SnapshotRef = Snapshot::builder() | ||
| .with_snapshot_id(current_snapshot.snapshot_id()) | ||
| .with_sequence_number(current_snapshot.sequence_number()) | ||
| .with_timestamp_ms(current_snapshot.timestamp_ms()) | ||
| .with_manifest_list(current_snapshot.manifest_list()) | ||
| .with_summary(Summary { | ||
| operation: Operation::Append, | ||
| additional_properties: HashMap::new(), | ||
| }) | ||
| .build() | ||
| .into(); | ||
| assert!(snapshot_without_schema_id.schema_id().is_none()); | ||
| assert!(current_snapshot.schema_id().is_some()); | ||
|
|
||
| let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); | ||
|
|
||
| // Cold miss: the schema-id-less snapshot populates the cache. | ||
| let inserted = object_cache | ||
| .get_manifest_list(&snapshot_without_schema_id, &fixture.table.metadata_ref()) | ||
| .await | ||
| .unwrap(); | ||
| assert_eq!(inserted.entries().len(), 1); | ||
|
|
||
| // Warm hit: the original snapshot carries a schema-id but points at the same | ||
| // manifest-list location, so it returns the same cached entry. | ||
| let cached = object_cache | ||
| .get_manifest_list(current_snapshot, &fixture.table.metadata_ref()) | ||
| .await | ||
| .unwrap(); | ||
| assert!( | ||
| Arc::ptr_eq(&inserted, &cached), | ||
| "snapshots with and without schema-id at one location must share a cache entry" | ||
| ); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn test_get_manifest_keys_on_first_row_id() { | ||
| use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Type}; | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Another alternative is to continue to use
Option<SchemaId>- but it doesn't seem necessary