Skip to content

Commit

Permalink
fix: query result with hot tier (#908)
Browse files Browse the repository at this point in the history
the default view for data is latest (recent) first. This requires
to send the reverse sorted order of hot tier file list for 
datafusion to process. 

This fixes the duplicate query results issue when server is 
configured with hot tier.
  • Loading branch information
nikhilsinhaparseable authored Sep 1, 2024
1 parent 40bf437 commit a155b61
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion server/src/hottier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,9 @@ impl HotTierManager {
.get_stream_hot_tier_manifest_for_date(stream, &date)
.await?;
hot_tier_manifest.files.push(parquet_file.clone());
hot_tier_manifest
.files
.sort_by_key(|file| file.file_path.clone());
// write the manifest file to the hot tier directory
let manifest_path = self
.hot_tier_path
Expand Down Expand Up @@ -501,14 +504,18 @@ impl HotTierManager {
.iter()
.any(|manifest_file| manifest_file.file_path.eq(&file.file_path))
});
let remaining_files: Vec<File> = manifest_files
hot_tier_files.sort_unstable_by(|a, b| b.file_path.cmp(&a.file_path));

let mut remaining_files: Vec<File> = manifest_files
.into_iter()
.filter(|manifest_file| {
hot_tier_files
.iter()
.all(|file| !file.file_path.eq(&manifest_file.file_path))
})
.collect();
remaining_files.sort_unstable_by(|a, b| b.file_path.cmp(&a.file_path));

Ok((hot_tier_files, remaining_files))
}

Expand Down

0 comments on commit a155b61

Please sign in to comment.