Skip to content

Commit a155b61

Browse files
fix: query result with hot tier (#908)
the default view for data is latest (recent) first. This requires to send the reverse sorted order of hot tier file list for datafusion to process. This fixes the duplicate query results issue when server is configured with hot tier.
1 parent 40bf437 commit a155b61

File tree

1 file changed

+8
-1
lines changed

1 file changed

+8
-1
lines changed

server/src/hottier.rs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,9 @@ impl HotTierManager {
409409
.get_stream_hot_tier_manifest_for_date(stream, &date)
410410
.await?;
411411
hot_tier_manifest.files.push(parquet_file.clone());
412+
hot_tier_manifest
413+
.files
414+
.sort_by_key(|file| file.file_path.clone());
412415
// write the manifest file to the hot tier directory
413416
let manifest_path = self
414417
.hot_tier_path
@@ -501,14 +504,18 @@ impl HotTierManager {
501504
.iter()
502505
.any(|manifest_file| manifest_file.file_path.eq(&file.file_path))
503506
});
504-
let remaining_files: Vec<File> = manifest_files
507+
hot_tier_files.sort_unstable_by(|a, b| b.file_path.cmp(&a.file_path));
508+
509+
let mut remaining_files: Vec<File> = manifest_files
505510
.into_iter()
506511
.filter(|manifest_file| {
507512
hot_tier_files
508513
.iter()
509514
.all(|file| !file.file_path.eq(&manifest_file.file_path))
510515
})
511516
.collect();
517+
remaining_files.sort_unstable_by(|a, b| b.file_path.cmp(&a.file_path));
518+
512519
Ok((hot_tier_files, remaining_files))
513520
}
514521

0 commit comments

Comments
 (0)