Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: Stream Sync on Ingest Server #708

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion server/src/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ impl StreamInfo {

for stream in storage.list_streams().await? {
let alerts = storage.get_alerts(&stream.name).await?;
let schema = storage.get_schema(&stream.name).await?;
let schema = storage.get_schema_for_the_first_time(&stream.name).await?;
let meta = storage.get_stream_metadata(&stream.name).await?;

let schema = update_schema_from_staging(&stream.name, schema);
Expand Down
25 changes: 25 additions & 0 deletions server/src/storage/object_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,15 @@ pub trait ObjectStorage: Sync + 'static {
.await
}

async fn get_schema_for_the_first_time(
&self,
stream_name: &str,
) -> Result<Schema, ObjectStorageError> {
let schema_path = RelativePathBuf::from_iter([stream_name, SCHEMA_FILE_NAME]);
let schema_map = self.get_object(&schema_path).await?;
Ok(serde_json::from_slice(&schema_map)?)
}

async fn get_schema(&self, stream_name: &str) -> Result<Schema, ObjectStorageError> {
let schema_map = self.get_object(&schema_path(stream_name)).await?;
Ok(serde_json::from_slice(&schema_map)?)
Expand Down Expand Up @@ -231,6 +240,22 @@ pub trait ObjectStorage: Sync + 'static {
self.put_object(&path, to_bytes(manifest)).await
}

/// for future use
async fn get_stats_for_first_time(
&self,
stream_name: &str,
) -> Result<Stats, ObjectStorageError> {
let path = RelativePathBuf::from_iter([stream_name, STREAM_METADATA_FILE_NAME]);
let stream_metadata = self.get_object(&path).await?;
let stream_metadata: Value =
serde_json::from_slice(&stream_metadata).expect("parseable config is valid json");
let stats = &stream_metadata["stats"];

let stats = serde_json::from_value(stats.clone()).unwrap_or_default();

Ok(stats)
}

async fn get_stats(&self, stream_name: &str) -> Result<Stats, ObjectStorageError> {
let stream_metadata = self.get_object(&stream_json_path(stream_name)).await?;
let stream_metadata: Value =
Expand Down
Loading