feature to support OpenTelemetry logs to be ingested on Parseable

fixes: #312
parseablehq · Feb 13, 2024 · af2e2d9 · af2e2d9
1 parent fb3fd21
commit af2e2d9
Show file tree

Hide file tree

Showing 13 changed files with 1,234 additions and 9 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/server/Cargo.toml b/server/Cargo.toml
@@ -103,6 +103,8 @@ http-auth-basic = "0.3.3"
 serde_repr = "0.1.17"
 hashlru = { version = "0.11.0", features = ["serde"] }
 path-clean = "1.0.1"
+prost = "0.12.3"
+
 
 [build-dependencies]
 cargo_toml = "0.15"
@@ -112,6 +114,7 @@ ureq = "2.6"
 vergen = { version = "8.1", features = ["build", "git", "cargo", "gitcl"] }
 zip = { version = "0.6", default_features = false, features = ["deflate"] }
 url = "2.4.0"
+prost-build = "0.12.3"
 
 [dev-dependencies]
 maplit = "1.0"

diff --git a/server/src/handlers/http.rs b/server/src/handlers/http.rs
@@ -45,6 +45,7 @@ mod llm;
 mod logstream;
 mod middleware;
 mod oidc;
+mod otel;
 mod query;
 mod rbac;
 mod role;

diff --git a/server/src/handlers/http/ingest.rs b/server/src/handlers/http/ingest.rs
@@ -34,8 +34,8 @@ use crate::handlers::{
 use crate::metadata::STREAM_INFO;
 use crate::utils::header_parsing::{collect_labelled_headers, ParseHeaderError};
 
-use super::kinesis;
 use super::logstream::error::CreateStreamError;
+use super::{kinesis, otel};
 
 // Handler for POST /api/v1/ingest
 // ingests events by extracting stream name from header
@@ -67,7 +67,7 @@ async fn flatten_and_push_logs(
         let log_source: String = log_source.to_str().unwrap().to_owned();
         match log_source.as_str() {
             LOG_SOURCE_KINESIS => json = kinesis::flatten_kinesis_logs(&body),
-            LOG_SOURCE_OTEL => {}
+            LOG_SOURCE_OTEL => json = otel::flatten_otel_logs(&body),
             _ => {
                 log::warn!("Unknown log source: {}", log_source);
                 push_logs(stream_name.to_string(), req.clone(), body).await?;