From d8565bd36d4c0d4510d79854025dee00ee353de0 Mon Sep 17 00:00:00 2001 From: mrferris Date: Wed, 8 Nov 2023 23:23:09 -0500 Subject: [PATCH] fix: creates audits as pending to prevent duplicate audits --- entity/src/content_audit.rs | 43 ++++++--- glados-audit/src/lib.rs | 101 ++++++++++++-------- glados-web/templates/audit_table.html | 5 +- glados-web/templates/content_dashboard.html | 20 ++-- 4 files changed, 106 insertions(+), 63 deletions(-) diff --git a/entity/src/content_audit.rs b/entity/src/content_audit.rs index f9016b6f..0d06ae3d 100644 --- a/entity/src/content_audit.rs +++ b/entity/src/content_audit.rs @@ -1,6 +1,7 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7 use crate::content; use crate::utils; +use anyhow::anyhow; use anyhow::{bail, Result}; use chrono::DateTime; use chrono::Utc; @@ -8,11 +9,13 @@ use clap::ValueEnum; use ethportal_api::OverlayContentKey; use sea_orm::{entity::prelude::*, ActiveValue::NotSet, Set}; -#[derive(Debug, Clone, Eq, PartialEq, EnumIter, DeriveActiveEnum)] +#[derive(Copy, Debug, Clone, Eq, PartialEq, EnumIter, DeriveActiveEnum)] #[sea_orm(rs_type = "i32", db_type = "Integer")] pub enum AuditResult { Failure = 0, Success = 1, + Pending = 2, + Errored = 3, } #[derive(Debug, Clone, Eq, Hash, PartialEq, EnumIter, DeriveActiveEnum, ValueEnum)] @@ -42,8 +45,10 @@ pub enum SelectionStrategy { impl AuditResult { pub fn as_text(&self) -> String { match self { - AuditResult::Failure => "fail".to_string(), - AuditResult::Success => "success".to_string(), + AuditResult::Failure => "Fail".to_string(), + AuditResult::Success => "Success".to_string(), + AuditResult::Pending => "Pending".to_string(), + AuditResult::Errored => "Error".to_string(), } } } @@ -104,35 +109,42 @@ impl Related for Entity { impl ActiveModelBehavior for ActiveModel {} +/// Create a new, pending audit entry. pub async fn create( content_key_model_id: i32, client_info_id: i32, node_id: i32, - query_successful: bool, strategy_used: SelectionStrategy, - trace_string: String, conn: &DatabaseConnection, ) -> Result { - // If no record exists, create one and return it - let audit_result = if query_successful { - AuditResult::Success - } else { - AuditResult::Failure - }; - let content_audit = ActiveModel { id: NotSet, content_key: Set(content_key_model_id), client_info: Set(Some(client_info_id)), node: Set(Some(node_id)), created_at: Set(Utc::now()), - result: Set(audit_result), + result: Set(AuditResult::Pending), strategy_used: Set(Some(strategy_used)), - trace: Set(trace_string), + trace: Set("".to_owned()), }; Ok(content_audit.insert(conn).await?) } +/// Retrieve the audit, update it from pending to success, failure, or errored. +pub async fn record_result( + audit_id: i32, + audit_result: AuditResult, + trace_string: String, + conn: &DatabaseConnection, +) -> Result { + let audit: Option = Entity::find_by_id(audit_id).one(conn).await?; + let audit = audit.ok_or(anyhow!("No audit found for id {}", audit_id))?; + let mut audit: ActiveModel = audit.into(); + audit.result = Set(audit_result); + audit.trace = Set(trace_string); + Ok(audit.update(conn).await?) +} + pub async fn get_audits( content_key: &T, conn: &DatabaseConnection, @@ -168,6 +180,9 @@ impl Model { pub fn is_success(&self) -> bool { self.result == AuditResult::Success } + pub fn is_pending(&self) -> bool { + self.result == AuditResult::Pending + } pub fn created_at_local_time(&self) -> String { self.created_at.with_timezone(&chrono::Local).to_rfc2822() } diff --git a/glados-audit/src/lib.rs b/glados-audit/src/lib.rs index b256480c..cdb0e214 100644 --- a/glados-audit/src/lib.rs +++ b/glados-audit/src/lib.rs @@ -264,46 +264,12 @@ async fn perform_single_audit( client.url = client.api.client_url.clone(), "auditing content", ); - let (content_response, trace) = if client.clone().supports_trace() { - match client.api.get_content_with_trace(&task.content_key).await { - Ok(c) => c, - Err(e) => { - error!( - content.key=hex_encode(task.content_key.to_bytes()), - err=?e, - "Problem requesting content with trace from Portal node." - ); - active_threads.fetch_sub(1, Ordering::Relaxed); - return; - } - } - } else { - match client.api.get_content(&task.content_key).await { - Ok(c) => (c, "".to_owned()), - Err(e) => { - error!( - content.key=hex_encode(task.content_key.to_bytes()), - err=?e, - "Problem requesting content from Portal node." - ); - active_threads.fetch_sub(1, Ordering::Relaxed); - return; - } - } - }; - - // If content was absent audit result is 'fail'. - let audit_result = match content_response { - Some(content_bytes) => content_is_valid(&task.content_key, &content_bytes.raw), - None => false, - }; let content_key_model = match content::get(&task.content_key, &conn).await { Ok(Some(m)) => m, Ok(None) => { error!( content.key=?task.content_key, - audit.pass=?audit_result, "Content key not found in db." ); active_threads.fetch_sub(1, Ordering::Relaxed); @@ -327,6 +293,7 @@ async fn perform_single_audit( err=?error, "Could not create/lookup client info in db." ); + active_threads.fetch_sub(1, Ordering::Relaxed); return; } }; @@ -338,19 +305,75 @@ async fn perform_single_audit( err=?err, "Failed to created node." ); + active_threads.fetch_sub(1, Ordering::Relaxed); return; } }; - if let Err(e) = content_audit::create( + + let pending_audit = match content_audit::create( content_key_model.id, client_info_id, node_id, - audit_result, task.strategy, - trace, &conn, ) .await + { + Ok(audit) => audit, + Err(e) => { + error!( + content.key=?task.content_key, + err=?e, + "Could not create audit entry in db." + ); + active_threads.fetch_sub(1, Ordering::Relaxed); + return; + } + }; + let (content_response, trace, errored) = if client.clone().supports_trace() { + match client.api.get_content_with_trace(&task.content_key).await { + Ok(c) => (c.0, c.1, false), + Err(e) => { + error!( + content.key=hex_encode(task.content_key.to_bytes()), + err=?e, + "Problem requesting content with trace from Portal node." + ); + (None, "".to_owned(), true) + } + } + } else { + match client.api.get_content(&task.content_key).await { + Ok(c) => (c, "".to_owned(), false), + Err(e) => { + error!( + content.key=hex_encode(task.content_key.to_bytes()), + err=?e, + "Problem requesting content from Portal node." + ); + (None, "".to_owned(), true) + } + } + }; + + let audit_result: content_audit::AuditResult = if errored { + content_audit::AuditResult::Errored + } else { + // If content is absent or invalid, audit is a failure. + match content_response { + Some(content) => { + if content_is_valid(&task.content_key, &content.raw) { + content_audit::AuditResult::Success + } else { + content_audit::AuditResult::Failure + } + } + None => content_audit::AuditResult::Failure, + } + }; + + // Update audit entry with result and trace. + if let Err(e) = content_audit::record_result(pending_audit.id, audit_result, trace, &conn).await { error!( content.key=?task.content_key, @@ -366,14 +389,14 @@ async fn perform_single_audit( Ok(Some(b)) => { info!( content.key=hex_encode(task.content_key.to_bytes()), - audit.pass=?audit_result, + audit.result=?audit_result, block = b.block_number, ); } Ok(None) => { error!( content.key=hex_encode(task.content_key.to_bytes()), - audit.pass=?audit_result, + audit.result=?audit_result, "Block metadata absent for key." ); } diff --git a/glados-web/templates/audit_table.html b/glados-web/templates/audit_table.html index d39759d3..d33621f2 100644 --- a/glados-web/templates/audit_table.html +++ b/glados-web/templates/audit_table.html @@ -56,8 +56,9 @@ {{ audit.id }}{% endif %} {% - if audit.is_success() %}Success{% else %}Fail{% endif %} + class="badge text-bg-{% if audit.is_success() %}success{% elseif audit.is_pending() %}warning{% else %}danger{% endif %}"> + {{ audit.result.as_text() }} + {{ content.protocol_id.as_text() }} {{ audit.strategy_as_text() }} {{ content.key_as_hex_short() diff --git a/glados-web/templates/content_dashboard.html b/glados-web/templates/content_dashboard.html index 191dbe3c..8be6f2ce 100644 --- a/glados-web/templates/content_dashboard.html +++ b/glados-web/templates/content_dashboard.html @@ -75,8 +75,9 @@

Recent content with audits

{{ audit.id }}{% endif %} {% - if audit.is_success() %}Success{% else %}Fail{% endif %} + class="badge text-bg-{% if audit.is_success() %}success{% elseif audit.is_pending() %}warning{% else %}danger{% endif %}"> + {{ audit.result.as_text() }} + {{ content.protocol_id.as_text() }} {{ audit.strategy_as_text() }} {{ content.key_as_hex_short() @@ -161,8 +162,9 @@

Recent audits

{{ audit.id }}{% endif %} {% - if audit.is_success() %}Success{% else %}Fail{% endif %} + class="badge text-bg-{% if audit.is_success() %}success{% elseif audit.is_pending() %}warning{% else %}danger{% endif %}"> + {{ audit.result.as_text() }} + {{ content.protocol_id.as_text() }} {{ audit.strategy_as_text() }} {{ content.key_as_hex_short() @@ -212,8 +214,9 @@

Recent audit successes

{{ audit.id }}{% endif %} {% - if audit.is_success() %}Success{% else %}Fail{% endif %} + class="badge text-bg-{% if audit.is_success() %}success{% elseif audit.is_pending() %}warning{% else %}danger{% endif %}"> + {{ audit.result.as_text() }} + {{ content.protocol_id.as_text() }} {{ audit.strategy_as_text() }} {{ content.key_as_hex_short() @@ -263,8 +266,9 @@

Recent audit failures

{{ audit.id }}{% endif %} {% - if audit.is_success() %}Success{% else %}Fail{% endif %} + class="badge text-bg-{% if audit.is_success() %}success{% elseif audit.is_pending() %}warning{% else %}danger{% endif %}"> + {{ audit.result.as_text() }} + {{ content.protocol_id.as_text() }} {{ audit.strategy_as_text() }} {{ content.key_as_hex_short()