docs: ADR for re-processing of documents

.dockerignore

@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+/data
+.trustify
+/target
+/.dockerignore
+/Containerfile

Cargo.toml

@@ -63,6 +63,8 @@ csaf = { version = "0.5.0", default-features = false }
 csaf-walker = { version = "0.14.1", default-features = false }
 csv = "1.3.0"
 cve = "0.4.0"
+cvss = { git = "https://github.com/dejanb/cvss" }
+cvss-old = { package = "cvss", version = "2" }
 deepsize = "0.2.0"
 fixedbitset = "0.5.7"
 flate2 = "1.0.35"
@@ -77,6 +79,7 @@ http = "1"
 human-date-parser = "0.3"
 humantime = "2"
 humantime-serde = "1"
+indicatif = "0.18.0"
 itertools = "0.14"
 jsn = "0.14"
 json-merge-patch = "0.0.1"
@@ -203,7 +206,7 @@ postgresql_commands = { version = "0.20.0", default-features = false, features =
 # required due to https://github.com/KenDJohnson/cpe-rs/pull/15
 #cpe = { git = "https://github.com/ctron/cpe-rs", rev = "c3c05e637f6eff7dd4933c2f56d070ee2ddfb44b" }
 # required due to https://github.com/voteblake/csaf-rs/pull/29
-csaf = { git = "https://github.com/trustification/csaf-rs" }
+csaf = { git = "https://github.com/trustification/csaf-rs", branch = "cvss" }
 # required due to https://github.com/gcmurphy/osv/pull/58
 #osv = { git = "https://github.com/ctron/osv", branch = "feature/drop_deps_1" }
 

Containerfile

@@ -0,0 +1,22 @@
+FROM registry.access.redhat.com/ubi9/ubi:latest AS builder
+
+RUN dnf install --setop install_weak_deps=false --nodocs -y git python gcc g++ cmake ninja-build openssl-devel xz
+
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN mkdir /build
+
+COPY . /build
+
+WORKDIR /build
+
+RUN ls
+
+RUN rm rust-toolchain.toml
+
+RUN cargo build --release
+
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+
+COPY --from=builder /build/target/release/trustd /usr/local/bin/

TODO.md

@@ -0,0 +1,4 @@
+# ToDo
+
+* [ ] Allowing skipping data part of the migration
+* [x] Allow concurrent instances (x of y)

common/Cargo.toml

@@ -17,6 +17,7 @@ deepsize = { workspace = true }
 hex = { workspace = true }
 hide = { workspace = true }
 human-date-parser = { workspace = true }
+humantime = { workspace = true }
 itertools = { workspace = true }
 lenient_semver = { workspace = true }
 log = { workspace = true }
@@ -32,6 +33,7 @@ sea-orm = { workspace = true, features = ["sea-query-binder", "sqlx-postgres", "
 sea-orm-migration = { workspace = true }
 sea-query = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
+serde-cyclonedx = { workspace = true }
 serde_json = { workspace = true }
 spdx-expression = { workspace = true }
 spdx-rs = { workspace = true }
@@ -45,7 +47,6 @@ urlencoding = { workspace = true }
 utoipa = { workspace = true, features = ["url"] }
 uuid = { workspace = true, features = ["v5", "serde"] }
 walker-common = { workspace = true, features = ["bzip2", "liblzma", "flate2"] }
-humantime = { workspace = true }
 
 [dev-dependencies]
 chrono = { workspace = true }

common/db/src/lib.rs

@@ -2,6 +2,7 @@ pub mod embedded;
 
 use anyhow::{Context, anyhow, ensure};
 use migration::Migrator;
+use migration::data::Runner;
 use postgresql_commands::{CommandBuilder, psql::PsqlBuilder};
 use sea_orm::{ConnectionTrait, Statement};
 use sea_orm_migration::prelude::MigratorTrait;
@@ -121,4 +122,8 @@ impl<'a> Database<'a> {
 
         Ok(db)
     }
+
+    pub async fn data_migrate(&self, runner: Runner) -> Result<(), anyhow::Error> {
+        runner.run::<Migrator>().await
+    }
 }

common/src/advisory/cyclonedx.rs

@@ -0,0 +1,27 @@
+use serde_cyclonedx::cyclonedx::v_1_6::CycloneDx;
+use std::collections::HashMap;
+
+/// extract CycloneDX SBOM general purpose properties
+pub fn extract_properties(sbom: &CycloneDx) -> HashMap<String, Option<String>> {
+    sbom.properties
+        .iter()
+        .flatten()
+        .map(|e| (e.name.clone(), e.value.clone()))
+        .collect()
+}
+
+/// extract CycloneDX SBOM general purpose properties, convert into [`serde_json::Value`]
+pub fn extract_properties_json(sbom: &CycloneDx) -> serde_json::Value {
+    serde_json::Value::Object(
+        extract_properties(sbom)
+            .into_iter()
+            .map(|(k, v)| {
+                (
+                    k,
+                    v.map(serde_json::Value::String)
+                        .unwrap_or(serde_json::Value::Null),
+                )
+            })
+            .collect(),
+    )
+}

common/src/advisory/mod.rs

@@ -1,3 +1,5 @@
+pub mod cyclonedx;
+
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use utoipa::ToSchema;

common/src/db/create.rs

@@ -0,0 +1,37 @@
+use sea_orm::{ConnectionTrait, DbErr};
+use sea_orm_migration::SchemaManager;
+use sea_query::{IntoIden, extension::postgres::Type};
+
+/// create a type, if it not already exists
+///
+/// This is required as Postgres doesn't support `CREATE TYPE IF NOT EXISTS`
+pub async fn create_enum_if_not_exists<T, I>(
+    manager: &SchemaManager<'_>,
+    name: impl IntoIden + Clone,
+    values: I,
+) -> Result<(), DbErr>
+where
+    T: IntoIden,
+    I: IntoIterator<Item = T>,
+{
+    let builder = manager.get_connection().get_database_backend();
+    let r#type = name.clone().into_iden();
+    let stmt = builder.build(Type::create().as_enum(name).values(values));
+    let stmt = format!(
+        r#"
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_type WHERE typname = '{name}'
+  ) THEN
+    {stmt};
+  END IF;
+END$$;
+"#,
+        name = r#type.to_string()
+    );
+
+    manager.get_connection().execute_unprepared(&stmt).await?;
+
+    Ok(())
+}

common/src/db/mod.rs

@@ -3,7 +3,10 @@ pub mod limiter;
 pub mod multi_model;
 pub mod query;
 
+mod create;
 mod func;
+
+pub use create::*;
 pub use func::*;
 
 use anyhow::Context;
@@ -103,6 +106,10 @@ impl Database {
     pub fn name(&self) -> &str {
         &self.name
     }
+
+    pub fn into_connection(self) -> DatabaseConnection {
+        self.db
+    }
 }
 
 impl Deref for Database {

data-migration.yaml

@@ -0,0 +1,80 @@
+kind: Job
+apiVersion: batch/v1
+metadata:
+  name: data-migration-test
+spec:
+  completions: 4
+  completionMode: Indexed
+  parallelism: 4 # same as completions
+  template:
+    spec:
+      restartPolicy: OnFailure
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: "kubernetes.io/arch"
+                  operator: In
+                  values: ["amd64"]
+      containers:
+        - name: run
+          image: quay.io/ctrontesting/trustd:latest
+          imagePullPolicy: Always
+          command:
+            - /usr/local/bin/trustd
+            - db
+            - data
+            - m0002010_add_advisory_scores # name of the migration
+          env:
+            - name: MIGRATION_DATA_CONCURRENT
+              value: "5" # in-process parallelism
+            - name: MIGRATION_DATA_TOTAL_RUNNER
+              value: "4" # same as completions
+            - name: MIGRATION_DATA_CURRENT_RUNNER
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index']
+
+            - name: TRUSTD_STORAGE_STRATEGY
+              value: s3
+            - name: TRUSTD_S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: storage-credentials
+                  key: aws_access_key_id
+            - name: TRUSTD_S3_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: storage-credentials
+                  key: aws_secret_access_key
+            - name: TRUSTD_S3_REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: aws-storage
+                  key: region
+            - name: TRUSTD_S3_BUCKET
+              value: trustify-default
+
+            - name: TRUSTD_DB_NAME
+              value: trustify_default
+            - name: TRUSTD_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: postgresql
+                  key: username
+            - name: TRUSTD_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgresql
+                  key: password
+            - name: TRUSTD_DB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: postgresql
+                  key: host
+            - name: TRUSTD_DB_PORT
+              value: "5432"
+
+            - name: RUST_LOG
+              value: info