chore: add new field custom_license_refs into license entity

Author: bxf12315

entity/src/license.rs

@@ -1,13 +1,14 @@
 use sea_orm::entity::prelude::*;
 
-#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, serde::Serialize)]
 #[sea_orm(table_name = "license")]
 pub struct Model {
     #[sea_orm(primary_key)]
     pub id: Uuid,
     pub text: String,
     pub spdx_licenses: Option<Vec<String>>,
     pub spdx_license_exceptions: Option<Vec<String>>,
+    pub custom_license_refs: Option<Vec<String>>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

migration/src/lib.rs

@@ -25,6 +25,7 @@ mod m0001100_remove_get_purl;
 mod m0001110_sbom_node_checksum_indexes;
 mod m0001120_sbom_external_node_indexes;
 mod m0001130_gover_cmp;
+mod m0001150_license_add_custom_license_refs;
 
 pub struct Migrator;
 
@@ -57,6 +58,7 @@ impl MigratorTrait for Migrator {
             Box::new(m0001110_sbom_node_checksum_indexes::Migration),
             Box::new(m0001120_sbom_external_node_indexes::Migration),
             Box::new(m0001130_gover_cmp::Migration),
+            Box::new(m0001150_license_add_custom_license_refs::Migration),
         ]
     }
 }

migration/src/m0001150_license_add_custom_license_refs.rs

@@ -0,0 +1,38 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(License::Table)
+                    .add_column(ColumnDef::new(License::CustomLicenseRefs).array(ColumnType::Text))
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(License::Table)
+                    .drop_column(License::CustomLicenseRefs)
+                    .to_owned(),
+            )
+            .await?;
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum License {
+    Table,
+    CustomLicenseRefs,
+}

modules/fundamental/tests/sbom/license.rs

@@ -1,16 +1,20 @@
 use flate2::read::GzDecoder;
-use sea_orm::EntityTrait;
+use sea_orm::{ColumnTrait, QuerySelect};
+use sea_orm::{EntityTrait, QueryFilter};
+use sea_query::Cond;
+use serde_json::{Value, json};
 use std::io::Read;
 use tar::Archive;
 use test_context::test_context;
 use test_log::test;
 use trustify_entity::sbom_package_license::LicenseCategory;
-use trustify_entity::{sbom_package, sbom_package_license};
+use trustify_entity::{license, sbom_package, sbom_package_license};
 use trustify_module_fundamental::license::{
     model::sbom_license::SbomNameId,
     service::{LicenseService, license_export::LicenseExporter},
 };
 use trustify_test_context::TrustifyContext;
+use trustify_test_context::subset::ContainsSubset;
 
 #[test_context(TrustifyContext)]
 #[test(tokio::test)]
@@ -39,6 +43,59 @@ async fn test_cyclonedx(ctx: &TrustifyContext) -> Result<(), anyhow::Error> {
     Ok(())
 }
 
+#[test_context(TrustifyContext)]
+#[test(tokio::test)]
+async fn test_custom_license_refs_spdx(ctx: &TrustifyContext) -> Result<(), anyhow::Error> {
+    let result = ctx
+        .ingest_document("spdx/SATELLITE-6.15-RHEL-8.json")
+        .await?;
+
+    assert_eq!(
+        Some("https://access.redhat.com/security/data/sbom/spdx/SATELLITE-6.15-RHEL-8"),
+        result.clone().document_id.as_deref()
+    );
+
+    let license_result = license::Entity::find()
+        .filter(
+            Cond::any()
+                .add(license::Column::Text.eq("LicenseRef-2 OR Ruby"))
+                .add(license::Column::Text.eq("LicenseRef-GPLv3 AND LicenseRef-21")),
+        )
+        .select_only()
+        .column(license::Column::Id)
+        .column(license::Column::Text)
+        .column(license::Column::SpdxLicenses)
+        .column(license::Column::SpdxLicenseExceptions)
+        .column(license::Column::CustomLicenseRefs)
+        .all(&ctx.db)
+        .await?;
+
+    assert!(
+        !license_result.is_empty(),
+        "No license found with the specified text."
+    );
+    let expected_result = json!([
+        {
+            "id": "107c5a51-d315-56fb-9d4c-dd337f242d2e",
+            "text": "LicenseRef-2 OR Ruby",
+            "spdx_licenses": ["Ruby"],
+            "spdx_license_exceptions": null,
+            "custom_license_refs": ["LicenseRef-2:GPLv2+"]
+        },
+        {
+            "id": "5bec012e-9891-5715-a550-09287ced2d54",
+            "text": "LicenseRef-GPLv3 AND LicenseRef-21",
+            "spdx_licenses": null,
+            "spdx_license_exceptions": null,
+            "custom_license_refs": ["LicenseRef-GPLv3:GPLv3", "LicenseRef-21:Public domain"]
+        }
+    ]);
+    let license_result_value: Value =
+        serde_json::to_value(&license_result).expect("Failed to serialize license_result to JSON");
+    assert!(expected_result.contains_subset(license_result_value));
+    Ok(())
+}
+
 #[test_context(TrustifyContext)]
 #[test(tokio::test)]
 async fn test_spdx(ctx: &TrustifyContext) -> Result<(), anyhow::Error> {

modules/ingestor/src/graph/sbom/common/license.rs

@@ -4,7 +4,7 @@ use spdx_expression::SpdxExpression;
 use std::collections::BTreeMap;
 use tracing::instrument;
 use trustify_common::db::chunk::EntityChunkedIter;
-use trustify_entity::license;
+use trustify_entity::{license, licensing_infos};
 use uuid::Uuid;
 
 const NAMESPACE: Uuid = Uuid::from_bytes([
@@ -22,7 +22,7 @@ impl LicenseInfo {
         Uuid::new_v5(&NAMESPACE, self.license.to_lowercase().as_bytes())
     }
 
-    pub fn spdx_info(&self) -> (Vec<String>, Vec<String>) {
+    pub fn spdx_info(&self) -> (Vec<String>, Vec<String>, Vec<String>) {
         SpdxExpression::parse(&self.license)
             .map(|parsed| {
                 let spdx_licenses = parsed
@@ -38,9 +38,20 @@ impl LicenseInfo {
                     .map(|e| e.to_string())
                     .collect::<Vec<_>>();
 
-                (spdx_licenses, spdx_license_exceptions)
+                let custom_license_refs = parsed
+                    .licenses()
+                    .iter()
+                    // There are two types of custom licenses:
+                    // 1 Ones that are defined within the current SBOM, for example those with the "LicenseRef" prefix.
+                    // 2 Ones that reference other SBOM documents, for example those with the "DocumentRef" prefix.
+                    // We only process custom licenses that are defined in the current document.
+                    .filter(|e| e.license_ref && e.document_ref.is_none())
+                    .map(|e| format!("LicenseRef-{}", e.identifier))
+                    .collect::<Vec<_>>();
+
+                (spdx_licenses, spdx_license_exceptions, custom_license_refs)
             })
-            .unwrap_or((vec![], vec![]))
+            .unwrap_or((vec![], vec![], vec![]))
     }
 }
 
@@ -51,19 +62,50 @@ pub struct LicenseCreator {
     /// Uses a [`BTreeMap`] to ensure we have a stable insertion order, avoiding deadlocks on the
     /// database.
     pub licenses: BTreeMap<Uuid, license::ActiveModel>,
+
+    pub custom_license_list: Vec<licensing_infos::ActiveModel>,
 }
 
 impl LicenseCreator {
     pub fn new() -> Self {
         Self {
             licenses: Default::default(),
+            custom_license_list: vec![],
         }
     }
 
+    pub fn put_custom_license_list(
+        &mut self,
+        custom_license_list: Vec<licensing_infos::ActiveModel>,
+    ) {
+        self.custom_license_list = custom_license_list;
+    }
+
     pub fn add(&mut self, info: &LicenseInfo) {
         let uuid = info.uuid();
 
-        let (spdx_licenses, spdx_exceptions) = info.spdx_info();
+        let (spdx_licenses, spdx_exceptions, custom_license_refs) = info.spdx_info();
+        let missing_custom_refs: Vec<_> = custom_license_refs
+            .iter()
+            .filter(|ref_id| {
+                !self
+                    .custom_license_list
+                    .iter()
+                    .any(|c| c.license_id == Set((*ref_id).to_string()))
+            })
+            .cloned()
+            .collect();
+        if !missing_custom_refs.is_empty() {
+            log::warn!(
+                "The following custom license refs are missing from custom_license_list: {:?}",
+                missing_custom_refs
+            );
+        }
+        let custom_license_refs_value = if custom_license_refs.is_empty() {
+            None
+        } else {
+            Some(self.construct_custom_license(custom_license_refs.clone()))
+        };
 
         self.licenses.entry(uuid).or_insert(license::ActiveModel {
             id: Set(uuid),
@@ -78,9 +120,30 @@ impl LicenseCreator {
             } else {
                 Set(Some(spdx_exceptions))
             },
+            custom_license_refs: Set(custom_license_refs_value),
         });
     }
 
+    fn construct_custom_license(&self, custom_license_ids: Vec<String>) -> Vec<String> {
+        use std::collections::HashMap;
+        // Build a HashMap from license_id to name for fast lookup
+        let license_map: HashMap<&String, &String> = self
+            .custom_license_list
+            .iter()
+            .filter_map(|c| {
+                if let (Set(license_id), Set(name)) = (&c.license_id, &c.name) {
+                    Some((license_id, name))
+                } else {
+                    None
+                }
+            })
+            .collect();
+        custom_license_ids
+            .into_iter()
+            .filter_map(|id| license_map.get(&id).map(|name| format!("{}:{}", id, name)))
+            .collect()
+    }
+
     #[instrument(skip_all, fields(num = self.licenses.len()), err(level=tracing::Level::INFO))]
     pub async fn create<C>(self, db: &C) -> Result<(), DbErr>
     where

modules/ingestor/src/graph/sbom/common/licensing_info.rs

@@ -63,6 +63,10 @@ impl LicensingInfoCreator {
         });
     }
 
+    pub fn get_copy_license_refs(&self) -> Vec<licensing_infos::ActiveModel> {
+        self.license_refs.clone()
+    }
+
     #[instrument(skip_all, fields(num = self.license_refs.len()), err)]
     pub async fn create<C>(self, db: &C) -> Result<(), DbErr>
     where

modules/ingestor/src/graph/sbom/mod.rs

@@ -424,7 +424,7 @@ impl SbomContext {
             license: license.to_string(),
         };
 
-        let (spdx_licenses, spdx_exceptions) = license_info.spdx_info();
+        let (spdx_licenses, spdx_exceptions, custom_license_refs) = license_info.spdx_info();
 
         let license = license::Entity::find_by_id(license_info.uuid())
             .one(connection)
@@ -446,6 +446,11 @@ impl SbomContext {
                 } else {
                     Set(Some(spdx_exceptions))
                 },
+                custom_license_refs: if custom_license_refs.is_empty() {
+                    Set(None)
+                } else {
+                    Set(Some(custom_license_refs))
+                },
             }
             .insert(connection)
             .await?

modules/ingestor/src/graph/sbom/spdx.rs

@@ -198,6 +198,8 @@ impl SbomContext {
             license_extracted_refs.add(extracted_licensing_info);
         }
 
+        licenses.put_custom_license_list(license_extracted_refs.get_copy_license_refs());
+
         let mut packages =
             PackageCreator::with_capacity(self.sbom.sbom_id, sbom_data.package_information.len());