diff --git a/CHANGELOG.md b/CHANGELOG.md index fefa2f65f..44fa7ac64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,18 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ### Data Format +- Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to + identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`, + `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following + schema update will be necessary: + ```sql + ALTER TABLE events ADD "product.full_name" text; + ALTER TABLE events ADD "product.name" text; + ALTER TABLE events ADD "product.vendor" text; + ALTER TABLE events ADD "product.version" text; + ALTER TABLE events ADD "product.vulnerabilities" text; + ``` + ### Bots #### Collectors diff --git a/NEWS.md b/NEWS.md index 12c225784..a6ccf4bd1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,6 +18,15 @@ Please refer to the change log for a full list of changes. ### Tools ### Data Format +To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema +update is necessary: +```sql +ALTER TABLE events ADD "product.full_name" text; +ALTER TABLE events ADD "product.name" text; +ALTER TABLE events ADD "product.vendor" text; +ALTER TABLE events ADD "product.version" text; +ALTER TABLE events ADD "product.vulnerabilities" text; +``` ### Configuration diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index 027643ac9..b9e57d6e2 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -209,6 +209,26 @@ "description": "Event data converted into foreign format, intended to be exported by output plugin.", "type": "JSON" }, + "product.full_name": { + "description": "A human readable product name. If a machine-readable format isn't available, this field should be used. It can directly use the version identification strings presented by the product. If not given, a good enough value can usually be constructed by concatenating product.product and product.version, or by consulting external sources such as the CPE Product Dictionary. Example: openssh_/8.9", + "type": "String" + }, + "product.name": { + "description": "Product name, recommended being as the product in the CPE format. Example: openssh", + "type": "LowercaseString" + }, + "product.vendor": { + "description": "Vendor name, recommended being as vendor in the CPE format. Example: openbsd", + "type": "LowercaseString" + }, + "product.version": { + "description": "Product version, recommended being as version in the CPE format. Example: 8.9", + "type": "LowercaseString" + }, + "product.vulnerabilities": { + "description": "List of vulnerability IDs, separated by semicolons. It's recommended to use a CVE ID where available, and other easily retrievable IDs in other cases, e.g. Github Advisory Database ID. Each vulnerability should only be listed once, and multiple values should be used if there are several different vulnerabilities. However, it's not necessary for a source to list all possible vulnerabilities for a given piece of software. Example: cve-2023-38408;cve-2023-28531;cve-2008-3844;cve-2007-2768", + "type": "LowercaseString" + }, "protocol.application": { "description": "e.g. vnc, ssh, sip, irc, http or smtp.", "length": 100, diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index ee22b60a6..6da2f1a7b 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -41,7 +41,8 @@ 'v320_update_turris_greylist_url', 'v322_url_replacement', 'v322_removed_feeds_and_bots', - 'v340_deprecations' + 'v340_deprecations', + 'v341_new_fields' ] @@ -974,6 +975,33 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization +def v341_new_fields(configuration, harmonization, dry_run, **kwargs): + """ + Add new fields to IntelMQ Data Format + """ + changed = None + if "event" not in harmonization: + return changed, configuration, harmonization + + builtin_harmonisation = load_configuration( + resource_filename("intelmq", "etc/harmonization.conf") + ) + for field in [ + "product.full_name", + "product.name", + "product.vendor", + "product.version", + "product.vulnerabilities", + ]: + if field not in harmonization["event"]: + if field not in builtin_harmonisation["event"]: + # ensure forward-compatibility if we ever remove something from harmonisation + continue + harmonization["event"][field] = builtin_harmonisation["event"][field] + changed = True + return changed, configuration, harmonization + + UPGRADES = OrderedDict([ ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)), ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), @@ -1004,7 +1032,8 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): ((3, 3, 0), ()), ((3, 3, 1), ()), ((3, 4, 0), (v340_deprecations, )), - ((3, 4, 1), ()), + ((3, 4, 1), (v341_new_fields, )), + ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 5a5f839f5..3a6cd03b4 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -47,6 +47,11 @@ CREATE TABLE events ( "misp.attribute_uuid" varchar(36), "misp.event_uuid" varchar(36), "output" json, + "product.full_name" text, + "product.name" text, + "product.vendor" text, + "product.version" text, + "product.vulnerabilities" text, "protocol.application" varchar(100), "protocol.transport" varchar(11), "raw" text, diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index a30800b9c..5dc650366 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -856,6 +856,17 @@ def test_v340_twitter_collector(self): self.assertIn('twitter-collector', result[0]) self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1]) + def test_v341_new_fields(self): + """ Test adding new harmonisation fields """ + result = upgrades.v341_new_fields({}, {"event": {"old-field": "must stay"}}, False) + self.assertTrue(result[0]) + self.assertIn("old-field", result[2]["event"]) + self.assertIn("product.full_name", result[2]["event"]) + self.assertIn("product.name", result[2]["event"]) + self.assertIn("product.vendor", result[2]["event"]) + self.assertIn("product.version", result[2]["event"]) + self.assertIn("product.vulnerabilities", result[2]["event"]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name,