Added news entry and upgrade function

Author: kamil-certat

NEWS.md

@@ -18,6 +18,11 @@ Please refer to the change log for a full list of changes.
 ### Tools
 
 ### Data Format
+To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema
+update is necessary:
+```sql
+ALTER TABLE events ADD severity varchar(10);
+```
 
 ### Configuration
 

intelmq/lib/upgrades.py

@@ -973,6 +973,18 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
         message = f"Found discontinued Twitter collector bot: {', '.join(found_twitter_collector)}"
     return message or changed, configuration, harmonization
 
+def v341_new_fields(configuration, harmonization, dry_run, **kwargs):
+    changed = False
+    if "severity" not in harmonization["event"]:
+        harmonization["event"]["severity"] = {
+            "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.",
+            "length": 10,
+            "regex": "^(critical|high|medium|low|info|undefined)$",
+            "type": "LowercaseString",
+        }
+        changed = True
+    return changed, configuration, harmonization
+
 
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
@@ -1004,7 +1016,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 0), ()),
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
-    ((3, 4, 1), ()),
+    ((3, 4, 1), (v341_new_fields, )),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/bin/initdb.sql

@@ -94,4 +94,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn");
 CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip");
 CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn");
 CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation");
-CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source");
+CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source");

intelmq/tests/lib/test_upgrades.py

@@ -856,6 +856,13 @@ def test_v340_twitter_collector(self):
         self.assertIn('twitter-collector', result[0])
         self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1])
 
+    def test_v341_new_fields(self):
+        """ Test adding new harmonisation fields """
+        result = upgrades.v341_new_fields({}, {"event": {"old-field": "must stay"}}, False)
+        self.assertTrue(result[0])
+        self.assertIn("old-field", result[1]["event"])
+        self.assertIn("severity", result[1]["event"])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,