Backports/708/v3

doc/userguide/configuration/suricata-yaml.rst

@@ -632,6 +632,7 @@ The detection-engine builds internal groups of signatures. Suricata loads signat
       toserver-groups: 25
     sgh-mpm-context: auto
     inspection-recursion-limit: 3000
+    guess-applayer-tx: no
 
 At all of these options, you can add (or change) a value. Most
 signatures have the adjustment to focus on one direction, meaning
@@ -666,6 +667,12 @@ complicated issues. It could end up in an 'endless loop' due to a bug,
 meaning it will repeat its actions over and over again. With the
 option inspection-recursion-limit you can limit this action.
 
+The ``guess-applayer-tx`` option controls whether the engine will try to guess
+and tie a transaction to a given alert if the matching signature doesn't have
+app-layer keywords. If enabled, AND ONLY ONE LIVE TRANSACTION EXISTS, that
+transaction's data will be added to the alert metadata. Note that this may not
+be the expected data, from an analyst's perspective.
+
 *Example 4	Detection-engine grouping tree*
 
 .. image:: suricata-yaml/grouping_tree.png

doc/userguide/output/eve/eve-json-output.rst

@@ -62,6 +62,21 @@ Alerts are event records for rule matches. They can be amended with
 metadata, such as the application layer record (HTTP, DNS, etc) an
 alert was generated for, and elements of the rule.
 
+The alert is amended with application layer metadata for signatures
+using application layer keywords. It is also the case for protocols
+over UDP as each single packet is expected to contain a PDU.
+
+For other signatures, the option ``guess-applayer-tx``
+can be used to force the detect engine to tie a transaction
+to an alert.
+This transaction is not guaranteed to be the relevant one,
+depending on your use case and how you define relevant here.
+**WARNING: If there are multiple live transactions, none will get
+picked up.** This is to reduce the chances of logging unrelated data, and may
+lead to alerts being logged without metadata, in some cases.
+The alert event will have ``tx_guessed: true`` to recognize
+such alerts.
+
 Metadata::
 
         - alert:

doc/userguide/upgrade.rst

@@ -59,6 +59,13 @@ Upgrading to 7.0.8
             7.0 releases. It will not be provided in
             Suricata 8. Please fix any rules that depend on this
             behavior.
+- Application layer metadata is logged with alerts by default **only for rules that
+  use application layer keywords**. For other rules, the configuration parameter
+  ``detect.guess-applayer-tx`` can be used to force the detect engine to guess a
+  transaction, which is not guaranteed to be the one you expect. **In this case,
+  the engine will NOT log any transaction metadata if there is more than one
+  live transaction, to reduce the chances of logging unrelated data.** This may
+  lead to what looks like a regression in behavior, but it is a considered choice.
 
 Upgrading 6.0 to 7.0
 --------------------

etc/schema.json

@@ -112,6 +112,10 @@
         "tx_id": {
             "type": "integer"
         },
+        "tx_guessed": {
+            "description": "the signature that triggered this alert didn't tie to a transaction, so the transaction (and metadata) logged is a forced estimation and may not be the one you expect",
+            "type": "boolean"
+        },
         "files": {
             "type": "array",
             "minItems": 1,

src/decode.h

@@ -281,6 +281,8 @@ typedef struct PacketAlert_ {
 #define PACKET_ALERT_RATE_FILTER_MODIFIED   0x10
 /** alert is in a frame, frame_id set */
 #define PACKET_ALERT_FLAG_FRAME 0x20
+/** alert in a tx was forced */
+#define PACKET_ALERT_FLAG_TX_GUESSED 0x040
 
 extern uint16_t packet_alert_max;
 #define PACKET_ALERT_MAX 15

src/detect-engine.c

@@ -2922,7 +2922,14 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
     SCLogDebug("de_ctx->inspection_recursion_limit: %d",
                de_ctx->inspection_recursion_limit);
 
-    /* parse port grouping whitelisting settings */
+    int guess_applayer = 0;
+    if ((ConfGetBool("detect.guess-applayer-tx", &guess_applayer)) == 1) {
+        if (guess_applayer == 1) {
+            de_ctx->guess_applayer = true;
+        }
+    }
+
+    /* parse port grouping priority settings */
 
     const char *ports = NULL;
     (void)ConfGet("detect.grouping.tcp-whitelist", &ports);

src/detect.c

@@ -814,16 +814,20 @@ static inline void DetectRulePacketRules(
         DetectRunPostMatch(tv, det_ctx, p, s);
 
         uint64_t txid = PACKET_ALERT_NOTX;
-        if ((alert_flags & PACKET_ALERT_FLAG_STREAM_MATCH) ||
-                (s->alproto != ALPROTO_UNKNOWN && pflow->proto == IPPROTO_UDP)) {
-            // if there is a stream match (TCP), or
-            // a UDP specific app-layer signature,
-            // try to use the good tx for the packet direction
-            if (pflow->alstate) {
-                uint8_t dir =
-                        (p->flowflags & FLOW_PKT_TOCLIENT) ? STREAM_TOCLIENT : STREAM_TOSERVER;
-                txid = AppLayerParserGetTransactionInspectId(pflow->alparser, dir);
+        if (pflow && pflow->alstate) {
+            uint8_t dir = (p->flowflags & FLOW_PKT_TOCLIENT) ? STREAM_TOCLIENT : STREAM_TOSERVER;
+            txid = AppLayerParserGetTransactionInspectId(pflow->alparser, dir);
+            if ((s->alproto != ALPROTO_UNKNOWN && pflow->proto == IPPROTO_UDP) ||
+                    (alert_flags & PACKET_ALERT_FLAG_STREAM_MATCH) ||
+                    (de_ctx->guess_applayer &&
+                            AppLayerParserGetTxCnt(pflow, pflow->alstate) == txid + 1)) {
+                // if there is a UDP specific app-layer signature,
+                // or only one live transaction
+                // try to use the good tx for the packet direction
                 alert_flags |= PACKET_ALERT_FLAG_TX;
+                if (pflow->proto != IPPROTO_UDP) {
+                    alert_flags |= PACKET_ALERT_FLAG_TX_GUESSED;
+                }
             }
         }
         AlertQueueAppend(det_ctx, s, p, txid, alert_flags);

src/detect.h

@@ -885,6 +885,9 @@ typedef struct DetectEngineCtx_ {
     /* maximum recursion depth for content inspection */
     int inspection_recursion_limit;
 
+    /* force app-layer tx finding for alerts with signatures not having app-layer keywords */
+    bool guess_applayer;
+
     /* registration id for per thread ctx for the filemagic/file.magic keywords */
     int filemagic_thread_ctx_id;
 

src/output-json-alert.c

@@ -378,6 +378,9 @@ void AlertJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, JsonBuil
     if (pa->flags & PACKET_ALERT_FLAG_TX) {
         jb_set_uint(js, "tx_id", pa->tx_id);
     }
+    if (pa->flags & PACKET_ALERT_FLAG_TX_GUESSED) {
+        jb_set_bool(js, "tx_guessed", true);
+    }
 
     jb_open_object(js, "alert");
 

suricata.yaml.in

@@ -1677,6 +1677,11 @@ detect:
     toserver-groups: 25
   sgh-mpm-context: auto
   inspection-recursion-limit: 3000
+  # try to tie an app-layer transaction for rules without app-layer keywords
+  # if there is only one live transaction for the flow
+  # allows to log app-layer metadata in alert
+  # but the transaction may not be the relevant one.
+  # guess-applayer-tx: no
   # If set to yes, the loading of signatures will be made after the capture
   # is started. This will limit the downtime in IPS mode.
   #delayed-detect: yes