app-layer: allow at compile-time more app-layer

So that we can have dynamically registered protocols.
Doing it at compile time, with CFLAGS=-DALPROTO_DYNAMIC_NB=1,
allows to keep the rest of the code using ALPROTO_MAX

Ticket: 5053

rust/src/core.rs

@@ -253,6 +253,13 @@ pub extern "C" fn rs_init(context: &'static SuricataContext)
     init_ffi(context);
 }
 
+#[no_mangle]
+pub extern "C" fn rs_update_alproto_failed(alproto: AppProto) {
+    unsafe {
+        ALPROTO_FAILED = alproto;
+    }
+}
+
 /// DetectEngineStateFree wrapper.
 pub fn sc_detect_engine_state_free(state: *mut DetectEngineState)
 {

src/app-layer-detect-proto.c

@@ -489,8 +489,8 @@ static inline AppProto PPGetProto(const AppLayerProtoDetectProbingParserElement
         if (AppProtoIsValid(alproto)) {
             SCReturnUInt(alproto);
         }
-        if (alproto == ALPROTO_FAILED ||
-            (pe->max_depth != 0 && buflen > pe->max_depth)) {
+        if (alproto == ALPROTO_FAILED || alproto == ALPROTO_INVALID ||
+                (pe->max_depth != 0 && buflen > pe->max_depth)) {
             alproto_masks[0] |= pe->alproto_mask;
         }
         pe = pe->next;

src/app-layer-protos.c

@@ -24,13 +24,16 @@
 
 #include "suricata-common.h"
 #include "app-layer-protos.h"
+#include "rust.h"
+
+AppProto ALPROTO_FAILED = ALPROTO_MAX_STATIC;
 
 typedef struct AppProtoStringTuple {
     AppProto alproto;
     const char *str;
 } AppProtoStringTuple;
 
-const AppProtoStringTuple AppProtoStrings[ALPROTO_MAX] = {
+AppProtoStringTuple AppProtoStrings[ALPROTO_MAX] = {
     { ALPROTO_UNKNOWN, "unknown" },
     { ALPROTO_HTTP1, "http1" },
     { ALPROTO_FTP, "ftp" },
@@ -65,10 +68,10 @@ const AppProtoStringTuple AppProtoStrings[ALPROTO_MAX] = {
     { ALPROTO_HTTP2, "http2" },
     { ALPROTO_BITTORRENT_DHT, "bittorrent-dht" },
     { ALPROTO_HTTP, "http" },
-    { ALPROTO_FAILED, "failed" },
 #ifdef UNITTESTS
     { ALPROTO_TEST, "test" },
 #endif
+    { ALPROTO_MAX_STATIC, "failed" },
 };
 
 const char *AppProtoToString(AppProto alproto)
@@ -104,3 +107,15 @@ AppProto StringToAppProto(const char *proto_name)
 
     return ALPROTO_UNKNOWN;
 }
+
+void RegisterAppProtoString(AppProto alproto, const char *proto_name)
+{
+    if (alproto == ALPROTO_FAILED && alproto + 1 < ALPROTO_MAX) {
+        AppProtoStrings[alproto].str = proto_name;
+        AppProtoStrings[alproto].alproto = alproto;
+        ALPROTO_FAILED++;
+        rs_update_alproto_failed(ALPROTO_FAILED);
+        AppProtoStrings[ALPROTO_FAILED].str = "failed";
+        AppProtoStrings[ALPROTO_FAILED].alproto = ALPROTO_FAILED;
+    }
+}

src/app-layer-protos.h

@@ -65,20 +65,24 @@ enum AppProtoEnum {
     // HTTP for any version (ALPROTO_HTTP1 (version 1) or ALPROTO_HTTP2)
     ALPROTO_HTTP,
 
-    /* used by the probing parser when alproto detection fails
-     * permanently for that particular stream */
-    ALPROTO_FAILED,
 #ifdef UNITTESTS
     ALPROTO_TEST,
 #endif /* UNITESTS */
     /* keep last */
-    ALPROTO_MAX,
+    ALPROTO_MAX_STATIC,
+    ALPROTO_INVALID = 0xffff,
 };
 // NOTE: if ALPROTO's get >= 256, update SignatureNonPrefilterStore
 
 /* not using the enum as that is a unsigned int, so 4 bytes */
 typedef uint16_t AppProto;
+extern AppProto ALPROTO_FAILED;
 
+#ifdef ALPROTO_DYNAMIC_NB
+#define ALPROTO_MAX ALPROTO_MAX_STATIC + 1 + ALPROTO_DYNAMIC_NB
+#else
+#define ALPROTO_MAX ALPROTO_MAX_STATIC + 1
+#endif
 static inline bool AppProtoIsValid(AppProto a)
 {
     return ((a > ALPROTO_UNKNOWN && a < ALPROTO_FAILED));
@@ -115,4 +119,6 @@ const char *AppProtoToString(AppProto alproto);
  */
 AppProto StringToAppProto(const char *proto_name);
 
+void RegisterAppProtoString(AppProto alproto, const char *proto_name);
+
 #endif /* __APP_LAYER_PROTOS_H__ */

src/app-layer.c

@@ -839,42 +839,39 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
                 tctx->alpd_tctx, f, p->payload, p->payload_len, IPPROTO_UDP, flags, &reverse_flow);
         PACKET_PROFILING_APP_PD_END(tctx);
 
-        switch (*alproto) {
-            case ALPROTO_UNKNOWN:
-                if (*alproto_otherdir != ALPROTO_UNKNOWN) {
-                    // Use recognized side
-                    f->alproto = *alproto_otherdir;
-                    // do not keep ALPROTO_UNKNOWN for this side so as not to loop
-                    *alproto = *alproto_otherdir;
-                    if (*alproto_otherdir == ALPROTO_FAILED) {
-                        SCLogDebug("ALPROTO_UNKNOWN flow %p", f);
-                    }
-                } else {
-                    // First side of protocol is unknown
-                    *alproto = ALPROTO_FAILED;
+        if (*alproto == ALPROTO_UNKNOWN) {
+            if (*alproto_otherdir != ALPROTO_UNKNOWN) {
+                // Use recognized side
+                f->alproto = *alproto_otherdir;
+                // do not keep ALPROTO_UNKNOWN for this side so as not to loop
+                *alproto = *alproto_otherdir;
+                if (*alproto_otherdir == ALPROTO_FAILED) {
+                    SCLogDebug("ALPROTO_UNKNOWN flow %p", f);
                 }
-                break;
-            case ALPROTO_FAILED:
-                if (*alproto_otherdir != ALPROTO_UNKNOWN) {
-                    // Use recognized side
-                    f->alproto = *alproto_otherdir;
-                    if (*alproto_otherdir == ALPROTO_FAILED) {
-                        SCLogDebug("ALPROTO_UNKNOWN flow %p", f);
-                    }
+            } else {
+                // First side of protocol is unknown
+                *alproto = ALPROTO_FAILED;
+            }
+        } else if (*alproto == ALPROTO_FAILED) {
+            if (*alproto_otherdir != ALPROTO_UNKNOWN) {
+                // Use recognized side
+                f->alproto = *alproto_otherdir;
+                if (*alproto_otherdir == ALPROTO_FAILED) {
+                    SCLogDebug("ALPROTO_UNKNOWN flow %p", f);
                 }
-                // else wait for second side of protocol
-                break;
-            default:
-                if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != ALPROTO_FAILED) {
-                    if (*alproto_otherdir != *alproto) {
-                        AppLayerDecoderEventsSetEventRaw(
-                                &p->app_layer_events, APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS);
-                        // data already sent to parser, we cannot change the protocol to use the one
-                        // of the server
-                    }
-                } else {
-                    f->alproto = *alproto;
+            }
+            // else wait for second side of protocol
+        } else {
+            if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != ALPROTO_FAILED) {
+                if (*alproto_otherdir != *alproto) {
+                    AppLayerDecoderEventsSetEventRaw(
+                            &p->app_layer_events, APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS);
+                    // data already sent to parser, we cannot change the protocol to use the one
+                    // of the server
                 }
+            } else {
+                f->alproto = *alproto;
+            }
         }
         if (*alproto_otherdir == ALPROTO_UNKNOWN) {
             if (f->alproto == ALPROTO_UNKNOWN) {

src/output.c

@@ -1332,10 +1332,10 @@ static EveJsonSimpleAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = {
     { ALPROTO_HTTP2, rs_http2_log_json },
     { ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log },
     { ALPROTO_HTTP, NULL }, // signature protocol, not for app-layer logging
-    { ALPROTO_FAILED, NULL },
 #ifdef UNITTESTS
     { ALPROTO_TEST, NULL },
 #endif /* UNITESTS */
+    { ALPROTO_MAX_STATIC, NULL },
 };
 
 EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto)