detect: helper to have pure rust keywords

detect: make number of keywords dynamic

Ticket: 4683

configure.ac

@@ -318,6 +318,17 @@
         CFLAGS="$CFLAGS -std=gnu99"
     fi
 
+    # check if our target supports -Wl,--start-group
+    AC_MSG_CHECKING(for -Wl,--start-group support)
+    OLDFLAGS=$LDFLAGS
+    LDFLAGS="-Wl,--start-group,--end-group"
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],[
+            have_linker_group_support=yes
+            AC_MSG_RESULT(yes)],
+            [AC_MSG_RESULT(no)])
+    LDFLAGS=$OLDFLAGS
+    AM_CONDITIONAL([LINKER_SUPPORTS_GROUP], [test "x$have_linker_group_support" = "xyes"])
+
     # check if our target supports thread local storage
     AC_MSG_CHECKING(for thread local storage gnu __thread support)
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <stdlib.h>]],

examples/lib/simple/Makefile.am

@@ -5,7 +5,7 @@ simple_SOURCES = main.c
 AM_CPPFLAGS = -I$(top_srcdir)/src
 
 simple_LDFLAGS = $(all_libraries) $(SECLDFLAGS)
-simple_LDADD = $(top_builddir)/src/libsuricata_c.a ../../$(RUST_SURICATA_LIB) $(RUST_LDADD)
+simple_LDADD = "-Wl,--start-group,$(top_builddir)/src/libsuricata_c.a,../../$(RUST_SURICATA_LIB),--end-group" $(RUST_LDADD)
 if HTP_LDADD
 simple_LDADD += ../../$(HTP_LDADD)
 endif

rust/cbindgen.toml

@@ -82,6 +82,7 @@ include = [
     "QuicState",
     "QuicTransaction",
     "FtpEvent",
+    "SCSigTableElmt",
 ]
 
 # A list of items to not include in the generated bindings

rust/src/detect/mod.rs

@@ -27,6 +27,9 @@ pub mod uri;
 pub mod requires;
 pub mod tojson;
 
+use crate::core::AppProto;
+use std::os::raw::{c_int, c_void};
+
 /// EnumString trait that will be implemented on enums that
 /// derive StringEnum.
 pub trait EnumString<T> {
@@ -43,6 +46,64 @@ pub trait EnumString<T> {
     fn from_str(s: &str) -> Option<Self> where Self: Sized;
 }
 
+#[repr(C)]
+#[allow(non_snake_case)]
+pub struct SCSigTableElmt {
+    pub name: *const libc::c_char,
+    pub desc: *const libc::c_char,
+    pub url: *const libc::c_char,
+    pub flags: u16,
+    pub Setup: unsafe extern "C" fn(
+        de: *mut c_void,
+        s: *mut c_void,
+        raw: *const std::os::raw::c_char,
+    ) -> c_int,
+    pub Free: Option<unsafe extern "C" fn(de: *mut c_void, ptr: *mut c_void)>,
+    pub AppLayerTxMatch: Option<
+        unsafe extern "C" fn(
+            de: *mut c_void,
+            f: *mut c_void,
+            flags: u8,
+            state: *mut c_void,
+            tx: *mut c_void,
+            sig: *const c_void,
+            ctx: *const c_void,
+        ) -> c_int,
+    >,
+}
+
+pub(crate) const SIGMATCH_NOOPT: u16 = 1; // BIT_U16(0) in detect.h
+pub(crate) const SIGMATCH_INFO_STICKY_BUFFER: u16 = 0x200; // BIT_U16(9)
+
+extern {
+    pub fn DetectBufferSetActiveList(de: *mut c_void, s: *mut c_void, bufid: c_int) -> c_int;
+    pub fn DetectHelperGetData(
+        de: *mut c_void, transforms: *const c_void, flow: *const c_void, flow_flags: u8,
+        tx: *const c_void, list_id: c_int,
+        get_buf: unsafe extern "C" fn(*const c_void, u8, *mut *const u8, *mut u32) -> bool,
+    ) -> *mut c_void;
+    pub fn DetectHelperBufferMpmRegister(
+        name: *const libc::c_char, desc: *const libc::c_char, alproto: AppProto, toclient: bool,
+        toserver: bool,
+        get_data: unsafe extern "C" fn(
+            *mut c_void,
+            *const c_void,
+            *const c_void,
+            u8,
+            *const c_void,
+            i32,
+        ) -> *mut c_void,
+    ) -> c_int;
+    pub fn DetectHelperKeywordRegister(kw: *const SCSigTableElmt) -> c_int;
+    pub fn DetectHelperBufferRegister(
+        name: *const libc::c_char, alproto: AppProto, toclient: bool, toserver: bool,
+    ) -> c_int;
+    pub fn DetectSignatureSetAppProto(s: *mut c_void, alproto: AppProto) -> c_int;
+    pub fn SigMatchAppendSMToList(
+        de: *mut c_void, s: *mut c_void, kwid: c_int, ctx: *const c_void, bufid: c_int,
+    ) -> *mut c_void;
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

src/Makefile.am

@@ -131,6 +131,7 @@ noinst_HEADERS = \
 	detect-engine-file.h \
 	detect-engine-frame.h \
 	detect-engine.h \
+	detect-engine-helper.h \
 	detect-engine-iponly.h \
 	detect-engine-loader.h \
 	detect-engine-mpm.h \
@@ -758,6 +759,7 @@ libsuricata_c_a_SOURCES = \
 	detect-engine-event.c \
 	detect-engine-file.c \
 	detect-engine-frame.c \
+	detect-engine-helper.c \
 	detect-engine-iponly.c \
 	detect-engine-loader.c \
 	detect-engine-mpm.c \
@@ -1320,7 +1322,13 @@ suricata_SOURCES = main.c
 
 # the library search path.
 suricata_LDFLAGS = $(all_libraries) ${SECLDFLAGS}
-suricata_LDADD = libsuricata_c.a $(RUST_SURICATA_LIB) $(HTP_LDADD) $(RUST_LDADD)
+# rust library depends also on c
+if LINKER_SUPPORTS_GROUP
+LDADD_GENERIC = "-Wl,--start-group,libsuricata_c.a,$(RUST_SURICATA_LIB),--end-group" $(HTP_LDADD) $(RUST_LDADD)
+else
+LDADD_GENERIC = libsuricata_c.a $(RUST_SURICATA_LIB) libsuricata_c.a $(RUST_SURICATA_LIB) $(HTP_LDADD) $(RUST_LDADD)
+endif
+suricata_LDADD = $(LDADD_GENERIC)
 suricata_DEPENDENCIES = libsuricata_c.a $(RUST_SURICATA_LIB)
 
 if BUILD_SHARED_LIBRARY
@@ -1356,7 +1364,7 @@ uninstall-local:
 if BUILD_FUZZTARGETS
 
 LDFLAGS_FUZZ = $(all_libraries) $(SECLDFLAGS)
-LDADD_FUZZ = libsuricata_c.a $(RUST_SURICATA_LIB) $(HTP_LDADD) $(RUST_LDADD)
+LDADD_FUZZ = $(LDADD_GENERIC)
 
 fuzz_applayerprotodetectgetproto_SOURCES = tests/fuzz/fuzz_applayerprotodetectgetproto.c
 fuzz_applayerprotodetectgetproto_LDFLAGS = $(LDFLAGS_FUZZ)

src/detect-engine-helper.c

@@ -0,0 +1,107 @@
+/* Copyright (C) 2023 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Philippe Antoine <[email protected]>
+ *
+ */
+
+#include "suricata-common.h"
+#include "detect-engine.h"
+#include "detect-engine-helper.h"
+#include "detect-engine-mpm.h"
+#include "detect-engine-prefilter.h"
+#include "detect-parse.h"
+
+int DetectHelperBufferRegister(const char *name, AppProto alproto, bool toclient, bool toserver)
+{
+    if (toserver) {
+        DetectAppLayerInspectEngineRegister(
+                name, alproto, SIG_FLAG_TOSERVER, 0, DetectEngineInspectGenericList, NULL);
+    }
+    if (toclient) {
+        DetectAppLayerInspectEngineRegister(
+                name, alproto, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectGenericList, NULL);
+    }
+    return DetectBufferTypeRegister(name);
+}
+
+InspectionBuffer *DetectHelperGetData(struct DetectEngineThreadCtx_ *det_ctx,
+        const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
+        const int list_id,
+        bool (*GetBuf)(void *txv, const uint8_t flow_flags, const uint8_t **buf, uint32_t *buf_len))
+{
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
+    if (buffer->inspect == NULL) {
+        const uint8_t *b = NULL;
+        uint32_t b_len = 0;
+
+        if (!GetBuf(txv, flow_flags, &b, &b_len))
+            return NULL;
+
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
+        InspectionBufferApplyTransforms(buffer, transforms);
+    }
+    return buffer;
+}
+
+int DetectHelperBufferMpmRegister(const char *name, const char *desc, AppProto alproto,
+        bool toclient, bool toserver, InspectionBufferGetDataPtr GetData)
+{
+    if (toserver) {
+        DetectAppLayerInspectEngineRegister(
+                name, alproto, SIG_FLAG_TOSERVER, 0, DetectEngineInspectBufferGeneric, GetData);
+        DetectAppLayerMpmRegister(
+                name, SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister, GetData, alproto, 0);
+    }
+    if (toclient) {
+        DetectAppLayerInspectEngineRegister(
+                name, alproto, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectBufferGeneric, GetData);
+        DetectAppLayerMpmRegister(
+                name, SIG_FLAG_TOCLIENT, 2, PrefilterGenericMpmRegister, GetData, alproto, 0);
+    }
+    DetectBufferTypeSetDescriptionByName(name, desc);
+    return DetectBufferTypeGetByName(name);
+}
+
+int DetectHelperKeywordRegister(const SCSigTableElmt *kw)
+{
+    if (DETECT_TBLSIZE_IDX >= DETECT_TBLSIZE) {
+        void *tmp = SCRealloc(
+                sigmatch_table, (DETECT_TBLSIZE + DETECT_TBLSIZE_STEP) * sizeof(SigTableElmt));
+        if (unlikely(tmp == NULL)) {
+            return -1;
+        }
+        sigmatch_table = tmp;
+        DETECT_TBLSIZE += DETECT_TBLSIZE_STEP;
+    }
+
+    sigmatch_table[DETECT_TBLSIZE_IDX].name = kw->name;
+    sigmatch_table[DETECT_TBLSIZE_IDX].desc = kw->desc;
+    sigmatch_table[DETECT_TBLSIZE_IDX].url = kw->url;
+    sigmatch_table[DETECT_TBLSIZE_IDX].flags = kw->flags;
+    sigmatch_table[DETECT_TBLSIZE_IDX].AppLayerTxMatch =
+            (int (*)(DetectEngineThreadCtx * det_ctx, Flow * f, uint8_t flags, void *alstate,
+                    void *txv, const Signature *s, const SigMatchCtx *ctx)) kw->AppLayerTxMatch;
+    sigmatch_table[DETECT_TBLSIZE_IDX].Setup =
+            (int (*)(DetectEngineCtx * de, Signature * s, const char *raw)) kw->Setup;
+    sigmatch_table[DETECT_TBLSIZE_IDX].Free = (void (*)(DetectEngineCtx * de, void *ptr)) kw->Free;
+    DETECT_TBLSIZE_IDX++;
+    return DETECT_TBLSIZE_IDX - 1;
+}

src/detect-engine-helper.h

@@ -0,0 +1,41 @@
+/* Copyright (C) 2023 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Philippe Antoine <[email protected]>
+ */
+
+#ifndef SURICATA_DETECT_ENGINE_HELPER_H
+#define SURICATA_DETECT_ENGINE_HELPER_H
+
+#include "app-layer-protos.h"
+#include "detect.h"
+#include "rust.h"
+
+int DetectHelperKeywordRegister(const SCSigTableElmt *kw);
+int DetectHelperBufferRegister(const char *name, AppProto alproto, bool toclient, bool toserver);
+
+typedef bool (*SimpleGetTxBuffer)(void *, uint8_t, const uint8_t **, uint32_t *);
+InspectionBuffer *DetectHelperGetData(struct DetectEngineThreadCtx_ *det_ctx,
+        const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
+        const int list_id, SimpleGetTxBuffer GetBuf);
+int DetectHelperBufferMpmRegister(const char *name, const char *desc, AppProto alproto,
+        bool toclient, bool toserver, InspectionBufferGetDataPtr GetData);
+
+#endif /* SURICATA_DETECT_ENGINE_HELPER_H */

src/detect-engine-register.c

@@ -340,6 +340,9 @@
 #include "util-mpm-ac.h"
 #include "runmodes.h"
 
+int DETECT_TBLSIZE = 0;
+int DETECT_TBLSIZE_IDX = DETECT_TBLSIZE_STATIC;
+
 static void PrintFeatureList(const SigTableElmt *e, char sep)
 {
     const uint16_t flags = e->flags;
@@ -408,7 +411,7 @@ static void SigMultilinePrint(int i, const char *prefix)
 
 int SigTableList(const char *keyword)
 {
-    size_t size = sizeof(sigmatch_table) / sizeof(SigTableElmt);
+    size_t size = DETECT_TBLSIZE;
     size_t i;
 
     if (keyword == NULL) {
@@ -483,15 +486,32 @@ int SigTableList(const char *keyword)
 
 static void DetectFileHandlerRegister(void)
 {
-    for (int i = 0; i < DETECT_TBLSIZE; i++) {
+    for (int i = 0; i < DETECT_TBLSIZE_STATIC; i++) {
         if (filehandler_table[i].name)
             DetectFileRegisterFileProtocols(&filehandler_table[i]);
     }
 }
 
+void SigTableCleanup(void)
+{
+    if (sigmatch_table != NULL) {
+        SCFree(sigmatch_table);
+        sigmatch_table = NULL;
+        DETECT_TBLSIZE = 0;
+    }
+}
+
 void SigTableSetup(void)
 {
-    memset(sigmatch_table, 0, sizeof(sigmatch_table));
+    if (sigmatch_table == NULL) {
+        DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC + DETECT_TBLSIZE_STEP;
+        sigmatch_table = SCCalloc(DETECT_TBLSIZE, sizeof(SigTableElmt));
+        if (sigmatch_table == NULL) {
+            DETECT_TBLSIZE = 0;
+            FatalError("Could not allocate sigmatch_table");
+            return;
+        }
+    }
 
     DetectSidRegister();
     DetectPriorityRegister();

src/detect-engine-register.h

@@ -374,10 +374,15 @@ enum DetectKeywordId {
     DETECT_AL_JA4_HASH,
 
     /* make sure this stays last */
-    DETECT_TBLSIZE,
+    DETECT_TBLSIZE_STATIC,
 };
 
+extern int DETECT_TBLSIZE;
+extern int DETECT_TBLSIZE_IDX;
+// step for reallocating sigmatch_table
+#define DETECT_TBLSIZE_STEP 256
 int SigTableList(const char *keyword);
+void SigTableCleanup(void);
 void SigTableSetup(void);
 void SigTableRegisterTests(void);
 

src/detect-engine.c

@@ -2493,6 +2493,14 @@ static DetectEngineCtx *DetectEngineCtxInitReal(
         goto error;
     }
 
+    de_ctx->sm_types_prefilter = SCCalloc(DETECT_TBLSIZE, sizeof(bool));
+    if (de_ctx->sm_types_prefilter == NULL) {
+        goto error;
+    }
+    de_ctx->sm_types_silent_error = SCCalloc(DETECT_TBLSIZE, sizeof(bool));
+    if (de_ctx->sm_types_silent_error == NULL) {
+        goto error;
+    }
     if (DetectEngineCtxLoadConf(de_ctx) == -1) {
         goto error;
     }
@@ -2625,6 +2633,8 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     SigGroupCleanup(de_ctx);
 
     SpmDestroyGlobalThreadCtx(de_ctx->spm_global_thread_ctx);
+    SCFree(de_ctx->sm_types_prefilter);
+    SCFree(de_ctx->sm_types_silent_error);
 
     MpmFactoryDeRegisterAllMpmCtxProfiles(de_ctx);