lib: user provided threads and packets - v9

.gitignore

@@ -69,3 +69,9 @@ test.sh
 /libsuricata-config
 !/libsuricata-config.in
 !/.readthedocs.yaml
+
+# Man pages.
+doc/userguide/*.1
+
+# Generated compile commands for LSP.
+/compile_commands.json

Makefile.am

@@ -14,7 +14,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \
 	     examples/plugins
 SUBDIRS = $(HTP_DIR) rust src plugins qa rules doc contrib etc python ebpf \
           $(SURICATA_UPDATE_DIR)
-DIST_SUBDIRS = $(SUBDIRS) examples/lib/simple
+DIST_SUBDIRS = $(SUBDIRS) examples/lib/simple examples/lib/libcapture
 
 CLEANFILES = stamp-h[0-9]*
 

configure.ac

@@ -2527,6 +2527,7 @@ AC_CONFIG_FILES(examples/plugins/c-json-filetype/Makefile)
 AC_CONFIG_FILES(examples/plugins/c-custom-loggers/Makefile)
 AC_CONFIG_FILES(examples/plugins/ci-capture/Makefile)
 AC_CONFIG_FILES(examples/lib/simple/Makefile examples/lib/simple/Makefile.example)
+AC_CONFIG_FILES(examples/lib/libcapture/Makefile examples/lib/libcapture/Makefile.example)
 AC_CONFIG_FILES(plugins/Makefile)
 AC_CONFIG_FILES(plugins/pfring/Makefile)
 AC_CONFIG_FILES(plugins/napatech/Makefile)

examples/lib/libcapture/.gitignore

@@ -0,0 +1,3 @@
+!/Makefile.example.in
+Makefile.example
+/libcapture

examples/lib/libcapture/Makefile.am

@@ -0,0 +1,12 @@
+bin_PROGRAMS = libcapture
+
+libcapture_SOURCES = main.c
+
+AM_CPPFLAGS = -I$(top_srcdir)/src
+
+libcapture_LDFLAGS = $(all_libraries) $(SECLDFLAGS)
+libcapture_LDADD = $(top_builddir)/src/libsuricata_c.a ../../$(RUST_SURICATA_LIB) $(RUST_LDADD)
+if HTP_LDADD
+libcapture_LDADD += ../../$(HTP_LDADD)
+endif
+libcapture_DEPENDENCIES = $(top_builddir)/src/libsuricata_c.a ../../$(RUST_SURICATA_LIB)

examples/lib/libcapture/Makefile.example.in

@@ -0,0 +1,16 @@
+LIBSURICATA_CONFIG ?=	@CONFIGURE_PREFIX@/bin/libsuricata-config
+
+SURICATA_LIBS =		`$(LIBSURICATA_CONFIG) --libs --static`
+SURICATA_CFLAGS :=	`$(LIBSURICATA_CONFIG) --cflags`
+
+# Currently the Suricata logging system requires this to be even for
+# plugins.
+CPPFLAGS +=    "-D__SCFILENAME__=\"$(*F)\""
+
+all: libcapture
+
+libcapture: main.c
+	$(CC) -o $@ $^ $(CPPFLAGS) $(CFLAGS) $(SURICATA_CFLAGS) $(SURICATA_LIBS)
+
+clean:
+	rm -f libcapture

examples/lib/libcapture/README.md

@@ -0,0 +1,52 @@
+# LibCapture Example
+
+This is an example of using the Suricata library with the "lib"
+capture method. The "lib" capture method is a simpler way of providing
+packets to Suricata if you need to drive the deliver of packets,
+rather than having Suricata run the main loop.
+
+## Building In Tree
+
+The Suricata build system has created a Makefile that should allow you
+to build this application in-tree on most supported platforms. To
+build simply run:
+
+```
+make
+```
+
+## Running
+
+```
+./libcapture -l . -- filename.pcap
+```
+
+For this example, any arguments before `--` are passed directly as
+Suricata command line arguments. Arguments after the first `--` are
+handled by this example program, and currently the only argument is a
+PCAP filename to be read.
+
+## Building Out of Tree
+
+A Makefile.example has also been generated to use as an example on how
+to build against the library in a standalone application.
+
+First build and install the Suricata library including:
+
+```
+make install-library
+make install-headers
+```
+
+Then run:
+
+```
+make -f Makefile.example
+```
+
+If you installed to a non-standard location, you need to ensure that
+`libsuricata-config` is in your path, for example:
+
+```
+PATH=/opt/suricata/bin:$PATH make -f Makefile.example
+```

examples/lib/libcapture/main.c

@@ -0,0 +1,239 @@
+/* Copyright (C) 2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+#include "suricata.h"
+#include "runmodes.h"
+#include "conf.h"
+#include "pcap.h"
+#include "runmode-lib.h"
+#include "tm-threads.h"
+#include "threadvars.h"
+#include "action-globals.h"
+#include "packet.h"
+
+static int worker_id = 1;
+
+/**
+ * Struct to pass arguments into a worker thread.
+ */
+struct WorkerArgs {
+    ThreadVars *tv;
+    char *pcap_filename;
+};
+
+/**
+ * Release packet callback.
+ *
+ * If there is any cleanup that needs to be done when Suricata is done
+ * with a packet, this is the place to do it.
+ *
+ * Important: If using a custom release function, you must also
+ * release or free the packet.
+ *
+ * Optionally this is where you would handle IPS like functionality
+ * such as forwarding the packet, or triggering some other mechanism
+ * to forward the packet.
+ */
+static void ReleasePacket(Packet *p)
+{
+    if (PacketCheckAction(p, ACTION_DROP)) {
+        SCLogNotice("Dropping packet!");
+    }
+
+    /* As we overode the default release function, we must release or
+     * free the packet. */
+    PacketFreeOrRelease(p);
+}
+
+/* Suricata worker thread in library mode.
+   The functions should be wrapped in an API layer. */
+static void *SimpleWorker(void *arg)
+{
+    struct WorkerArgs *args = arg;
+    ThreadVars *tv = args->tv;
+
+    /* Start worker. */
+    if (SCRunModeLibSpawnWorker(tv) != 0) {
+        pthread_exit(NULL);
+    }
+
+    /* Replay pcap. */
+    pcap_t *fp = pcap_open_offline(args->pcap_filename, NULL);
+    if (fp == NULL) {
+        pthread_exit(NULL);
+    }
+
+    LiveDevice *device = LiveGetDevice("lib0");
+    assert(device != NULL);
+
+    int datalink = pcap_datalink(fp);
+    int count = 0;
+    struct pcap_pkthdr pkthdr;
+    const u_char *packet;
+    while ((packet = pcap_next(fp, &pkthdr)) != NULL) {
+
+        /* Have we been asked to stop? */
+        if (suricata_ctl_flags & SURICATA_STOP) {
+            goto done;
+        }
+
+        Packet *p = PacketGetFromQueueOrAlloc();
+        if (unlikely(p == NULL)) {
+            /* Memory allocation error. */
+            goto done;
+        }
+
+        /* If we are processing a PCAP and it is the first packet we need to set the timestamp. */
+        SCTime_t timestamp = SCTIME_FROM_TIMEVAL(&pkthdr.ts);
+        if (count == 0) {
+            TmThreadsInitThreadsTimestamp(timestamp);
+        }
+
+        /* Setup the packet, these will become functions to avoid
+         * internal Packet access. */
+        SCPacketSetSource(p, PKT_SRC_WIRE);
+        SCPacketSetTime(p, SCTIME_FROM_TIMEVAL(&pkthdr.ts));
+        SCPacketSetDatalink(p, datalink);
+        SCPacketSetLiveDevice(p, device);
+        SCPacketSetReleasePacket(p, ReleasePacket);
+
+        if (PacketSetData(p, packet, pkthdr.len) == -1) {
+            TmqhOutputPacketpool(tv, p);
+            goto done;
+        }
+
+        if (TmThreadsSlotProcessPkt(tv, tv->tm_slots, p) != TM_ECODE_OK) {
+            TmqhOutputPacketpool(tv, p);
+            goto done;
+        }
+
+        (void)SC_ATOMIC_ADD(device->pkts, 1);
+        count++;
+    }
+
+done:
+    pcap_close(fp);
+
+    /* Stop the engine. */
+    EngineStop();
+
+    /* Cleanup.
+     *
+     * Note that there is some thread synchronization between this
+     * function and SuricataShutdown such that they must be run
+     * concurrently at this time before either will exit. */
+    SCTmThreadsSlotPktAcqLoopFinish(tv);
+
+    SCLogNotice("Worker thread exiting");
+    pthread_exit(NULL);
+}
+
+int main(int argc, char **argv)
+{
+    SuricataPreInit(argv[0]);
+
+    /* Parse command line options. This is optional, you could
+     * directly configure Suricata through the Conf API.
+       The last argument is the PCAP file to replay. */
+    SCParseCommandLine(argc, argv);
+
+    /* Find our list of pcap files, after the "--". */
+    while (argc) {
+        bool end = strncmp(argv[0], "--", 2) == 0;
+        argv++;
+        argc--;
+        if (end) {
+            break;
+        }
+    }
+    if (argc == 0) {
+        fprintf(stderr, "ERROR: No PCAP files provided\n");
+        return 1;
+    }
+
+    /* Set the runmode to library mode. Perhaps in the future this
+       should be done in some library bootstrap function. */
+    SCRunmodeSet(RUNMODE_LIB);
+
+    /* Validate/finalize the runmode. */
+    if (SCFinalizeRunMode() != TM_ECODE_OK) {
+        exit(EXIT_FAILURE);
+    }
+
+    /* Handle internal runmodes. Typically you wouldn't do this as a
+     * library user, however this example is showing how to replicate
+     * the Suricata application with the library. */
+    switch (SCStartInternalRunMode(argc, argv)) {
+        case TM_ECODE_DONE:
+            exit(EXIT_SUCCESS);
+        case TM_ECODE_FAILED:
+            exit(EXIT_FAILURE);
+    }
+
+    /* Load configuration file, could be done earlier but must be done
+     * before SuricataInit, but even then its still optional as you
+     * may be programmatically configuration Suricata. */
+    if (SCLoadYamlConfig() != TM_ECODE_OK) {
+        exit(EXIT_FAILURE);
+    }
+
+    /* Set "offline" runmode to replay a pcap in library mode. */
+    if (!ConfSetFromString("runmode=offline", 1)) {
+        exit(EXIT_FAILURE);
+    }
+
+    if (LiveRegisterDevice("lib0") < 0) {
+        fprintf(stderr, "LiveRegisterDevice failed");
+        exit(1);
+    }
+
+    SuricataInit();
+
+    /* Create and start worker on its own thread, passing the PCAP file
+       as argument. This needs to be done in between SuricataInit and
+       SuricataPostInit. */
+    pthread_t worker;
+    ThreadVars *tv = SCRunModeLibCreateThreadVars(worker_id++);
+    if (!tv) {
+        FatalError("Failed to create ThreadVars");
+    }
+    struct WorkerArgs args = {
+        .tv = tv,
+        .pcap_filename = argv[argc - 1],
+    };
+    if (pthread_create(&worker, NULL, SimpleWorker, &args) != 0) {
+        exit(EXIT_FAILURE);
+    }
+
+    SuricataPostInit();
+
+    /* Run the main loop, this just waits for the worker thread to
+     * call EngineStop signalling Suricata that it is done reading the
+     * pcap. */
+    SuricataMainLoop();
+
+    /* Shutdown engine. */
+    SCLogNotice("Shutting down");
+
+    /* Note that there is some thread synchronization between this
+     * function and SCTmThreadsSlotPktAcqLoopFinish that require them
+     * to be run concurrently at this time. */
+    SuricataShutdown();
+    GlobalsDestroy();
+
+    return EXIT_SUCCESS;
+}

examples/lib/simple/.gitignore

@@ -1,2 +1,3 @@
 !/Makefile.example.in
 Makefile.example
+/simple

examples/plugins/ci-capture/source.c

@@ -20,6 +20,7 @@
 #include "tm-modules.h"
 #include "tm-threads-common.h"
 #include "tm-threads.h"
+#include "packet.h"
 
 #include "source.h"
 
@@ -69,11 +70,11 @@ static TmEcode ReceiveLoop(ThreadVars *tv, void *data, void *slot)
     if (unlikely(p == NULL)) {
         return TM_ECODE_FAILED;
     }
-    PKT_SET_SRC(p, PKT_SRC_WIRE);
+    SCPacketSetSource(p, PKT_SRC_WIRE);
     struct timeval now;
     gettimeofday(&now, NULL);
-    p->ts = SCTIME_FROM_TIMEVAL(&now);
-    p->datalink = LINKTYPE_ETHERNET;
+    SCPacketSetTime(p, SCTIME_FROM_TIMEVAL(&now));
+    SCPacketSetDatalink(p, LINKTYPE_ETHERNET);
     p->flags |= PKT_IGNORE_CHECKSUM;
 
     if (unlikely(PacketCopyData(p, DNS_REQUEST, sizeof(DNS_REQUEST)) != 0)) {