From ab42eaa4d062cb964afb87806e70b0522aca47d5 Mon Sep 17 00:00:00 2001 From: Angelo Mirabella Date: Wed, 3 Apr 2024 01:06:39 -0700 Subject: [PATCH 01/20] libsuricata: add library runmode Add library source and runmode modules. Reorganized library example to create a worker thread and replay a pcap file using the library mode. No API layer is added at this stage. Edits by Jason Ish: - fix guard - add copyright/license headers --- examples/lib/simple/Makefile.example.in | 8 +- examples/lib/simple/main.c | 75 +++++++++- src/Makefile.am | 4 + src/runmode-lib.c | 154 ++++++++++++++++++++ src/runmode-lib.h | 57 ++++++++ src/runmodes.c | 7 + src/runmodes.h | 1 + src/source-lib.c | 180 ++++++++++++++++++++++++ src/source-lib.h | 51 +++++++ src/suricata.c | 9 ++ src/suricata.h | 3 + src/tm-modules.c | 1 + src/tm-threads-common.h | 1 + src/tm-threads.c | 106 +++++++++++++- src/tm-threads.h | 2 + 15 files changed, 650 insertions(+), 9 deletions(-) create mode 100644 src/runmode-lib.c create mode 100644 src/runmode-lib.h create mode 100644 src/source-lib.c create mode 100644 src/source-lib.h diff --git a/examples/lib/simple/Makefile.example.in b/examples/lib/simple/Makefile.example.in index 724beccdb140..c9da97560e4d 100644 --- a/examples/lib/simple/Makefile.example.in +++ b/examples/lib/simple/Makefile.example.in @@ -1,12 +1,16 @@ LIBSURICATA_CONFIG ?= @CONFIGURE_PREFIX@/bin/libsuricata-config -SURICATA_LIBS = `$(LIBSURICATA_CONFIG) --libs` +SURICATA_LIBS = `$(LIBSURICATA_CONFIG) --libs --static` SURICATA_CFLAGS := `$(LIBSURICATA_CONFIG) --cflags` +# Currently the Suricata logging system requires this to be even for +# plugins. +CPPFLAGS += "-D__SCFILENAME__=\"$(*F)\"" + all: simple simple: main.c - $(CC) -o $@ $^ $(CFLAGS) $(SURICATA_CFLAGS) $(SURICATA_LIBS) + $(CC) -o $@ $^ $(CPPFLAGS) $(CFLAGS) $(SURICATA_CFLAGS) $(SURICATA_LIBS) clean: rm -f simple diff --git a/examples/lib/simple/main.c b/examples/lib/simple/main.c index f9c09fb0f550..7e6900e5f40c 100644 --- a/examples/lib/simple/main.c +++ b/examples/lib/simple/main.c @@ -16,14 +16,62 @@ */ #include "suricata.h" +#include "conf.h" +#include "pcap.h" +#include "runmode-lib.h" +#include "source-lib.h" +#include "threadvars.h" + +/* Suricata worker thread in library mode. + The functions should be wrapped in an API layer. */ +static void *SimpleWorker(void *arg) +{ + char *pcap_file = (char *)arg; + + /* Create worker. */ + ThreadVars *tv = RunModeCreateWorker(); + if (!tv) { + pthread_exit(NULL); + } + + /* Start worker. */ + if (RunModeSpawnWorker(tv) != 0) { + pthread_exit(NULL); + } + + /* Replay pcap. */ + pcap_t *fp = pcap_open_offline(pcap_file, NULL); + if (fp == NULL) { + pthread_exit(NULL); + } + + int datalink = pcap_datalink(fp); + struct pcap_pkthdr pkthdr; + const u_char *packet; + while ((packet = pcap_next(fp, &pkthdr)) != NULL) { + if (TmModuleLibHandlePacket(tv, packet, datalink, pkthdr.ts, pkthdr.len, 0, 0, NULL) != 0) { + pthread_exit(NULL); + } + } + pcap_close(fp); + + /* Cleanup. */ + RunModeDestroyWorker(tv); + pthread_exit(NULL); +} int main(int argc, char **argv) { SuricataPreInit(argv[0]); /* Parse command line options. This is optional, you could - * directly configure Suricata through the Conf API. */ - SCParseCommandLine(argc, argv); + * directly configure Suricata through the Conf API. + The last argument is the PCAP file to replay. */ + SCParseCommandLine(argc - 1, argv); + + /* Set lib runmode. There is currently no way to set it via + the Conf API. */ + SuricataSetLibRunmode(); /* Validate/finalize the runmode. */ if (SCFinalizeRunMode() != TM_ECODE_OK) { @@ -47,12 +95,27 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } + /* Set "offline" runmode to replay a pcap in library mode. */ + if (!ConfSetFromString("runmode=offline", 1)) { + exit(EXIT_FAILURE); + } + SuricataInit(); - SuricataPostInit(); - /* Suricata is now running, but we enter a loop to keep it running - * until it shouldn't be running anymore. */ - SuricataMainLoop(); + /* Create and start worker on its own thread, passing the PCAP file + as argument. This needs to be done in between SuricataInit and + SuricataPostInit. */ + pthread_t worker; + if (pthread_create(&worker, NULL, SimpleWorker, argv[argc - 1]) != 0) { + exit(EXIT_FAILURE); + } + + /* Need to introduce a little sleep to allow the worker thread to + initialize before SuricataPostInit invokes TmThreadContinueThreads(). + This should be handle at the API level. */ + usleep(100); + + SuricataPostInit(); /* Shutdown engine. */ SuricataShutdown(); diff --git a/src/Makefile.am b/src/Makefile.am index 615816953542..ed2f9163db97 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -400,6 +400,7 @@ noinst_HEADERS = \ runmode-erf-dag.h \ runmode-erf-file.h \ runmode-ipfw.h \ + runmode-lib.h \ runmode-netmap.h \ runmode-nflog.h \ runmode-nfq.h \ @@ -417,6 +418,7 @@ noinst_HEADERS = \ source-erf-dag.h \ source-erf-file.h \ source-ipfw.h \ + source-lib.h \ source-netmap.h \ source-nflog.h \ source-nfq.h \ @@ -964,6 +966,7 @@ libsuricata_c_a_SOURCES = \ runmode-erf-dag.c \ runmode-erf-file.c \ runmode-ipfw.c \ + runmode-lib.c \ runmode-netmap.c \ runmode-nflog.c \ runmode-nfq.c \ @@ -980,6 +983,7 @@ libsuricata_c_a_SOURCES = \ source-erf-dag.c \ source-erf-file.c \ source-ipfw.c \ + source-lib.c \ source-netmap.c \ source-nflog.c \ source-nfq.c \ diff --git a/src/runmode-lib.c b/src/runmode-lib.c new file mode 100644 index 000000000000..1441dabe4bcb --- /dev/null +++ b/src/runmode-lib.c @@ -0,0 +1,154 @@ +/* Copyright (C) 2023-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Angelo Mirabella + * + * Library runmode. + */ +#include "suricata-common.h" +#include "runmode-lib.h" +#include "runmodes.h" +#include "tm-threads.h" +#include "util-device.h" + +static int g_thread_id = 0; + +/** \brief register runmodes for suricata as a library */ +void RunModeIdsLibRegister(void) +{ + RunModeRegisterNewRunMode(RUNMODE_LIB, "offline", "Library offline mode (pcap replaying)", + RunModeIdsLibOffline, NULL); + RunModeRegisterNewRunMode(RUNMODE_LIB, "live", "Library live mode", RunModeIdsLibLive, NULL); + return; +} + +/** \brief runmode for offline packet processing (pcap files) */ +int RunModeIdsLibOffline(void) +{ + TimeModeSetOffline(); + + return 0; +} + +/** \brief runmode for live packet processing */ +int RunModeIdsLibLive(void) +{ + TimeModeSetLive(); + + return 0; +} + +const char *RunModeLibGetDefaultMode(void) +{ + return "live"; +} + +/** \brief create a "fake" worker thread in charge of processing the packets. + * + * This method just creates a context representing the worker, which is handled from the library + * client. No actual thread (pthread_t) is created. + * + * \return Pointer to ThreadVars structure representing the worker thread */ +void *RunModeCreateWorker(void) +{ + char tname[TM_THREAD_NAME_MAX]; + TmModule *tm_module = NULL; + snprintf(tname, sizeof(tname), "%s#%02d", thread_name_workers, ++g_thread_id); + + ThreadVars *tv = TmThreadCreatePacketHandler( + tname, "packetpool", "packetpool", "packetpool", "packetpool", "lib"); + if (tv == NULL) { + SCLogError("TmThreadsCreate failed"); + return NULL; + } + + tm_module = TmModuleGetByName("DecodeLib"); + if (tm_module == NULL) { + SCLogError("TmModuleGetByName DecodeLib failed"); + return NULL; + } + TmSlotSetFuncAppend(tv, tm_module, NULL); + + tm_module = TmModuleGetByName("FlowWorker"); + if (tm_module == NULL) { + SCLogError("TmModuleGetByName for FlowWorker failed"); + return NULL; + } + TmSlotSetFuncAppend(tv, tm_module, NULL); + + TmThreadAppend(tv, tv->type); + + return tv; +} + +/** \brief start the "fake" worker. + * + * This method performs all the initialization tasks. + */ +int RunModeSpawnWorker(void *td) +{ + ThreadVars *tv = (ThreadVars *)td; + + if (TmThreadLibSpawn(tv) != TM_ECODE_OK) { + SCLogError("TmThreadLibSpawn failed"); + return -1; + } + + TmThreadsSetFlag(tv, THV_RUNNING); + return 0; +} + +/** \brief destroy a worker thread */ +void RunModeDestroyWorker(void *td) +{ + ThreadVars *tv = (ThreadVars *)td; + TmSlot *s = tv->tm_slots; + TmEcode r; + TmSlot *slot = NULL; + + StatsSyncCounters(tv); + + TmThreadsSetFlag(tv, THV_FLOW_LOOP); + + /* process all pseudo packets the flow timeout may throw at us */ + TmThreadTimeoutLoop(tv, s); + + TmThreadsSetFlag(tv, THV_RUNNING_DONE); + TmThreadWaitForFlag(tv, THV_DEINIT); + + PacketPoolDestroy(); + + for (slot = s; slot != NULL; slot = slot->slot_next) { + if (slot->SlotThreadExitPrintStats != NULL) { + slot->SlotThreadExitPrintStats(tv, SC_ATOMIC_GET(slot->slot_data)); + } + + if (slot->SlotThreadDeinit != NULL) { + r = slot->SlotThreadDeinit(tv, SC_ATOMIC_GET(slot->slot_data)); + if (r != TM_ECODE_OK) { + break; + } + } + } + + tv->stream_pq = NULL; + --g_thread_id; + SCLogDebug("%s ending", tv->name); + TmThreadsSetFlag(tv, THV_CLOSED); +} diff --git a/src/runmode-lib.h b/src/runmode-lib.h new file mode 100644 index 000000000000..c33cb4e65bf3 --- /dev/null +++ b/src/runmode-lib.h @@ -0,0 +1,57 @@ +/* Copyright (C) 2023-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Angelo Mirabella + * + * Library runmode. + */ + +#ifndef SURICATA_RUNMODE_LIB_H +#define SURICATA_RUNMODE_LIB_H + +/** \brief register runmodes for suricata as a library */ +void RunModeIdsLibRegister(void); + +/** \brief runmode for live packet processing */ +int RunModeIdsLibLive(void); + +/** \brief runmode for offline packet processing (pcap files) */ +int RunModeIdsLibOffline(void); + +/** \brief runmode default mode (live) */ +const char *RunModeLibGetDefaultMode(void); + +/** \brief create a "fake" worker thread in charge of processing the packets. + * + * This method just creates a context representing the worker, which is handled from the library + * client. No actual thread (pthread_t) is created. + * + * \return Pointer to ThreadVars structure representing the worker thread */ +void *RunModeCreateWorker(void); + +/** \brief start the "fake" worker. + * + * This method performs all the initialization tasks. + */ +int RunModeSpawnWorker(void *); + +/** \brief destroy a worker thread */ +void RunModeDestroyWorker(void *); + +#endif /* SURICATA_RUNMODE_LIB_H */ diff --git a/src/runmodes.c b/src/runmodes.c index dd322cf7daf2..beea5114e9fc 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -35,6 +35,7 @@ #include "runmode-erf-dag.h" #include "runmode-erf-file.h" #include "runmode-ipfw.h" +#include "runmode-lib.h" #include "runmode-netmap.h" #include "runmode-nflog.h" #include "runmode-nfq.h" @@ -158,6 +159,8 @@ static const char *RunModeTranslateModeToName(int runmode) #else return "DPDK(DISABLED)"; #endif + case RUNMODE_LIB: + return "LIB"; default: FatalError("Unknown runtime mode. Aborting"); @@ -230,6 +233,7 @@ void RunModeRegisterRunModes(void) RunModeUnixSocketRegister(); RunModeIpsWinDivertRegister(); RunModeDpdkRegister(); + RunModeIdsLibRegister(); #ifdef UNITTESTS UtRunModeRegister(); #endif @@ -342,6 +346,9 @@ static const char *RunModeGetConfOrDefault(int capture_mode, const char *capture custom_mode = RunModeDpdkGetDefaultMode(); break; #endif + case RUNMODE_LIB: + custom_mode = RunModeLibGetDefaultMode(); + break; default: return NULL; } diff --git a/src/runmodes.h b/src/runmodes.h index cce5fcbbaa42..70148a20c52d 100644 --- a/src/runmodes.h +++ b/src/runmodes.h @@ -37,6 +37,7 @@ enum RunModes { RUNMODE_AFXDP_DEV, RUNMODE_NETMAP, RUNMODE_DPDK, + RUNMODE_LIB, RUNMODE_UNITTEST, RUNMODE_UNIX_SOCKET, RUNMODE_WINDIVERT, diff --git a/src/source-lib.c b/src/source-lib.c new file mode 100644 index 000000000000..6c304fc30069 --- /dev/null +++ b/src/source-lib.c @@ -0,0 +1,180 @@ +/* Copyright (C) 2023-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Angelo Mirabella + * + * LIB packet and stream decoding support + * + */ + +#include "suricata-common.h" +#include "source-lib.h" +#include "util-device.h" + +static TmEcode DecodeLibThreadInit(ThreadVars *tv, const void *initdata, void **data); +static TmEcode DecodeLibThreadDeinit(ThreadVars *tv, void *data); +static TmEcode DecodeLib(ThreadVars *tv, Packet *p, void *data); + +/* Set time to the first packet timestamp when replaying a PCAP. */ +static bool time_set = false; + +/** \brief register a "Decode" module for suricata as a library. + * + * The "Decode" module is the first module invoked when processing a packet */ +void TmModuleDecodeLibRegister(void) +{ + tmm_modules[TMM_DECODELIB].name = "DecodeLib"; + tmm_modules[TMM_DECODELIB].ThreadInit = DecodeLibThreadInit; + tmm_modules[TMM_DECODELIB].Func = DecodeLib; + tmm_modules[TMM_DECODELIB].ThreadExitPrintStats = NULL; + tmm_modules[TMM_DECODELIB].ThreadDeinit = DecodeLibThreadDeinit; + tmm_modules[TMM_DECODELIB].cap_flags = 0; + tmm_modules[TMM_DECODELIB].flags = TM_FLAG_DECODE_TM; +} + +/** \brief initialize the "Decode" module. + * + * \param tv Pointer to the per-thread structure. + * \param initdata Pointer to initialization context. + * \param data Pointer to the initialized context. + * \return Error code. + */ +TmEcode DecodeLibThreadInit(ThreadVars *tv, const void *initdata, void **data) +{ + SCEnter(); + DecodeThreadVars *dtv = NULL; + + dtv = DecodeThreadVarsAlloc(tv); + + if (dtv == NULL) + SCReturnInt(TM_ECODE_FAILED); + + DecodeRegisterPerfCounters(dtv, tv); + + *data = (void *)dtv; + + SCReturnInt(TM_ECODE_OK); +} + +/** \brief deinitialize the "Decode" module. + * + * \param tv Pointer to the per-thread structure. + * \param data Pointer to the context. + * \return Error code. + */ +TmEcode DecodeLibThreadDeinit(ThreadVars *tv, void *data) +{ + if (data != NULL) + DecodeThreadVarsFree(tv, data); + + time_set = false; + SCReturnInt(TM_ECODE_OK); +} + +/** \brief main decoding function. + * + * This method receives a packet and tries to identify layer 2 to 4 layers. + * + * \param tv Pointer to the per-thread structure. + * \param p Pointer to the packet. + * \param data Pointer to the context. + * \return Error code. + */ +TmEcode DecodeLib(ThreadVars *tv, Packet *p, void *data) +{ + SCEnter(); + DecodeThreadVars *dtv = (DecodeThreadVars *)data; + + BUG_ON(PKT_IS_PSEUDOPKT(p)); + + /* update counters */ + DecodeUpdatePacketCounters(tv, dtv, p); + + /* If suri has set vlan during reading, we increase vlan counter */ + if (p->vlan_idx) { + StatsIncr(tv, dtv->counter_vlan); + } + + /* call the decoder */ + DecodeLinkLayer(tv, dtv, p->datalink, p, GET_PKT_DATA(p), GET_PKT_LEN(p)); + + PacketDecodeFinalize(tv, dtv, p); + + SCReturnInt(TM_ECODE_OK); +} + +/** \brief process a single packet. + * + * \param tv Pointer to the per-thread structure. + * \param data Pointer to the raw packet. + * \param datalink Datalink type. + * \param ts Timeval structure. + * \param len Packet length. + * \param tenant_id Tenant id of the detection engine to use. + * \param flags Packet flags (packet checksum, rule profiling...). + * \param iface Sniffing interface this packet comes from (can be NULL). + * \return Error code. + */ +int TmModuleLibHandlePacket(ThreadVars *tv, const uint8_t *data, int datalink, struct timeval ts, + uint32_t len, uint32_t tenant_id, uint32_t flags, const char *iface) +{ + + /* If the packet is NULL, consider it as a read timeout. */ + if (data == NULL) { + TmThreadsSetFlag(tv, THV_CAPTURE_INJECT_PKT); + TmThreadsCaptureHandleTimeout(tv, NULL); + SCReturnInt(TM_ECODE_OK); + } + + Packet *p = PacketGetFromQueueOrAlloc(); + if (unlikely(p == NULL)) { + SCReturnInt(TM_ECODE_FAILED); + } + + /* If we are processing a PCAP and it is the first packet we need to set the timestamp. */ + SCTime_t timestamp = SCTIME_FROM_TIMEVAL(&ts); + if (!time_set && !TimeModeIsLive()) { + TmThreadsInitThreadsTimestamp(timestamp); + time_set = true; + } + + PKT_SET_SRC(p, PKT_SRC_WIRE); + p->ts = timestamp; + p->datalink = datalink; + p->tenant_id = tenant_id; + p->flags |= flags; + + /* Set the sniffing interface. */ + if (iface) { + p->livedev = LiveGetDevice(iface); + } + + if (PacketSetData(p, data, len) == -1) { + TmqhOutputPacketpool(tv, p); + SCReturnInt(TM_ECODE_FAILED); + } + + SCLogDebug("pktlen: %" PRIu32 " (pkt %p, pkt data %p)", GET_PKT_LEN(p), p, GET_PKT_DATA(p)); + + if (TmThreadsSlotProcessPkt(tv, tv->tm_slots, p) != TM_ECODE_OK) { + SCReturnInt(TM_ECODE_FAILED); + } + + SCReturnInt(TM_ECODE_OK); +} diff --git a/src/source-lib.h b/src/source-lib.h new file mode 100644 index 000000000000..3f0594bd8ead --- /dev/null +++ b/src/source-lib.h @@ -0,0 +1,51 @@ +/* Copyright (C) 2023-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Angelo Mirabella + * + * LIB packet and stream decoding support + * + */ + +#ifndef SURICATA_SOURCE_LIB_H +#define SURICATA_SOURCE_LIB_H + +#include "tm-threads.h" + +/** \brief register a "Decode" module for suricata as a library. + * + * The "Decode" module is the first module invoked when processing a packet */ +void TmModuleDecodeLibRegister(void); + +/** \brief process a single packet. + * + * \param tv Pointer to the per-thread structure. + * \param data Pointer to the raw packet. + * \param datalink Datalink type. + * \param ts Timeval structure. + * \param len Packet length. + * \param tenant_id Tenant id of the detection engine to use. + * \param flags Packet flags (packet checksum, rule profiling...). + * \param iface Sniffing interface this packet comes from (can be NULL). + * \return Error code. + */ +int TmModuleLibHandlePacket(ThreadVars *tv, const uint8_t *data, int datalink, struct timeval ts, + uint32_t len, uint32_t tenant_id, uint32_t flags, const char *iface); + +#endif /* SURICATA_SOURCE_LIB_H */ diff --git a/src/suricata.c b/src/suricata.c index 7c238c48cc5e..8fdc48c93fd4 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -90,6 +90,7 @@ #include "source-nfq-prototypes.h" #include "source-nflog.h" #include "source-ipfw.h" +#include "source-lib.h" #include "source-pcap.h" #include "source-pcap-file.h" #include "source-pcap-file-helper.h" @@ -949,6 +950,9 @@ void RegisterAllModules(void) /* Dpdk */ TmModuleReceiveDPDKRegister(); TmModuleDecodeDPDKRegister(); + + /* Library */ + TmModuleDecodeLibRegister(); } TmEcode SCLoadYamlConfig(void) @@ -3049,3 +3053,8 @@ void SuricataPostInit(void) } SCPledge(); } + +void SuricataSetLibRunmode(void) +{ + suricata.run_mode = RUNMODE_LIB; +} diff --git a/src/suricata.h b/src/suricata.h index 70393541f6fc..285a1b2c43e8 100644 --- a/src/suricata.h +++ b/src/suricata.h @@ -225,4 +225,7 @@ int WindowsInitService(int argc, char **argv); const char *GetProgramVersion(void); +/* Library only methods. */ +void SuricataSetLibRunmode(void); + #endif /* SURICATA_SURICATA_H */ diff --git a/src/tm-modules.c b/src/tm-modules.c index 8f9a2022e151..df3dc3c80843 100644 --- a/src/tm-modules.c +++ b/src/tm-modules.c @@ -182,6 +182,7 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE(TMM_DECODEPCAPFILE); CASE_CODE(TMM_RECEIVEDPDK); CASE_CODE(TMM_DECODEDPDK); + CASE_CODE(TMM_DECODELIB); CASE_CODE (TMM_RECEIVEPLUGIN); CASE_CODE (TMM_DECODEPLUGIN); CASE_CODE (TMM_RESPONDREJECT); diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index eb88168b025e..cd2ace674069 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -64,6 +64,7 @@ typedef enum { TMM_RECEIVEWINDIVERT, TMM_VERDICTWINDIVERT, TMM_DECODEWINDIVERT, + TMM_DECODELIB, TMM_FLOWMANAGER, TMM_FLOWRECYCLER, diff --git a/src/tm-threads.c b/src/tm-threads.c index 5da183c311be..5152af74fdcc 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -163,7 +163,7 @@ TmEcode TmThreadsSlotVarRun(ThreadVars *tv, Packet *p, TmSlot *slot) * is run until the flow engine kills the thread and the queue is * empty. */ -static int TmThreadTimeoutLoop(ThreadVars *tv, TmSlot *s) +int TmThreadTimeoutLoop(ThreadVars *tv, TmSlot *s) { TmSlot *fw_slot = tv->tm_flowworker; int r = TM_ECODE_OK; @@ -379,6 +379,89 @@ bool TmThreadsWaitForUnpause(ThreadVars *tv) return true; } +static void *TmThreadsLib(void *td) +{ + ThreadVars *tv = (ThreadVars *)td; + TmSlot *s = tv->tm_slots; + TmEcode r = TM_ECODE_OK; + TmSlot *slot = NULL; + + /* Set the thread name */ + SCSetThreadName(tv->name); + + if (tv->thread_setup_flags != 0) + TmThreadSetupOptions(tv); + + /* Drop the capabilities for this thread */ + SCDropCaps(tv); + + PacketPoolInit(); + + /* check if we are setup properly */ + if (s == NULL || tv->tmqh_in == NULL || tv->tmqh_out == NULL) { + SCLogError("TmSlot or ThreadVars badly setup: s=%p, tmqh_in=%p," + " tmqh_out=%p", + s, tv->tmqh_in, tv->tmqh_out); + TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); + return NULL; + } + + for (slot = s; slot != NULL; slot = slot->slot_next) { + if (slot->SlotThreadInit != NULL) { + void *slot_data = NULL; + r = slot->SlotThreadInit(tv, slot->slot_initdata, &slot_data); + if (r != TM_ECODE_OK) { + if (r == TM_ECODE_DONE) { + EngineDone(); + TmThreadsSetFlag(tv, THV_CLOSED | THV_INIT_DONE | THV_RUNNING_DONE); + goto error; + } else { + TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); + goto error; + } + } + (void)SC_ATOMIC_SET(slot->slot_data, slot_data); + } + + /* if the flowworker module is the first, get the threads input queue */ + if (slot == (TmSlot *)tv->tm_slots && (slot->tm_id == TMM_FLOWWORKER)) { + tv->stream_pq = tv->inq->pq; + tv->tm_flowworker = slot; + SCLogDebug("pre-stream packetqueue %p (inq)", tv->stream_pq); + tv->flow_queue = FlowQueueNew(); + if (tv->flow_queue == NULL) { + TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); + return NULL; + } + /* setup a queue */ + } else if (slot->tm_id == TMM_FLOWWORKER) { + tv->stream_pq_local = SCCalloc(1, sizeof(PacketQueue)); + if (tv->stream_pq_local == NULL) + FatalError("failed to alloc PacketQueue"); + SCMutexInit(&tv->stream_pq_local->mutex_q, NULL); + tv->stream_pq = tv->stream_pq_local; + tv->tm_flowworker = slot; + SCLogDebug("pre-stream packetqueue %p (local)", tv->stream_pq); + tv->flow_queue = FlowQueueNew(); + if (tv->flow_queue == NULL) { + TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); + return NULL; + } + } + } + StatsSetupPrivate(tv); + + TmThreadsSetFlag(tv, THV_INIT_DONE); + + TmThreadsWaitForUnpause(tv); + + return NULL; + +error: + tv->stream_pq = NULL; + return NULL; +} + static void *TmThreadsSlotVar(void *td) { ThreadVars *tv = (ThreadVars *)td; @@ -622,6 +705,8 @@ static TmEcode TmThreadSetSlots(ThreadVars *tv, const char *name, void *(*fn_p)( tv->tm_func = TmThreadsManagement; } else if (strcmp(name, "command") == 0) { tv->tm_func = TmThreadsManagement; + } else if (strcmp(name, "lib") == 0) { + tv->tm_func = TmThreadsLib; } else if (strcmp(name, "custom") == 0) { if (fn_p == NULL) goto error; @@ -1708,6 +1793,25 @@ TmEcode TmThreadSpawn(ThreadVars *tv) return TM_ECODE_OK; } +/** + * \brief Spawns a "fake" lib thread associated with the ThreadVars instance tv + * + * \retval TM_ECODE_OK on success and TM_ECODE_FAILED on failure + */ +TmEcode TmThreadLibSpawn(ThreadVars *tv) +{ + if (tv->tm_func == NULL) { + printf("ERROR: no thread function set\n"); + return TM_ECODE_FAILED; + } + + tv->tm_func((void *)tv); + + TmThreadWaitForFlag(tv, THV_INIT_DONE | THV_RUNNING_DONE); + + return TM_ECODE_OK; +} + /** * \brief Initializes the mutex and condition variables for this TV * diff --git a/src/tm-threads.h b/src/tm-threads.h index dde0f2029ee9..e1d60e0417c2 100644 --- a/src/tm-threads.h +++ b/src/tm-threads.h @@ -94,6 +94,8 @@ ThreadVars *TmThreadCreateMgmtThreadByName(const char *name, const char *module, ThreadVars *TmThreadCreateCmdThreadByName(const char *name, const char *module, int mucond); TmEcode TmThreadSpawn(ThreadVars *); +TmEcode TmThreadLibSpawn(ThreadVars *); +int TmThreadTimeoutLoop(ThreadVars *tv, TmSlot *s); void TmThreadKillThreadsFamily(int family); void TmThreadKillThreads(void); void TmThreadClearThreadsFamily(int family); From a2d3caeb333c8ee51d807581862c3d6e82e3df5c Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 10 Apr 2024 10:58:56 -0600 Subject: [PATCH 02/20] examples: move lib capture example to libcapture To keep the simple example simple, move the lib based capture method example to its own example. --- Makefile.am | 2 +- configure.ac | 1 + examples/lib/libcapture/.gitignore | 2 + examples/lib/libcapture/Makefile.am | 12 ++ examples/lib/libcapture/Makefile.example.in | 16 +++ examples/lib/libcapture/README.md | 41 +++++++ examples/lib/libcapture/main.c | 125 ++++++++++++++++++++ examples/lib/simple/Makefile.example.in | 8 +- examples/lib/simple/main.c | 75 +----------- 9 files changed, 206 insertions(+), 76 deletions(-) create mode 100644 examples/lib/libcapture/.gitignore create mode 100644 examples/lib/libcapture/Makefile.am create mode 100644 examples/lib/libcapture/Makefile.example.in create mode 100644 examples/lib/libcapture/README.md create mode 100644 examples/lib/libcapture/main.c diff --git a/Makefile.am b/Makefile.am index 20e50bdc4a03..177b3e8affba 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,7 +14,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \ examples/plugins SUBDIRS = $(HTP_DIR) rust src plugins qa rules doc contrib etc python ebpf \ $(SURICATA_UPDATE_DIR) -DIST_SUBDIRS = $(SUBDIRS) examples/lib/simple +DIST_SUBDIRS = $(SUBDIRS) examples/lib/simple examples/lib/libcapture CLEANFILES = stamp-h[0-9]* diff --git a/configure.ac b/configure.ac index ca964d9039a0..998dd05a63b9 100644 --- a/configure.ac +++ b/configure.ac @@ -2527,6 +2527,7 @@ AC_CONFIG_FILES(examples/plugins/c-json-filetype/Makefile) AC_CONFIG_FILES(examples/plugins/c-custom-loggers/Makefile) AC_CONFIG_FILES(examples/plugins/ci-capture/Makefile) AC_CONFIG_FILES(examples/lib/simple/Makefile examples/lib/simple/Makefile.example) +AC_CONFIG_FILES(examples/lib/libcapture/Makefile examples/lib/libcapture/Makefile.example) AC_CONFIG_FILES(plugins/Makefile) AC_CONFIG_FILES(plugins/pfring/Makefile) AC_CONFIG_FILES(plugins/napatech/Makefile) diff --git a/examples/lib/libcapture/.gitignore b/examples/lib/libcapture/.gitignore new file mode 100644 index 000000000000..579841d2a528 --- /dev/null +++ b/examples/lib/libcapture/.gitignore @@ -0,0 +1,2 @@ +!/Makefile.example.in +Makefile.example diff --git a/examples/lib/libcapture/Makefile.am b/examples/lib/libcapture/Makefile.am new file mode 100644 index 000000000000..231ca6f9625b --- /dev/null +++ b/examples/lib/libcapture/Makefile.am @@ -0,0 +1,12 @@ +bin_PROGRAMS = libcapture + +libcapture_SOURCES = main.c + +AM_CPPFLAGS = -I$(top_srcdir)/src + +libcapture_LDFLAGS = $(all_libraries) $(SECLDFLAGS) +libcapture_LDADD = $(top_builddir)/src/libsuricata_c.a ../../$(RUST_SURICATA_LIB) $(RUST_LDADD) +if HTP_LDADD +libcapture_LDADD += ../../$(HTP_LDADD) +endif +libcapture_DEPENDENCIES = $(top_builddir)/src/libsuricata_c.a ../../$(RUST_SURICATA_LIB) diff --git a/examples/lib/libcapture/Makefile.example.in b/examples/lib/libcapture/Makefile.example.in new file mode 100644 index 000000000000..b21a50f750c5 --- /dev/null +++ b/examples/lib/libcapture/Makefile.example.in @@ -0,0 +1,16 @@ +LIBSURICATA_CONFIG ?= @CONFIGURE_PREFIX@/bin/libsuricata-config + +SURICATA_LIBS = `$(LIBSURICATA_CONFIG) --libs --static` +SURICATA_CFLAGS := `$(LIBSURICATA_CONFIG) --cflags` + +# Currently the Suricata logging system requires this to be even for +# plugins. +CPPFLAGS += "-D__SCFILENAME__=\"$(*F)\"" + +all: libcapture + +libcapture: main.c + $(CC) -o $@ $^ $(CPPFLAGS) $(CFLAGS) $(SURICATA_CFLAGS) $(SURICATA_LIBS) + +clean: + rm -f libcapture diff --git a/examples/lib/libcapture/README.md b/examples/lib/libcapture/README.md new file mode 100644 index 000000000000..4e2d51785e6f --- /dev/null +++ b/examples/lib/libcapture/README.md @@ -0,0 +1,41 @@ +# LibCapture Example + +This is an example of using the Suricata library with the "lib" +capture method. The "lib" capture method is a simpler way of providing +packets to Suricata if you need to drive the deliver of packets, +rather than having Suricata run the main loop. + +## Building In Tree + +The Suricata build system has created a Makefile that should allow you +to build this application in-tree on most supported platforms. To +build simply run: + +``` +make +``` + +## Building Out of Tree + +A Makefile.example has also been generated to use as an example on how +to build against the library in a standalone application. + +First build and install the Suricata library including: + +``` +make install-library +make install-headers +``` + +Then run: + +``` +make -f Makefile.example +``` + +If you installed to a non-standard location, you need to ensure that +`libsuricata-config` is in your path, for example: + +``` +PATH=/opt/suricata/bin:$PATH make -f Makefile.example +``` diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c new file mode 100644 index 000000000000..7e6900e5f40c --- /dev/null +++ b/examples/lib/libcapture/main.c @@ -0,0 +1,125 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata.h" +#include "conf.h" +#include "pcap.h" +#include "runmode-lib.h" +#include "source-lib.h" +#include "threadvars.h" + +/* Suricata worker thread in library mode. + The functions should be wrapped in an API layer. */ +static void *SimpleWorker(void *arg) +{ + char *pcap_file = (char *)arg; + + /* Create worker. */ + ThreadVars *tv = RunModeCreateWorker(); + if (!tv) { + pthread_exit(NULL); + } + + /* Start worker. */ + if (RunModeSpawnWorker(tv) != 0) { + pthread_exit(NULL); + } + + /* Replay pcap. */ + pcap_t *fp = pcap_open_offline(pcap_file, NULL); + if (fp == NULL) { + pthread_exit(NULL); + } + + int datalink = pcap_datalink(fp); + struct pcap_pkthdr pkthdr; + const u_char *packet; + while ((packet = pcap_next(fp, &pkthdr)) != NULL) { + if (TmModuleLibHandlePacket(tv, packet, datalink, pkthdr.ts, pkthdr.len, 0, 0, NULL) != 0) { + pthread_exit(NULL); + } + } + pcap_close(fp); + + /* Cleanup. */ + RunModeDestroyWorker(tv); + pthread_exit(NULL); +} + +int main(int argc, char **argv) +{ + SuricataPreInit(argv[0]); + + /* Parse command line options. This is optional, you could + * directly configure Suricata through the Conf API. + The last argument is the PCAP file to replay. */ + SCParseCommandLine(argc - 1, argv); + + /* Set lib runmode. There is currently no way to set it via + the Conf API. */ + SuricataSetLibRunmode(); + + /* Validate/finalize the runmode. */ + if (SCFinalizeRunMode() != TM_ECODE_OK) { + exit(EXIT_FAILURE); + } + + /* Handle internal runmodes. Typically you wouldn't do this as a + * library user, however this example is showing how to replicate + * the Suricata application with the library. */ + switch (SCStartInternalRunMode(argc, argv)) { + case TM_ECODE_DONE: + exit(EXIT_SUCCESS); + case TM_ECODE_FAILED: + exit(EXIT_FAILURE); + } + + /* Load configuration file, could be done earlier but must be done + * before SuricataInit, but even then its still optional as you + * may be programmatically configuration Suricata. */ + if (SCLoadYamlConfig() != TM_ECODE_OK) { + exit(EXIT_FAILURE); + } + + /* Set "offline" runmode to replay a pcap in library mode. */ + if (!ConfSetFromString("runmode=offline", 1)) { + exit(EXIT_FAILURE); + } + + SuricataInit(); + + /* Create and start worker on its own thread, passing the PCAP file + as argument. This needs to be done in between SuricataInit and + SuricataPostInit. */ + pthread_t worker; + if (pthread_create(&worker, NULL, SimpleWorker, argv[argc - 1]) != 0) { + exit(EXIT_FAILURE); + } + + /* Need to introduce a little sleep to allow the worker thread to + initialize before SuricataPostInit invokes TmThreadContinueThreads(). + This should be handle at the API level. */ + usleep(100); + + SuricataPostInit(); + + /* Shutdown engine. */ + SuricataShutdown(); + GlobalsDestroy(); + + return EXIT_SUCCESS; +} diff --git a/examples/lib/simple/Makefile.example.in b/examples/lib/simple/Makefile.example.in index c9da97560e4d..724beccdb140 100644 --- a/examples/lib/simple/Makefile.example.in +++ b/examples/lib/simple/Makefile.example.in @@ -1,16 +1,12 @@ LIBSURICATA_CONFIG ?= @CONFIGURE_PREFIX@/bin/libsuricata-config -SURICATA_LIBS = `$(LIBSURICATA_CONFIG) --libs --static` +SURICATA_LIBS = `$(LIBSURICATA_CONFIG) --libs` SURICATA_CFLAGS := `$(LIBSURICATA_CONFIG) --cflags` -# Currently the Suricata logging system requires this to be even for -# plugins. -CPPFLAGS += "-D__SCFILENAME__=\"$(*F)\"" - all: simple simple: main.c - $(CC) -o $@ $^ $(CPPFLAGS) $(CFLAGS) $(SURICATA_CFLAGS) $(SURICATA_LIBS) + $(CC) -o $@ $^ $(CFLAGS) $(SURICATA_CFLAGS) $(SURICATA_LIBS) clean: rm -f simple diff --git a/examples/lib/simple/main.c b/examples/lib/simple/main.c index 7e6900e5f40c..f9c09fb0f550 100644 --- a/examples/lib/simple/main.c +++ b/examples/lib/simple/main.c @@ -16,62 +16,14 @@ */ #include "suricata.h" -#include "conf.h" -#include "pcap.h" -#include "runmode-lib.h" -#include "source-lib.h" -#include "threadvars.h" - -/* Suricata worker thread in library mode. - The functions should be wrapped in an API layer. */ -static void *SimpleWorker(void *arg) -{ - char *pcap_file = (char *)arg; - - /* Create worker. */ - ThreadVars *tv = RunModeCreateWorker(); - if (!tv) { - pthread_exit(NULL); - } - - /* Start worker. */ - if (RunModeSpawnWorker(tv) != 0) { - pthread_exit(NULL); - } - - /* Replay pcap. */ - pcap_t *fp = pcap_open_offline(pcap_file, NULL); - if (fp == NULL) { - pthread_exit(NULL); - } - - int datalink = pcap_datalink(fp); - struct pcap_pkthdr pkthdr; - const u_char *packet; - while ((packet = pcap_next(fp, &pkthdr)) != NULL) { - if (TmModuleLibHandlePacket(tv, packet, datalink, pkthdr.ts, pkthdr.len, 0, 0, NULL) != 0) { - pthread_exit(NULL); - } - } - pcap_close(fp); - - /* Cleanup. */ - RunModeDestroyWorker(tv); - pthread_exit(NULL); -} int main(int argc, char **argv) { SuricataPreInit(argv[0]); /* Parse command line options. This is optional, you could - * directly configure Suricata through the Conf API. - The last argument is the PCAP file to replay. */ - SCParseCommandLine(argc - 1, argv); - - /* Set lib runmode. There is currently no way to set it via - the Conf API. */ - SuricataSetLibRunmode(); + * directly configure Suricata through the Conf API. */ + SCParseCommandLine(argc, argv); /* Validate/finalize the runmode. */ if (SCFinalizeRunMode() != TM_ECODE_OK) { @@ -95,28 +47,13 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - /* Set "offline" runmode to replay a pcap in library mode. */ - if (!ConfSetFromString("runmode=offline", 1)) { - exit(EXIT_FAILURE); - } - SuricataInit(); - - /* Create and start worker on its own thread, passing the PCAP file - as argument. This needs to be done in between SuricataInit and - SuricataPostInit. */ - pthread_t worker; - if (pthread_create(&worker, NULL, SimpleWorker, argv[argc - 1]) != 0) { - exit(EXIT_FAILURE); - } - - /* Need to introduce a little sleep to allow the worker thread to - initialize before SuricataPostInit invokes TmThreadContinueThreads(). - This should be handle at the API level. */ - usleep(100); - SuricataPostInit(); + /* Suricata is now running, but we enter a loop to keep it running + * until it shouldn't be running anymore. */ + SuricataMainLoop(); + /* Shutdown engine. */ SuricataShutdown(); GlobalsDestroy(); From 34baee7954b6363be9fb494651801aa402d0f83c Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 10 Apr 2024 11:11:59 -0600 Subject: [PATCH 03/20] .gitignore: add more files - the generated binaries for lib examples - LSP files - man pages --- .gitignore | 6 ++++++ examples/lib/libcapture/.gitignore | 1 + examples/lib/simple/.gitignore | 1 + 3 files changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index 66416e27d14e..ed57b1b3ec3b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,9 @@ test.sh /libsuricata-config !/libsuricata-config.in !/.readthedocs.yaml + +# Man pages. +doc/userguide/*.1 + +# Generated compile commands for LSP. +/compile_commands.json diff --git a/examples/lib/libcapture/.gitignore b/examples/lib/libcapture/.gitignore index 579841d2a528..a6aa38650590 100644 --- a/examples/lib/libcapture/.gitignore +++ b/examples/lib/libcapture/.gitignore @@ -1,2 +1,3 @@ !/Makefile.example.in Makefile.example +/libcapture diff --git a/examples/lib/simple/.gitignore b/examples/lib/simple/.gitignore index 579841d2a528..c7c8b84913b4 100644 --- a/examples/lib/simple/.gitignore +++ b/examples/lib/simple/.gitignore @@ -1,2 +1,3 @@ !/Makefile.example.in Makefile.example +/simple From 3ea5c21b4b4beef8a963fcebda874d419c99b30f Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 18 Apr 2024 00:33:09 -0600 Subject: [PATCH 04/20] threads: don't attempt to join threads with an id of 0 Worker threads not created by Suricata, but instead a library user should not be joined, as Suricata does not have access to their thread handle, and it may in-fact be an unjoinable thread, such as the main process. When the thread ID is 0, assume the thread is "externally" managed, but still mark is as dead to satisfy Suricata's view of the thread. --- src/tm-threads.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tm-threads.c b/src/tm-threads.c index 5152af74fdcc..41b75a360fe4 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -1355,9 +1355,12 @@ static int TmThreadKillThread(ThreadVars *tv) } } - /* join it and flag it as dead */ - pthread_join(tv->t, NULL); - SCLogDebug("thread %s stopped", tv->name); + /* Join the thread and flag as dead, unless the thread ID is 0 as + * its not a thread created by Suricata. */ + if (tv->t) { + pthread_join(tv->t, NULL); + SCLogDebug("thread %s stopped", tv->name); + } TmThreadsSetFlag(tv, THV_DEAD); return 1; } From 2fdb2bedbbecceb30965ce9aa439ee0eb4d837c5 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 18 Apr 2024 16:53:47 -0600 Subject: [PATCH 05/20] examples/libcapture: better command line handling Use the more conventional "--" command line handling to separate the arguments. The first set will be passed to Suricata, and the args after "--" will be handled by the example. Currently this is a single PCAP filename, but will be extended to a list of PCAP filenames. --- examples/lib/libcapture/README.md | 11 +++++++++++ examples/lib/libcapture/main.c | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/examples/lib/libcapture/README.md b/examples/lib/libcapture/README.md index 4e2d51785e6f..13ec431dbc0b 100644 --- a/examples/lib/libcapture/README.md +++ b/examples/lib/libcapture/README.md @@ -15,6 +15,17 @@ build simply run: make ``` +## Running + +``` +./libcapture -l . -- filename.pcap +``` + +For this example, any arguments before `--` are passed directly as +Suricata command line arguments. Arguments after the first `--` are +handled by this example program, and currently the only argument is a +PCAP filename to be read. + ## Building Out of Tree A Makefile.example has also been generated to use as an example on how diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index 7e6900e5f40c..e768b20e28fa 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -67,7 +67,21 @@ int main(int argc, char **argv) /* Parse command line options. This is optional, you could * directly configure Suricata through the Conf API. The last argument is the PCAP file to replay. */ - SCParseCommandLine(argc - 1, argv); + SCParseCommandLine(argc, argv); + + /* Find our list of pcap files, after the "--". */ + while (argc) { + bool end = strncmp(argv[0], "--", 2) == 0; + argv++; + argc--; + if (end) { + break; + } + } + if (argc == 0) { + fprintf(stderr, "ERROR: No PCAP files provided\n"); + return 1; + } /* Set lib runmode. There is currently no way to set it via the Conf API. */ From f53597121e86818a4674da1a1aaf1d8e9c179bdf Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 24 Apr 2024 16:10:15 -0600 Subject: [PATCH 06/20] librunmode: rename threadvars creation function Also use a proper return type (ThreadVars *). --- examples/lib/libcapture/main.c | 2 +- src/runmode-lib.c | 8 +------- src/runmode-lib.h | 14 +++++++++----- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index e768b20e28fa..9d43c2977359 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -29,7 +29,7 @@ static void *SimpleWorker(void *arg) char *pcap_file = (char *)arg; /* Create worker. */ - ThreadVars *tv = RunModeCreateWorker(); + ThreadVars *tv = SCRunModeLibCreateThreadVars(); if (!tv) { pthread_exit(NULL); } diff --git a/src/runmode-lib.c b/src/runmode-lib.c index 1441dabe4bcb..97f8d9d78efb 100644 --- a/src/runmode-lib.c +++ b/src/runmode-lib.c @@ -59,13 +59,7 @@ const char *RunModeLibGetDefaultMode(void) return "live"; } -/** \brief create a "fake" worker thread in charge of processing the packets. - * - * This method just creates a context representing the worker, which is handled from the library - * client. No actual thread (pthread_t) is created. - * - * \return Pointer to ThreadVars structure representing the worker thread */ -void *RunModeCreateWorker(void) +ThreadVars *SCRunModeLibCreateThreadVars(void) { char tname[TM_THREAD_NAME_MAX]; TmModule *tm_module = NULL; diff --git a/src/runmode-lib.h b/src/runmode-lib.h index c33cb4e65bf3..86a1b812d05e 100644 --- a/src/runmode-lib.h +++ b/src/runmode-lib.h @@ -25,6 +25,8 @@ #ifndef SURICATA_RUNMODE_LIB_H #define SURICATA_RUNMODE_LIB_H +#include "threadvars.h" + /** \brief register runmodes for suricata as a library */ void RunModeIdsLibRegister(void); @@ -37,13 +39,15 @@ int RunModeIdsLibOffline(void); /** \brief runmode default mode (live) */ const char *RunModeLibGetDefaultMode(void); -/** \brief create a "fake" worker thread in charge of processing the packets. +/** + * \brief Create ThreadVars for use by a user provided thread. * - * This method just creates a context representing the worker, which is handled from the library - * client. No actual thread (pthread_t) is created. + * Unlike other runmodes, this does not spawn a thread, as the threads + * are controlled by the application using Suricata as a library. * - * \return Pointer to ThreadVars structure representing the worker thread */ -void *RunModeCreateWorker(void); + * \return Pointer to allocated ThreadVars or NULL on failure + */ +ThreadVars *SCRunModeLibCreateThreadVars(void); /** \brief start the "fake" worker. * From e09a1965ec3961612755f43397095bb6f8d38b58 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 24 Apr 2024 16:47:11 -0600 Subject: [PATCH 07/20] lib: remove global worker id variable Update ThreadVars creation in lib mode to have the worker_id provided by the user. --- examples/lib/libcapture/main.c | 4 +++- src/runmode-lib.c | 8 ++------ src/runmode-lib.h | 4 +++- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index 9d43c2977359..5806af7c723e 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -22,6 +22,8 @@ #include "source-lib.h" #include "threadvars.h" +static int worker_id = 1; + /* Suricata worker thread in library mode. The functions should be wrapped in an API layer. */ static void *SimpleWorker(void *arg) @@ -29,7 +31,7 @@ static void *SimpleWorker(void *arg) char *pcap_file = (char *)arg; /* Create worker. */ - ThreadVars *tv = SCRunModeLibCreateThreadVars(); + ThreadVars *tv = SCRunModeLibCreateThreadVars(worker_id++); if (!tv) { pthread_exit(NULL); } diff --git a/src/runmode-lib.c b/src/runmode-lib.c index 97f8d9d78efb..b5643d99c833 100644 --- a/src/runmode-lib.c +++ b/src/runmode-lib.c @@ -25,9 +25,6 @@ #include "runmode-lib.h" #include "runmodes.h" #include "tm-threads.h" -#include "util-device.h" - -static int g_thread_id = 0; /** \brief register runmodes for suricata as a library */ void RunModeIdsLibRegister(void) @@ -59,11 +56,11 @@ const char *RunModeLibGetDefaultMode(void) return "live"; } -ThreadVars *SCRunModeLibCreateThreadVars(void) +ThreadVars *SCRunModeLibCreateThreadVars(int worker_id) { char tname[TM_THREAD_NAME_MAX]; TmModule *tm_module = NULL; - snprintf(tname, sizeof(tname), "%s#%02d", thread_name_workers, ++g_thread_id); + snprintf(tname, sizeof(tname), "%s#%02d", thread_name_workers, worker_id); ThreadVars *tv = TmThreadCreatePacketHandler( tname, "packetpool", "packetpool", "packetpool", "packetpool", "lib"); @@ -142,7 +139,6 @@ void RunModeDestroyWorker(void *td) } tv->stream_pq = NULL; - --g_thread_id; SCLogDebug("%s ending", tv->name); TmThreadsSetFlag(tv, THV_CLOSED); } diff --git a/src/runmode-lib.h b/src/runmode-lib.h index 86a1b812d05e..44c8c3a4748c 100644 --- a/src/runmode-lib.h +++ b/src/runmode-lib.h @@ -45,9 +45,11 @@ const char *RunModeLibGetDefaultMode(void); * Unlike other runmodes, this does not spawn a thread, as the threads * are controlled by the application using Suricata as a library. * + * \param worker_id an ID to give this ThreadVars instance + * * \return Pointer to allocated ThreadVars or NULL on failure */ -ThreadVars *SCRunModeLibCreateThreadVars(void); +ThreadVars *SCRunModeLibCreateThreadVars(int worker_id); /** \brief start the "fake" worker. * From 13ecdebbba8fbeacaa62cfd2c67a13720a87b91e Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 3 Jun 2024 17:04:10 -0600 Subject: [PATCH 08/20] librunmode: take pointer to LiveDevice, not name In the library capture example, show how the packet counter can be updated. --- examples/lib/libcapture/main.c | 15 ++++++++++++++- src/source-lib.c | 12 ++++-------- src/source-lib.h | 5 +++-- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index 5806af7c723e..e3c298912793 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -47,13 +47,21 @@ static void *SimpleWorker(void *arg) pthread_exit(NULL); } + LiveDevice *device = LiveGetDevice("lib0"); + assert(device != NULL); + int datalink = pcap_datalink(fp); + int count = 0; struct pcap_pkthdr pkthdr; const u_char *packet; while ((packet = pcap_next(fp, &pkthdr)) != NULL) { - if (TmModuleLibHandlePacket(tv, packet, datalink, pkthdr.ts, pkthdr.len, 0, 0, NULL) != 0) { + if (TmModuleLibHandlePacket(tv, device, packet, datalink, pkthdr.ts, pkthdr.len, 0, 0) != + 0) { pthread_exit(NULL); } + + (void)SC_ATOMIC_ADD(device->pkts, 1); + count++; } pcap_close(fp); @@ -116,6 +124,11 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } + if (LiveRegisterDevice("lib0") < 0) { + fprintf(stderr, "LiveRegisterDevice failed"); + exit(1); + } + SuricataInit(); /* Create and start worker on its own thread, passing the PCAP file diff --git a/src/source-lib.c b/src/source-lib.c index 6c304fc30069..7e8f55415ecd 100644 --- a/src/source-lib.c +++ b/src/source-lib.c @@ -122,17 +122,17 @@ TmEcode DecodeLib(ThreadVars *tv, Packet *p, void *data) /** \brief process a single packet. * * \param tv Pointer to the per-thread structure. + * \param device Pionter to LiveDevice instance * \param data Pointer to the raw packet. * \param datalink Datalink type. * \param ts Timeval structure. * \param len Packet length. * \param tenant_id Tenant id of the detection engine to use. * \param flags Packet flags (packet checksum, rule profiling...). - * \param iface Sniffing interface this packet comes from (can be NULL). * \return Error code. */ -int TmModuleLibHandlePacket(ThreadVars *tv, const uint8_t *data, int datalink, struct timeval ts, - uint32_t len, uint32_t tenant_id, uint32_t flags, const char *iface) +int TmModuleLibHandlePacket(ThreadVars *tv, LiveDevice *device, const uint8_t *data, int datalink, + struct timeval ts, uint32_t len, uint32_t tenant_id, uint32_t flags) { /* If the packet is NULL, consider it as a read timeout. */ @@ -159,11 +159,7 @@ int TmModuleLibHandlePacket(ThreadVars *tv, const uint8_t *data, int datalink, s p->datalink = datalink; p->tenant_id = tenant_id; p->flags |= flags; - - /* Set the sniffing interface. */ - if (iface) { - p->livedev = LiveGetDevice(iface); - } + p->livedev = device; if (PacketSetData(p, data, len) == -1) { TmqhOutputPacketpool(tv, p); diff --git a/src/source-lib.h b/src/source-lib.h index 3f0594bd8ead..432f12aa505b 100644 --- a/src/source-lib.h +++ b/src/source-lib.h @@ -27,6 +27,7 @@ #define SURICATA_SOURCE_LIB_H #include "tm-threads.h" +#include "util-device.h" /** \brief register a "Decode" module for suricata as a library. * @@ -45,7 +46,7 @@ void TmModuleDecodeLibRegister(void); * \param iface Sniffing interface this packet comes from (can be NULL). * \return Error code. */ -int TmModuleLibHandlePacket(ThreadVars *tv, const uint8_t *data, int datalink, struct timeval ts, - uint32_t len, uint32_t tenant_id, uint32_t flags, const char *iface); +int TmModuleLibHandlePacket(ThreadVars *tv, LiveDevice *device, const uint8_t *data, int datalink, + struct timeval ts, uint32_t len, uint32_t tenant_id, uint32_t flags); #endif /* SURICATA_SOURCE_LIB_H */ From a2a09aa06aa1bbf34f79f4016c977109983eb69f Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 23 Aug 2024 12:49:20 -0600 Subject: [PATCH 09/20] threads: refactor TmThreadsSlotPktAcqLoop for user threads Refactor TmThreadsSlotPktAcqLoop for user provided thread by breaking out the init and finish code into their own functions. For user provided threads, Suricata should not "drive" the thread, but the setup and finish code is the same. The finish function is exported so it can be called by the user application when its receive loop or equivalent is done. Also remove obsolete comment. --- src/runmode-lib.c | 34 +-------- src/tm-threads.c | 188 +++++++++++++++++----------------------------- src/tm-threads.h | 1 + 3 files changed, 69 insertions(+), 154 deletions(-) diff --git a/src/runmode-lib.c b/src/runmode-lib.c index b5643d99c833..2699d47e44c5 100644 --- a/src/runmode-lib.c +++ b/src/runmode-lib.c @@ -108,37 +108,5 @@ int RunModeSpawnWorker(void *td) /** \brief destroy a worker thread */ void RunModeDestroyWorker(void *td) { - ThreadVars *tv = (ThreadVars *)td; - TmSlot *s = tv->tm_slots; - TmEcode r; - TmSlot *slot = NULL; - - StatsSyncCounters(tv); - - TmThreadsSetFlag(tv, THV_FLOW_LOOP); - - /* process all pseudo packets the flow timeout may throw at us */ - TmThreadTimeoutLoop(tv, s); - - TmThreadsSetFlag(tv, THV_RUNNING_DONE); - TmThreadWaitForFlag(tv, THV_DEINIT); - - PacketPoolDestroy(); - - for (slot = s; slot != NULL; slot = slot->slot_next) { - if (slot->SlotThreadExitPrintStats != NULL) { - slot->SlotThreadExitPrintStats(tv, SC_ATOMIC_GET(slot->slot_data)); - } - - if (slot->SlotThreadDeinit != NULL) { - r = slot->SlotThreadDeinit(tv, SC_ATOMIC_GET(slot->slot_data)); - if (r != TM_ECODE_OK) { - break; - } - } - } - - tv->stream_pq = NULL; - SCLogDebug("%s ending", tv->name); - TmThreadsSetFlag(tv, THV_CLOSED); + SCTmThreadsSlotPktAcqLoopFinish((ThreadVars *)td); } diff --git a/src/tm-threads.c b/src/tm-threads.c index 41b75a360fe4..819fbc048dfa 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -204,37 +204,9 @@ int TmThreadTimeoutLoop(ThreadVars *tv, TmSlot *s) return r; } -/* - - pcap/nfq - - pkt read - callback - process_pkt - - pkt read - process_pkt - - slot: - setup - - pkt_ack_loop(tv, slot_data) - - deinit - - process_pkt: - while(s) - run s; - queue; - - */ - -static void *TmThreadsSlotPktAcqLoop(void *td) +static bool TmThreadsSlotPktAcqLoopInit(ThreadVars *tv) { - ThreadVars *tv = (ThreadVars *)td; TmSlot *s = tv->tm_slots; - TmEcode r = TM_ECODE_OK; - TmSlot *slot = NULL; SCSetThreadName(tv->name); @@ -244,21 +216,10 @@ static void *TmThreadsSlotPktAcqLoop(void *td) CaptureStatsSetup(tv); PacketPoolInit(); - /* check if we are setup properly */ - if (s == NULL || s->PktAcqLoop == NULL || tv->tmqh_in == NULL || tv->tmqh_out == NULL) { - SCLogError("TmSlot or ThreadVars badly setup: s=%p," - " PktAcqLoop=%p, tmqh_in=%p," - " tmqh_out=%p", - s, s ? s->PktAcqLoop : NULL, tv->tmqh_in, tv->tmqh_out); - TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); - pthread_exit((void *) -1); - return NULL; - } - - for (slot = s; slot != NULL; slot = slot->slot_next) { + for (TmSlot *slot = s; slot != NULL; slot = slot->slot_next) { if (slot->SlotThreadInit != NULL) { void *slot_data = NULL; - r = slot->SlotThreadInit(tv, slot->slot_initdata, &slot_data); + TmEcode r = slot->SlotThreadInit(tv, slot->slot_initdata, &slot_data); if (r != TM_ECODE_OK) { if (r == TM_ECODE_DONE) { EngineDone(); @@ -280,8 +241,7 @@ static void *TmThreadsSlotPktAcqLoop(void *td) tv->flow_queue = FlowQueueNew(); if (tv->flow_queue == NULL) { TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); - pthread_exit((void *) -1); - return NULL; + goto error; } /* setup a queue */ } else if (slot->tm_id == TMM_FLOWWORKER) { @@ -295,8 +255,7 @@ static void *TmThreadsSlotPktAcqLoop(void *td) tv->flow_queue = FlowQueueNew(); if (tv->flow_queue == NULL) { TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); - pthread_exit((void *) -1); - return NULL; + goto error; } } } @@ -304,22 +263,18 @@ static void *TmThreadsSlotPktAcqLoop(void *td) StatsSetupPrivate(tv); TmThreadsSetFlag(tv, THV_INIT_DONE); - bool run = TmThreadsWaitForUnpause(tv); - while (run) { - r = s->PktAcqLoop(tv, SC_ATOMIC_GET(s->slot_data), s); + return true; + +error: + return false; +} + +bool SCTmThreadsSlotPktAcqLoopFinish(ThreadVars *tv) +{ + TmSlot *s = tv->tm_slots; + bool rc = true; - if (r == TM_ECODE_FAILED) { - TmThreadsSetFlag(tv, THV_FAILED); - run = false; - } - if (TmThreadsCheckFlag(tv, THV_KILL_PKTACQ) || suricata_ctl_flags) { - run = false; - } - if (r == TM_ECODE_DONE) { - run = false; - } - } StatsSyncCounters(tv); TmThreadsSetFlag(tv, THV_FLOW_LOOP); @@ -332,20 +287,67 @@ static void *TmThreadsSlotPktAcqLoop(void *td) PacketPoolDestroy(); - for (slot = s; slot != NULL; slot = slot->slot_next) { + for (TmSlot *slot = s; slot != NULL; slot = slot->slot_next) { if (slot->SlotThreadExitPrintStats != NULL) { slot->SlotThreadExitPrintStats(tv, SC_ATOMIC_GET(slot->slot_data)); } if (slot->SlotThreadDeinit != NULL) { - r = slot->SlotThreadDeinit(tv, SC_ATOMIC_GET(slot->slot_data)); + TmEcode r = slot->SlotThreadDeinit(tv, SC_ATOMIC_GET(slot->slot_data)); if (r != TM_ECODE_OK) { TmThreadsSetFlag(tv, THV_CLOSED); - goto error; + rc = false; } } } + tv->stream_pq = NULL; + SCLogDebug("%s ending", tv->name); + TmThreadsSetFlag(tv, THV_CLOSED); + return rc; +} + +static void *TmThreadsSlotPktAcqLoop(void *td) +{ + ThreadVars *tv = (ThreadVars *)td; + TmSlot *s = tv->tm_slots; + TmEcode r = TM_ECODE_OK; + + /* check if we are setup properly */ + if (s == NULL || s->PktAcqLoop == NULL || tv->tmqh_in == NULL || tv->tmqh_out == NULL) { + SCLogError("TmSlot or ThreadVars badly setup: s=%p," + " PktAcqLoop=%p, tmqh_in=%p," + " tmqh_out=%p", + s, s ? s->PktAcqLoop : NULL, tv->tmqh_in, tv->tmqh_out); + TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); + pthread_exit((void *)-1); + return NULL; + } + + if (!TmThreadsSlotPktAcqLoopInit(td)) { + goto error; + } + + bool run = TmThreadsWaitForUnpause(tv); + + while (run) { + r = s->PktAcqLoop(tv, SC_ATOMIC_GET(s->slot_data), s); + + if (r == TM_ECODE_FAILED) { + TmThreadsSetFlag(tv, THV_FAILED); + run = false; + } + if (TmThreadsCheckFlag(tv, THV_KILL_PKTACQ) || suricata_ctl_flags) { + run = false; + } + if (r == TM_ECODE_DONE) { + run = false; + } + } + if (!SCTmThreadsSlotPktAcqLoopFinish(tv)) { + goto error; + } + tv->stream_pq = NULL; SCLogDebug("%s ending", tv->name); TmThreadsSetFlag(tv, THV_CLOSED); @@ -383,19 +385,6 @@ static void *TmThreadsLib(void *td) { ThreadVars *tv = (ThreadVars *)td; TmSlot *s = tv->tm_slots; - TmEcode r = TM_ECODE_OK; - TmSlot *slot = NULL; - - /* Set the thread name */ - SCSetThreadName(tv->name); - - if (tv->thread_setup_flags != 0) - TmThreadSetupOptions(tv); - - /* Drop the capabilities for this thread */ - SCDropCaps(tv); - - PacketPoolInit(); /* check if we are setup properly */ if (s == NULL || tv->tmqh_in == NULL || tv->tmqh_out == NULL) { @@ -406,52 +395,9 @@ static void *TmThreadsLib(void *td) return NULL; } - for (slot = s; slot != NULL; slot = slot->slot_next) { - if (slot->SlotThreadInit != NULL) { - void *slot_data = NULL; - r = slot->SlotThreadInit(tv, slot->slot_initdata, &slot_data); - if (r != TM_ECODE_OK) { - if (r == TM_ECODE_DONE) { - EngineDone(); - TmThreadsSetFlag(tv, THV_CLOSED | THV_INIT_DONE | THV_RUNNING_DONE); - goto error; - } else { - TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); - goto error; - } - } - (void)SC_ATOMIC_SET(slot->slot_data, slot_data); - } - - /* if the flowworker module is the first, get the threads input queue */ - if (slot == (TmSlot *)tv->tm_slots && (slot->tm_id == TMM_FLOWWORKER)) { - tv->stream_pq = tv->inq->pq; - tv->tm_flowworker = slot; - SCLogDebug("pre-stream packetqueue %p (inq)", tv->stream_pq); - tv->flow_queue = FlowQueueNew(); - if (tv->flow_queue == NULL) { - TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); - return NULL; - } - /* setup a queue */ - } else if (slot->tm_id == TMM_FLOWWORKER) { - tv->stream_pq_local = SCCalloc(1, sizeof(PacketQueue)); - if (tv->stream_pq_local == NULL) - FatalError("failed to alloc PacketQueue"); - SCMutexInit(&tv->stream_pq_local->mutex_q, NULL); - tv->stream_pq = tv->stream_pq_local; - tv->tm_flowworker = slot; - SCLogDebug("pre-stream packetqueue %p (local)", tv->stream_pq); - tv->flow_queue = FlowQueueNew(); - if (tv->flow_queue == NULL) { - TmThreadsSetFlag(tv, THV_CLOSED | THV_RUNNING_DONE); - return NULL; - } - } + if (!TmThreadsSlotPktAcqLoopInit(tv)) { + goto error; } - StatsSetupPrivate(tv); - - TmThreadsSetFlag(tv, THV_INIT_DONE); TmThreadsWaitForUnpause(tv); diff --git a/src/tm-threads.h b/src/tm-threads.h index e1d60e0417c2..540113d9bbf4 100644 --- a/src/tm-threads.h +++ b/src/tm-threads.h @@ -291,6 +291,7 @@ void TmThreadsGetMinimalTimestamp(struct timeval *ts); SCTime_t TmThreadsGetThreadTime(const int idx); uint16_t TmThreadsGetWorkerThreadMax(void); bool TmThreadsTimeSubsysIsReady(void); +bool SCTmThreadsSlotPktAcqLoopFinish(ThreadVars *tv); /** \brief Wait for a thread to become unpaused. * From aa4a80d2c020e6aca7ca98d0bcf00347bfb85c37 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 23 Aug 2024 14:52:34 -0600 Subject: [PATCH 10/20] lib-runmode: consistent naming style And add SC prefix. --- examples/lib/libcapture/main.c | 4 ++-- src/runmode-lib.c | 16 ++++++++-------- src/runmode-lib.h | 12 ++++++------ src/runmodes.c | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index e3c298912793..8527253ee3bd 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -37,7 +37,7 @@ static void *SimpleWorker(void *arg) } /* Start worker. */ - if (RunModeSpawnWorker(tv) != 0) { + if (SCRunModeLibSpawnWorker(tv) != 0) { pthread_exit(NULL); } @@ -66,7 +66,7 @@ static void *SimpleWorker(void *arg) pcap_close(fp); /* Cleanup. */ - RunModeDestroyWorker(tv); + SCRunModeLibDestroyWorker(tv); pthread_exit(NULL); } diff --git a/src/runmode-lib.c b/src/runmode-lib.c index 2699d47e44c5..94ed43b83432 100644 --- a/src/runmode-lib.c +++ b/src/runmode-lib.c @@ -27,16 +27,16 @@ #include "tm-threads.h" /** \brief register runmodes for suricata as a library */ -void RunModeIdsLibRegister(void) +void SCRunModeLibIdsRegister(void) { RunModeRegisterNewRunMode(RUNMODE_LIB, "offline", "Library offline mode (pcap replaying)", - RunModeIdsLibOffline, NULL); - RunModeRegisterNewRunMode(RUNMODE_LIB, "live", "Library live mode", RunModeIdsLibLive, NULL); + SCRunModeLibIdsOffline, NULL); + RunModeRegisterNewRunMode(RUNMODE_LIB, "live", "Library live mode", SCRunModeLibIdsLive, NULL); return; } /** \brief runmode for offline packet processing (pcap files) */ -int RunModeIdsLibOffline(void) +int SCRunModeLibIdsOffline(void) { TimeModeSetOffline(); @@ -44,14 +44,14 @@ int RunModeIdsLibOffline(void) } /** \brief runmode for live packet processing */ -int RunModeIdsLibLive(void) +int SCRunModeLibIdsLive(void) { TimeModeSetLive(); return 0; } -const char *RunModeLibGetDefaultMode(void) +const char *SCRunModeLibGetDefaultMode(void) { return "live"; } @@ -92,7 +92,7 @@ ThreadVars *SCRunModeLibCreateThreadVars(int worker_id) * * This method performs all the initialization tasks. */ -int RunModeSpawnWorker(void *td) +int SCRunModeLibSpawnWorker(void *td) { ThreadVars *tv = (ThreadVars *)td; @@ -106,7 +106,7 @@ int RunModeSpawnWorker(void *td) } /** \brief destroy a worker thread */ -void RunModeDestroyWorker(void *td) +void SCRunModeLibDestroyWorker(void *td) { SCTmThreadsSlotPktAcqLoopFinish((ThreadVars *)td); } diff --git a/src/runmode-lib.h b/src/runmode-lib.h index 44c8c3a4748c..0e37d3db4851 100644 --- a/src/runmode-lib.h +++ b/src/runmode-lib.h @@ -28,16 +28,16 @@ #include "threadvars.h" /** \brief register runmodes for suricata as a library */ -void RunModeIdsLibRegister(void); +void SCRunModeLibIdsRegister(void); /** \brief runmode for live packet processing */ -int RunModeIdsLibLive(void); +int SCRunModeLibIdsLive(void); /** \brief runmode for offline packet processing (pcap files) */ -int RunModeIdsLibOffline(void); +int SCRunModeLibIdsOffline(void); /** \brief runmode default mode (live) */ -const char *RunModeLibGetDefaultMode(void); +const char *SCRunModeLibGetDefaultMode(void); /** * \brief Create ThreadVars for use by a user provided thread. @@ -55,9 +55,9 @@ ThreadVars *SCRunModeLibCreateThreadVars(int worker_id); * * This method performs all the initialization tasks. */ -int RunModeSpawnWorker(void *); +int SCRunModeLibSpawnWorker(void *); /** \brief destroy a worker thread */ -void RunModeDestroyWorker(void *); +void SCRunModeLibDestroyWorker(void *); #endif /* SURICATA_RUNMODE_LIB_H */ diff --git a/src/runmodes.c b/src/runmodes.c index beea5114e9fc..a562be0ebd1c 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -233,7 +233,7 @@ void RunModeRegisterRunModes(void) RunModeUnixSocketRegister(); RunModeIpsWinDivertRegister(); RunModeDpdkRegister(); - RunModeIdsLibRegister(); + SCRunModeLibIdsRegister(); #ifdef UNITTESTS UtRunModeRegister(); #endif @@ -347,7 +347,7 @@ static const char *RunModeGetConfOrDefault(int capture_mode, const char *capture break; #endif case RUNMODE_LIB: - custom_mode = RunModeLibGetDefaultMode(); + custom_mode = SCRunModeLibGetDefaultMode(); break; default: return NULL; From 1ad4b9d77c8065abac12dcd7b9549d3efc076e4a Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 23 Aug 2024 14:54:24 -0600 Subject: [PATCH 11/20] lib-source: reorganize to avoid static prototypes --- src/source-lib.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/src/source-lib.c b/src/source-lib.c index 7e8f55415ecd..822d7668f3f1 100644 --- a/src/source-lib.c +++ b/src/source-lib.c @@ -27,27 +27,9 @@ #include "source-lib.h" #include "util-device.h" -static TmEcode DecodeLibThreadInit(ThreadVars *tv, const void *initdata, void **data); -static TmEcode DecodeLibThreadDeinit(ThreadVars *tv, void *data); -static TmEcode DecodeLib(ThreadVars *tv, Packet *p, void *data); - /* Set time to the first packet timestamp when replaying a PCAP. */ static bool time_set = false; -/** \brief register a "Decode" module for suricata as a library. - * - * The "Decode" module is the first module invoked when processing a packet */ -void TmModuleDecodeLibRegister(void) -{ - tmm_modules[TMM_DECODELIB].name = "DecodeLib"; - tmm_modules[TMM_DECODELIB].ThreadInit = DecodeLibThreadInit; - tmm_modules[TMM_DECODELIB].Func = DecodeLib; - tmm_modules[TMM_DECODELIB].ThreadExitPrintStats = NULL; - tmm_modules[TMM_DECODELIB].ThreadDeinit = DecodeLibThreadDeinit; - tmm_modules[TMM_DECODELIB].cap_flags = 0; - tmm_modules[TMM_DECODELIB].flags = TM_FLAG_DECODE_TM; -} - /** \brief initialize the "Decode" module. * * \param tv Pointer to the per-thread structure. @@ -55,7 +37,7 @@ void TmModuleDecodeLibRegister(void) * \param data Pointer to the initialized context. * \return Error code. */ -TmEcode DecodeLibThreadInit(ThreadVars *tv, const void *initdata, void **data) +static TmEcode DecodeLibThreadInit(ThreadVars *tv, const void *initdata, void **data) { SCEnter(); DecodeThreadVars *dtv = NULL; @@ -78,7 +60,7 @@ TmEcode DecodeLibThreadInit(ThreadVars *tv, const void *initdata, void **data) * \param data Pointer to the context. * \return Error code. */ -TmEcode DecodeLibThreadDeinit(ThreadVars *tv, void *data) +static TmEcode DecodeLibThreadDeinit(ThreadVars *tv, void *data) { if (data != NULL) DecodeThreadVarsFree(tv, data); @@ -96,7 +78,7 @@ TmEcode DecodeLibThreadDeinit(ThreadVars *tv, void *data) * \param data Pointer to the context. * \return Error code. */ -TmEcode DecodeLib(ThreadVars *tv, Packet *p, void *data) +static TmEcode DecodeLib(ThreadVars *tv, Packet *p, void *data) { SCEnter(); DecodeThreadVars *dtv = (DecodeThreadVars *)data; @@ -174,3 +156,17 @@ int TmModuleLibHandlePacket(ThreadVars *tv, LiveDevice *device, const uint8_t *d SCReturnInt(TM_ECODE_OK); } + +/** \brief register a "Decode" module for suricata as a library. + * + * The "Decode" module is the first module invoked when processing a packet */ +void TmModuleDecodeLibRegister(void) +{ + tmm_modules[TMM_DECODELIB].name = "DecodeLib"; + tmm_modules[TMM_DECODELIB].ThreadInit = DecodeLibThreadInit; + tmm_modules[TMM_DECODELIB].Func = DecodeLib; + tmm_modules[TMM_DECODELIB].ThreadExitPrintStats = NULL; + tmm_modules[TMM_DECODELIB].ThreadDeinit = DecodeLibThreadDeinit; + tmm_modules[TMM_DECODELIB].cap_flags = 0; + tmm_modules[TMM_DECODELIB].flags = TM_FLAG_DECODE_TM; +} From 8db7420462571aab846a36d642c99cba4bf62019 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 26 Aug 2024 14:18:22 -0600 Subject: [PATCH 12/20] runmodes: typedef runmode enum and use as type Also remove function to set the library mode. This is easy enough to do with SCRunmodeSet, and we don't want to add a specific setter for each and every runmode. --- examples/lib/libcapture/main.c | 7 ++++--- src/runmodes.c | 10 +++++----- src/runmodes.h | 10 +++++----- src/suricata.c | 9 ++------- src/suricata.h | 11 ++++------- src/util-privs.c | 2 ++ 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index 8527253ee3bd..d86fcb0ace29 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -16,6 +16,7 @@ */ #include "suricata.h" +#include "runmodes.h" #include "conf.h" #include "pcap.h" #include "runmode-lib.h" @@ -93,9 +94,9 @@ int main(int argc, char **argv) return 1; } - /* Set lib runmode. There is currently no way to set it via - the Conf API. */ - SuricataSetLibRunmode(); + /* Set the runmode to library mode. Perhaps in the future this + should be done in some library bootstrap function. */ + SCRunmodeSet(RUNMODE_LIB); /* Validate/finalize the runmode. */ if (SCFinalizeRunMode() != TM_ECODE_OK) { diff --git a/src/runmodes.c b/src/runmodes.c index a562be0ebd1c..ad450283fe74 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -79,7 +79,7 @@ const char *thread_name_counter_wakeup = "CW"; */ typedef struct RunMode_ { /* the runmode type */ - enum RunModes runmode; + enum SCRunModes runmode; const char *name; const char *description; /* runmode function */ @@ -175,7 +175,7 @@ static const char *RunModeTranslateModeToName(int runmode) * \param runmode The runmode type. * \param runmode_custom_id The runmode custom id. */ -static RunMode *RunModeGetCustomMode(enum RunModes runmode, const char *custom_mode) +static RunMode *RunModeGetCustomMode(enum SCRunModes runmode, const char *custom_mode) { if (runmode < RUNMODE_USER_MAX) { for (int i = 0; i < runmodes[runmode].cnt; i++) { @@ -470,7 +470,7 @@ int RunModeNeedsBypassManager(void) * \param description Description for this runmode. * \param RunModeFunc The function to be run for this runmode. */ -void RunModeRegisterNewRunMode(enum RunModes runmode, const char *name, const char *description, +void RunModeRegisterNewRunMode(enum SCRunModes runmode, const char *name, const char *description, int (*RunModeFunc)(void), int (*RunModeIsIPSEnabled)(void)) { if (RunModeGetCustomMode(runmode, name) != NULL) { @@ -532,7 +532,7 @@ int RunModeOutputFiledataEnabled(void) return filedata_logger_count > 0; } -bool IsRunModeSystem(enum RunModes run_mode_to_check) +bool IsRunModeSystem(enum SCRunModes run_mode_to_check) { switch (run_mode_to_check) { case RUNMODE_PCAP_FILE: @@ -545,7 +545,7 @@ bool IsRunModeSystem(enum RunModes run_mode_to_check) } } -bool IsRunModeOffline(enum RunModes run_mode_to_check) +bool IsRunModeOffline(enum SCRunModes run_mode_to_check) { switch(run_mode_to_check) { case RUNMODE_CONF_TEST: diff --git a/src/runmodes.h b/src/runmodes.h index 70148a20c52d..31db6a4a187b 100644 --- a/src/runmodes.h +++ b/src/runmodes.h @@ -24,7 +24,7 @@ #define SURICATA_RUNMODES_H /* Run mode */ -enum RunModes { +typedef enum SCRunModes { RUNMODE_UNKNOWN = 0, RUNMODE_PCAP_DEV, RUNMODE_PCAP_FILE, @@ -60,7 +60,7 @@ enum RunModes { #endif RUNMODE_DUMP_FEATURES, RUNMODE_MAX, -}; +} SCRunMode; /* Run Mode Global Thread Names */ extern const char *thread_name_autofp; @@ -82,7 +82,7 @@ void RunModeListRunmodes(void); int RunModeEngineIsIPS(int capture_mode, const char *runmode, const char *capture_plugin_name); void RunModeDispatch(int, const char *, const char *capture_plugin_name, const char *capture_plugin_args); void RunModeRegisterRunModes(void); -void RunModeRegisterNewRunMode(enum RunModes, const char *, const char *, int (*RunModeFunc)(void), +void RunModeRegisterNewRunMode(SCRunMode, const char *, const char *, int (*RunModeFunc)(void), int (*RunModeIsIPSEnabled)(void)); void RunModeInitializeThreadSettings(void); void RunModeInitializeOutputs(void); @@ -91,8 +91,8 @@ void RunModeShutDown(void); /* bool indicating if filedata logger is enabled */ int RunModeOutputFiledataEnabled(void); /** bool indicating if run mode is offline */ -bool IsRunModeOffline(enum RunModes run_mode_to_check); -bool IsRunModeSystem(enum RunModes run_mode_to_check); +bool IsRunModeOffline(SCRunMode run_mode_to_check); +bool IsRunModeSystem(SCRunMode run_mode_to_check); void RunModeEnablesBypassManager(void); int RunModeNeedsBypassManager(void); diff --git a/src/suricata.c b/src/suricata.c index 8fdc48c93fd4..b8717542ba1d 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -258,12 +258,12 @@ int RunmodeIsUnittests(void) } #endif -int SCRunmodeGet(void) +SCRunMode SCRunmodeGet(void) { return suricata.run_mode; } -void SCRunmodeSet(int run_mode) +void SCRunmodeSet(SCRunMode run_mode) { suricata.run_mode = run_mode; } @@ -3053,8 +3053,3 @@ void SuricataPostInit(void) } SCPledge(); } - -void SuricataSetLibRunmode(void) -{ - suricata.run_mode = RUNMODE_LIB; -} diff --git a/src/suricata.h b/src/suricata.h index 285a1b2c43e8..ac7e71a4a9e2 100644 --- a/src/suricata.h +++ b/src/suricata.h @@ -121,8 +121,8 @@ enum { #include "runmodes.h" typedef struct SCInstance_ { - enum RunModes run_mode; - enum RunModes aux_run_mode; + enum SCRunModes run_mode; + enum SCRunModes aux_run_mode; char pcap_dev[128]; char *sig_file; @@ -189,14 +189,14 @@ int RunmodeIsUnittests(void); /** * \brief Get the current run mode. */ -int SCRunmodeGet(void); +SCRunMode SCRunmodeGet(void); /** * \brief Set the current run mode. * * Mainly exposed outside of suricata.c as a unit-test helper. */ -void SCRunmodeSet(int run_mode); +void SCRunmodeSet(SCRunMode run_mode); int SuriHasSigFile(void); @@ -225,7 +225,4 @@ int WindowsInitService(int argc, char **argv); const char *GetProgramVersion(void); -/* Library only methods. */ -void SuricataSetLibRunmode(void); - #endif /* SURICATA_SURICATA_H */ diff --git a/src/util-privs.c b/src/util-privs.c index 835247090dc1..da82234ba424 100644 --- a/src/util-privs.c +++ b/src/util-privs.c @@ -72,6 +72,8 @@ void SCDropMainThreadCaps(uint32_t userid, uint32_t groupid) CAP_SYS_NICE, -1); break; + default: + break; } if (capng_change_id(userid, groupid, CAPNG_DROP_SUPP_GRP | From 9990bfbd2c3d23624ed0e26d3d26626a38ed2654 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 12 Sep 2024 14:25:01 -0600 Subject: [PATCH 13/20] examples: update library example for ips drop Add a release packet callback where the action can be checked for drop. --- examples/lib/libcapture/main.c | 60 ++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index d86fcb0ace29..0f7cdb9b3185 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -22,9 +22,35 @@ #include "runmode-lib.h" #include "source-lib.h" #include "threadvars.h" +#include "action-globals.h" +#include "packet.h" static int worker_id = 1; +/** + * Release packet callback. + * + * If there is any cleanup that needs to be done when Suricata is done + * with a packet, this is the place to do it. + * + * Important: If using a custom release function, you must also + * release or free the packet. + * + * Optionally this is where you would handle IPS like functionality + * such as forwarding the packet, or triggering some other mechanism + * to forward the packet. + */ +static void ReleasePacket(Packet *p) +{ + if (PacketCheckAction(p, ACTION_DROP)) { + SCLogNotice("Dropping packet!"); + } + + /* As we overode the default release function, we must release or + * free the packet. */ + PacketFreeOrRelease(p); +} + /* Suricata worker thread in library mode. The functions should be wrapped in an API layer. */ static void *SimpleWorker(void *arg) @@ -56,14 +82,42 @@ static void *SimpleWorker(void *arg) struct pcap_pkthdr pkthdr; const u_char *packet; while ((packet = pcap_next(fp, &pkthdr)) != NULL) { - if (TmModuleLibHandlePacket(tv, device, packet, datalink, pkthdr.ts, pkthdr.len, 0, 0) != - 0) { - pthread_exit(NULL); + + Packet *p = PacketGetFromQueueOrAlloc(); + if (unlikely(p == NULL)) { + /* Memory allocation error. */ + goto done; + } + + /* If we are processing a PCAP and it is the first packet we need to set the timestamp. */ + SCTime_t timestamp = SCTIME_FROM_TIMEVAL(&pkthdr.ts); + if (count == 0) { + TmThreadsInitThreadsTimestamp(timestamp); + } + + /* Setup the packet, these will become functions to avoid + * internal Packet access. */ + PKT_SET_SRC(p, PKT_SRC_WIRE); + p->ts = SCTIME_FROM_TIMEVAL(&pkthdr.ts); + p->datalink = datalink; + p->livedev = device; + p->ReleasePacket = ReleasePacket; + + if (PacketSetData(p, packet, pkthdr.len) == -1) { + TmqhOutputPacketpool(tv, p); + goto done; + } + + if (TmThreadsSlotProcessPkt(tv, tv->tm_slots, p) != TM_ECODE_OK) { + TmqhOutputPacketpool(tv, p); + goto done; } (void)SC_ATOMIC_ADD(device->pkts, 1); count++; } + +done: pcap_close(fp); /* Cleanup. */ From 1bedfd6247fe4a9565c735985c9552e28a98967f Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 30 Sep 2024 16:34:04 -0600 Subject: [PATCH 14/20] examples/lib: create threadvars from main thread This also allows us to remove the sleep, as the ThreadVars are now guaranteed to be created before PostInit. --- examples/lib/libcapture/main.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index 0f7cdb9b3185..a0a2f1f649ba 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -27,6 +27,14 @@ static int worker_id = 1; +/** + * Struct to pass arguments into a worker thread. + */ +struct WorkerArgs { + ThreadVars *tv; + char *pcap_filename; +}; + /** * Release packet callback. * @@ -55,13 +63,8 @@ static void ReleasePacket(Packet *p) The functions should be wrapped in an API layer. */ static void *SimpleWorker(void *arg) { - char *pcap_file = (char *)arg; - - /* Create worker. */ - ThreadVars *tv = SCRunModeLibCreateThreadVars(worker_id++); - if (!tv) { - pthread_exit(NULL); - } + struct WorkerArgs *args = arg; + ThreadVars *tv = args->tv; /* Start worker. */ if (SCRunModeLibSpawnWorker(tv) != 0) { @@ -69,7 +72,7 @@ static void *SimpleWorker(void *arg) } /* Replay pcap. */ - pcap_t *fp = pcap_open_offline(pcap_file, NULL); + pcap_t *fp = pcap_open_offline(args->pcap_filename, NULL); if (fp == NULL) { pthread_exit(NULL); } @@ -190,15 +193,18 @@ int main(int argc, char **argv) as argument. This needs to be done in between SuricataInit and SuricataPostInit. */ pthread_t worker; - if (pthread_create(&worker, NULL, SimpleWorker, argv[argc - 1]) != 0) { + ThreadVars *tv = SCRunModeLibCreateThreadVars(worker_id++); + if (!tv) { + FatalError("Failed to create ThreadVars"); + } + struct WorkerArgs args = { + .tv = tv, + .pcap_filename = argv[argc - 1], + }; + if (pthread_create(&worker, NULL, SimpleWorker, &args) != 0) { exit(EXIT_FAILURE); } - /* Need to introduce a little sleep to allow the worker thread to - initialize before SuricataPostInit invokes TmThreadContinueThreads(). - This should be handle at the API level. */ - usleep(100); - SuricataPostInit(); /* Shutdown engine. */ From d015280b495f058d37bcc2924ce799160e7f67ea Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 4 Oct 2024 16:27:28 -0600 Subject: [PATCH 15/20] lib: remove SCRunModeLibDestroyWorker This was a one line wrapper around SCTmThreadsSlotPktAcqLoopFinish, so library users can call that directly instead. --- src/runmode-lib.c | 6 ------ src/runmode-lib.h | 3 --- 2 files changed, 9 deletions(-) diff --git a/src/runmode-lib.c b/src/runmode-lib.c index 94ed43b83432..c094f6f86803 100644 --- a/src/runmode-lib.c +++ b/src/runmode-lib.c @@ -104,9 +104,3 @@ int SCRunModeLibSpawnWorker(void *td) TmThreadsSetFlag(tv, THV_RUNNING); return 0; } - -/** \brief destroy a worker thread */ -void SCRunModeLibDestroyWorker(void *td) -{ - SCTmThreadsSlotPktAcqLoopFinish((ThreadVars *)td); -} diff --git a/src/runmode-lib.h b/src/runmode-lib.h index 0e37d3db4851..f6e5ff65fbfa 100644 --- a/src/runmode-lib.h +++ b/src/runmode-lib.h @@ -57,7 +57,4 @@ ThreadVars *SCRunModeLibCreateThreadVars(int worker_id); */ int SCRunModeLibSpawnWorker(void *); -/** \brief destroy a worker thread */ -void SCRunModeLibDestroyWorker(void *); - #endif /* SURICATA_RUNMODE_LIB_H */ From d7e623bc55635b67bcfe816baf24e16afbfc8620 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 4 Oct 2024 16:28:05 -0600 Subject: [PATCH 16/20] examples/lib: use a main loop to wait for exit Instead of immediately entering shutdown, use SuricataMainLoop to wait for the end of the pcap. --- examples/lib/libcapture/main.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index a0a2f1f649ba..5d2973e85c54 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -86,6 +86,11 @@ static void *SimpleWorker(void *arg) const u_char *packet; while ((packet = pcap_next(fp, &pkthdr)) != NULL) { + /* Have we been asked to stop? */ + if (suricata_ctl_flags & SURICATA_STOP) { + goto done; + } + Packet *p = PacketGetFromQueueOrAlloc(); if (unlikely(p == NULL)) { /* Memory allocation error. */ @@ -123,8 +128,17 @@ static void *SimpleWorker(void *arg) done: pcap_close(fp); - /* Cleanup. */ - SCRunModeLibDestroyWorker(tv); + /* Stop the engine. */ + EngineStop(); + + /* Cleanup. + * + * Note that there is some thread synchronization between this + * function and SuricataShutdown such that they must be run + * concurrently at this time before either will exit. */ + SCTmThreadsSlotPktAcqLoopFinish(tv); + + SCLogNotice("Worker thread exiting"); pthread_exit(NULL); } @@ -207,7 +221,17 @@ int main(int argc, char **argv) SuricataPostInit(); + /* Run the main loop, this just waits for the worker thread to + * call EngineStop signalling Suricata that it is done reading the + * pcap. */ + SuricataMainLoop(); + /* Shutdown engine. */ + SCLogNotice("Shutting down"); + + /* Note that there is some thread synchronization between this + * function and SCTmThreadsSlotPktAcqLoopFinish that require them + * to be run concurrently at this time. */ SuricataShutdown(); GlobalsDestroy(); From b55e416e76d09627f852111463000b0eff973eb4 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 7 Oct 2024 16:08:59 -0600 Subject: [PATCH 17/20] source-lib: remove TmModuleLibHandlePacket This is better handled directly in the library user program so the user has more control of the Packet structure, before and after handling. --- src/source-lib.c | 56 ------------------------------------------------ src/source-lib.h | 15 ------------- 2 files changed, 71 deletions(-) diff --git a/src/source-lib.c b/src/source-lib.c index 822d7668f3f1..a9d3e9007f1d 100644 --- a/src/source-lib.c +++ b/src/source-lib.c @@ -101,62 +101,6 @@ static TmEcode DecodeLib(ThreadVars *tv, Packet *p, void *data) SCReturnInt(TM_ECODE_OK); } -/** \brief process a single packet. - * - * \param tv Pointer to the per-thread structure. - * \param device Pionter to LiveDevice instance - * \param data Pointer to the raw packet. - * \param datalink Datalink type. - * \param ts Timeval structure. - * \param len Packet length. - * \param tenant_id Tenant id of the detection engine to use. - * \param flags Packet flags (packet checksum, rule profiling...). - * \return Error code. - */ -int TmModuleLibHandlePacket(ThreadVars *tv, LiveDevice *device, const uint8_t *data, int datalink, - struct timeval ts, uint32_t len, uint32_t tenant_id, uint32_t flags) -{ - - /* If the packet is NULL, consider it as a read timeout. */ - if (data == NULL) { - TmThreadsSetFlag(tv, THV_CAPTURE_INJECT_PKT); - TmThreadsCaptureHandleTimeout(tv, NULL); - SCReturnInt(TM_ECODE_OK); - } - - Packet *p = PacketGetFromQueueOrAlloc(); - if (unlikely(p == NULL)) { - SCReturnInt(TM_ECODE_FAILED); - } - - /* If we are processing a PCAP and it is the first packet we need to set the timestamp. */ - SCTime_t timestamp = SCTIME_FROM_TIMEVAL(&ts); - if (!time_set && !TimeModeIsLive()) { - TmThreadsInitThreadsTimestamp(timestamp); - time_set = true; - } - - PKT_SET_SRC(p, PKT_SRC_WIRE); - p->ts = timestamp; - p->datalink = datalink; - p->tenant_id = tenant_id; - p->flags |= flags; - p->livedev = device; - - if (PacketSetData(p, data, len) == -1) { - TmqhOutputPacketpool(tv, p); - SCReturnInt(TM_ECODE_FAILED); - } - - SCLogDebug("pktlen: %" PRIu32 " (pkt %p, pkt data %p)", GET_PKT_LEN(p), p, GET_PKT_DATA(p)); - - if (TmThreadsSlotProcessPkt(tv, tv->tm_slots, p) != TM_ECODE_OK) { - SCReturnInt(TM_ECODE_FAILED); - } - - SCReturnInt(TM_ECODE_OK); -} - /** \brief register a "Decode" module for suricata as a library. * * The "Decode" module is the first module invoked when processing a packet */ diff --git a/src/source-lib.h b/src/source-lib.h index 432f12aa505b..76cad144e295 100644 --- a/src/source-lib.h +++ b/src/source-lib.h @@ -34,19 +34,4 @@ * The "Decode" module is the first module invoked when processing a packet */ void TmModuleDecodeLibRegister(void); -/** \brief process a single packet. - * - * \param tv Pointer to the per-thread structure. - * \param data Pointer to the raw packet. - * \param datalink Datalink type. - * \param ts Timeval structure. - * \param len Packet length. - * \param tenant_id Tenant id of the detection engine to use. - * \param flags Packet flags (packet checksum, rule profiling...). - * \param iface Sniffing interface this packet comes from (can be NULL). - * \return Error code. - */ -int TmModuleLibHandlePacket(ThreadVars *tv, LiveDevice *device, const uint8_t *data, int datalink, - struct timeval ts, uint32_t len, uint32_t tenant_id, uint32_t flags); - #endif /* SURICATA_SOURCE_LIB_H */ From fe54fe5c1893bdb6217454a99ac343adc78cbc5a Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 9 Oct 2024 12:30:43 -0600 Subject: [PATCH 18/20] packet: add set functions for some packet fields - SCPacketSetReleasePacket - SCPacketSetLiveDevice - SCPacketSetDatalink - SCPacketSetTime - SCPacketSetSource Prevents direct access by library users and provides more ABI stability. --- src/packet.c | 25 +++++++++++++++++++++++++ src/packet.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/packet.c b/src/packet.c index cb6dcf618380..844506783a37 100644 --- a/src/packet.c +++ b/src/packet.c @@ -161,3 +161,28 @@ void PacketDestructor(Packet *p) AppLayerDecoderEventsFreeEvents(&p->app_layer_events); PACKET_PROFILING_RESET(p); } + +inline void SCPacketSetReleasePacket(Packet *p, void (*ReleasePacket)(Packet *p)) +{ + p->ReleasePacket = ReleasePacket; +} + +inline void SCPacketSetLiveDevice(Packet *p, LiveDevice *device) +{ + p->livedev = device; +} + +inline void SCPacketSetDatalink(Packet *p, int datalink) +{ + p->datalink = datalink; +} + +inline void SCPacketSetTime(Packet *p, SCTime_t ts) +{ + p->ts = ts; +} + +inline void SCPacketSetSource(Packet *p, enum PktSrcEnum source) +{ + p->pkt_src = source; +} diff --git a/src/packet.h b/src/packet.h index 0b03a5963192..633b118d3bf3 100644 --- a/src/packet.h +++ b/src/packet.h @@ -19,6 +19,7 @@ #define SURICATA_PACKET_H #include "decode.h" +#include "util-device.h" void PacketDrop(Packet *p, const uint8_t action, enum PacketDropReason r); bool PacketCheckAction(const Packet *p, const uint8_t a); @@ -36,4 +37,31 @@ void PacketReinit(Packet *p); void PacketRecycle(Packet *p); void PacketDestructor(Packet *p); +/** \brief Set a packet release function. + * + * Set a custom release function for packet. This is required if extra + * non-standard packet was done that needs to be cleaned up when + * Suricata is done with a packet. + * + * Its also where IPS actions may be done. + */ +void SCPacketSetReleasePacket(Packet *p, void (*ReleasePacket)(Packet *p)); + +/** \brief Set a packets live device. */ +void SCPacketSetLiveDevice(Packet *p, LiveDevice *device); + +/** \brief Set a packets data link type. */ +void SCPacketSetDatalink(Packet *p, int datalink); + +/** \brief Set the timestamp for a packet. + * + * \param ts A timestamp in SCTime_t format. See SCTIME_FROM_TIMEVAL + * for conversion from struct timeval. + */ +void SCPacketSetTime(Packet *p, SCTime_t ts); + +/** \brief Set packet source. + */ +void SCPacketSetSource(Packet *p, enum PktSrcEnum source); + #endif From ab60fc0cea07b3f80b1a842582929c3fc1595784 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Wed, 9 Oct 2024 15:21:32 -0600 Subject: [PATCH 19/20] examples: use packet setter functions Instead of direct field access. --- examples/lib/libcapture/main.c | 10 +++++----- examples/plugins/ci-capture/source.c | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index 5d2973e85c54..99be01b39c9a 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -105,11 +105,11 @@ static void *SimpleWorker(void *arg) /* Setup the packet, these will become functions to avoid * internal Packet access. */ - PKT_SET_SRC(p, PKT_SRC_WIRE); - p->ts = SCTIME_FROM_TIMEVAL(&pkthdr.ts); - p->datalink = datalink; - p->livedev = device; - p->ReleasePacket = ReleasePacket; + SCPacketSetSource(p, PKT_SRC_WIRE); + SCPacketSetTime(p, SCTIME_FROM_TIMEVAL(&pkthdr.ts)); + SCPacketSetDatalink(p, datalink); + SCPacketSetLiveDevice(p, device); + SCPacketSetReleasePacket(p, ReleasePacket); if (PacketSetData(p, packet, pkthdr.len) == -1) { TmqhOutputPacketpool(tv, p); diff --git a/examples/plugins/ci-capture/source.c b/examples/plugins/ci-capture/source.c index c6953b2f2590..c6bce18613b4 100644 --- a/examples/plugins/ci-capture/source.c +++ b/examples/plugins/ci-capture/source.c @@ -20,6 +20,7 @@ #include "tm-modules.h" #include "tm-threads-common.h" #include "tm-threads.h" +#include "packet.h" #include "source.h" @@ -69,11 +70,11 @@ static TmEcode ReceiveLoop(ThreadVars *tv, void *data, void *slot) if (unlikely(p == NULL)) { return TM_ECODE_FAILED; } - PKT_SET_SRC(p, PKT_SRC_WIRE); + SCPacketSetSource(p, PKT_SRC_WIRE); struct timeval now; gettimeofday(&now, NULL); - p->ts = SCTIME_FROM_TIMEVAL(&now); - p->datalink = LINKTYPE_ETHERNET; + SCPacketSetTime(p, SCTIME_FROM_TIMEVAL(&now)); + SCPacketSetDatalink(p, LINKTYPE_ETHERNET); p->flags |= PKT_IGNORE_CHECKSUM; if (unlikely(PacketCopyData(p, DNS_REQUEST, sizeof(DNS_REQUEST)) != 0)) { From 32442e294632cd974e86a86ff3a07e6fd8a709c4 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 14 Jan 2025 16:26:21 -0600 Subject: [PATCH 20/20] examples/libcapture: include cleanup Requires tm-threads.h. --- examples/lib/libcapture/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/lib/libcapture/main.c b/examples/lib/libcapture/main.c index 99be01b39c9a..bc844d528019 100644 --- a/examples/lib/libcapture/main.c +++ b/examples/lib/libcapture/main.c @@ -20,7 +20,7 @@ #include "conf.h" #include "pcap.h" #include "runmode-lib.h" -#include "source-lib.h" +#include "tm-threads.h" #include "threadvars.h" #include "action-globals.h" #include "packet.h"