Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

detect: set mpm/prefilter during signature parsing #12382

Closed
wants to merge 1 commit into from
Closed

detect: set mpm/prefilter during signature parsing #12382

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
detect: set mpm/prefilter during signature parsing 
In preparation of flowbit prefilter work that needs this info
earlier.

Track potential prefilter sm's to avoid unnecessary looping during
setup.
  • Loading branch information
@victorjulien
victorjulien committed Jan 15, 2025
commit bd3e65b6de4e3dc76f15d9fab0a0f93821ceefb0
84 changes: 31 additions & 53 deletions src/detect-byte-extract.c
View file
Original file line number Diff line number Diff line change
@@ -1945,9 +1945,8 @@ static int DetectByteExtractTest43(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_OFFSET_VAR |
DETECT_CONTENT_OFFSET) ||
cd->offset != bed->local_id) {
cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) ||
cd->offset != bed->local_id) {
printf("three failed\n");
result = 0;
goto end;
@@ -2049,9 +2048,8 @@ static int DetectByteExtractTest44(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_OFFSET_VAR |
DETECT_CONTENT_OFFSET) ||
cd->offset != bed1->local_id) {
cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) ||
cd->offset != bed1->local_id) {
printf("four failed\n");
result = 0;
goto end;
@@ -2158,10 +2156,8 @@ static int DetectByteExtractTest45(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DEPTH_VAR |
DETECT_CONTENT_DEPTH) ||
cd->depth != bed->local_id ||
cd->offset != 0) {
cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) ||
cd->depth != bed->local_id || cd->offset != 0) {
printf("three failed\n");
result = 0;
goto end;
@@ -2263,9 +2259,8 @@ static int DetectByteExtractTest46(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DEPTH_VAR |
DETECT_CONTENT_DEPTH) ||
cd->depth != bed1->local_id) {
cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) ||
cd->depth != bed1->local_id) {
printf("four failed\n");
result = 0;
goto end;
@@ -2372,11 +2367,9 @@ static int DetectByteExtractTest47(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
DETECT_CONTENT_DISTANCE) ||
cd->distance != bed->local_id ||
cd->offset != 0 ||
cd->depth != 0) {
cd->flags !=
(DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM) ||
cd->distance != bed->local_id || cd->offset != 0 || cd->depth != 0) {
printf("three failed\n");
result = 0;
goto end;
@@ -2478,12 +2471,9 @@ static int DetectByteExtractTest48(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
DETECT_CONTENT_DISTANCE |
DETECT_CONTENT_DISTANCE_NEXT) ||
cd->distance != bed1->local_id ||
cd->depth != 0 ||
cd->offset != 0) {
cd->flags != (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE |
DETECT_CONTENT_DISTANCE_NEXT | DETECT_CONTENT_MPM) ||
cd->distance != bed1->local_id || cd->depth != 0 || cd->offset != 0) {
printf("four failed\n");
result = 0;
goto end;
@@ -2592,12 +2582,8 @@ static int DetectByteExtractTest49(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_WITHIN) ||
cd->within != bed->local_id ||
cd->offset != 0 ||
cd->depth != 0 ||
cd->distance != 0) {
cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) ||
cd->within != bed->local_id || cd->offset != 0 || cd->depth != 0 || cd->distance != 0) {
printf("three failed\n");
result = 0;
goto end;
@@ -2699,13 +2685,10 @@ static int DetectByteExtractTest50(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_WITHIN|
DETECT_CONTENT_WITHIN_NEXT) ||
cd->within != bed1->local_id ||
cd->depth != 0 ||
cd->offset != 0 ||
cd->distance != 0) {
cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN |
DETECT_CONTENT_WITHIN_NEXT | DETECT_CONTENT_MPM) ||
cd->within != bed1->local_id || cd->depth != 0 || cd->offset != 0 ||
cd->distance != 0) {
printf("four failed\n");
result = 0;
goto end;
@@ -2970,7 +2953,7 @@ static int DetectByteExtractTest53(void)
SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_PMATCH];
FAIL_IF(sm->type != DETECT_CONTENT);
DetectContentData *cd = (DetectContentData *)sm->ctx;
FAIL_IF(cd->flags != 0);
FAIL_IF(cd->flags != (DETECT_CONTENT_MPM | DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED));

sm = sm->next;
FAIL_IF_NULL(sm);
@@ -3092,11 +3075,8 @@ static int DetectByteExtractTest54(void)
goto end;
}
bjd = (DetectBytejumpData *)sm->ctx;
if (bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR || bjd->offset != 1) {
printf("four failed\n");
result = 0;
goto end;
}
FAIL_IF(bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR);
FAIL_IF(bjd->offset != 1);

if (sm->next != NULL)
goto end;
@@ -3197,12 +3177,10 @@ static int DetectByteExtractTest55(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_DISTANCE |
DETECT_CONTENT_WITHIN) ||
cd->within != bed1->local_id ||
cd->distance != bed2->local_id) {
cd->flags !=
(DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) ||
cd->within != bed1->local_id || cd->distance != bed2->local_id) {
printf("four failed: ");
goto end;
}
@@ -3829,8 +3807,8 @@ static int DetectByteExtractTest60(void)
goto end;
}
cd = (DetectContentData *)sm->ctx;
if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
printf("one failed\n");
result = 0;
goto end;
@@ -3950,8 +3928,8 @@ static int DetectByteExtractTest61(void)
goto end;
}
cd = (DetectContentData *)sm->ctx;
if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
printf("one failed\n");
result = 0;
goto end;
2 changes: 1 addition & 1 deletion src/detect-content.c
View file
Original file line number Diff line number Diff line change
@@ -1504,7 +1504,7 @@ static int DetectContentParseTest19(void)
FAIL_IF_NOT(s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL);

DetectContentData *data = (DetectContentData *)sm->ctx;
FAIL_IF_NOT(data->flags == DETECT_CONTENT_DISTANCE);
FAIL_IF_NOT(data->flags == (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM));

s = DetectEngineAppendSig(de_ctx,
"alert tcp any any -> any any "
34 changes: 0 additions & 34 deletions src/detect-engine-build.c
View file
Original file line number Diff line number Diff line change
@@ -1703,7 +1703,6 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
}
}

extern int g_skip_prefilter;
/**
* \brief Preprocess signature, classify ip-only, etc, build sig array
*
@@ -1787,39 +1786,6 @@ int SigPrepareStage1(DetectEngineCtx *de_ctx)

RuleSetScore(s);

/* if keyword engines are enabled in the config, handle them here */
if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
!(s->flags & SIG_FLAG_PREFILTER)) {
int prefilter_list = DETECT_TBLSIZE;

// TODO buffers?

/* get the keyword supporting prefilter with the lowest type */
for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
if (sigmatch_table[sm->type].SupportsPrefilter(s)) {
prefilter_list = MIN(prefilter_list, sm->type);
}
}
}
}

/* apply that keyword as prefilter */
if (prefilter_list != DETECT_TBLSIZE) {
for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
if (sm->type == prefilter_list) {
s->init_data->prefilter_sm = sm;
s->flags |= SIG_FLAG_PREFILTER;
SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name);
break;
}
}
}
}
}

/* run buffer type callbacks if any */
for (int x = 0; x < DETECT_SM_LIST_MAX; x++) {
if (s->init_data->smlists[x])
1 change: 0 additions & 1 deletion src/detect-engine-loader.c
View file
Original file line number Diff line number Diff line change
@@ -167,7 +167,6 @@ static int DetectLoadSigFile(
sig = DetectEngineAppendSig(de_ctx, line);
if (sig != NULL) {
if (rule_engine_analysis_set || fp_engine_analysis_set) {
RetrieveFPForSig(de_ctx, sig);
if (fp_engine_analysis_set) {
EngineAnalysisFP(de_ctx, sig, line);
}
5 changes: 0 additions & 5 deletions src/detect-engine-mpm.c
View file
Original file line number Diff line number Diff line change
@@ -2418,12 +2418,7 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
{
uint32_t cnt = 0;
for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
if (s->flags & SIG_FLAG_PREFILTER)
continue;

RetrieveFPForSig(de_ctx, s);
if (s->init_data->mpm_sm != NULL) {
s->flags |= SIG_FLAG_PREFILTER;
cnt++;
}
}
4 changes: 4 additions & 0 deletions src/detect-flow.c
View file
Original file line number Diff line number Diff line change
@@ -445,14 +445,18 @@ void DetectFlowFree(DetectEngineCtx *de_ctx, void *ptr)
static void
PrefilterPacketFlowMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx)
{
SCEnter();

const PrefilterPacketHeaderCtx *ctx = pectx;

if (!PrefilterPacketHeaderExtraMatch(ctx, p))
return;

if (FlowMatch(p->flags, p->flowflags, ctx->v1.u16[0], ctx->v1.u16[1])) {
SCLogDebug("match: adding sids");
PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt);
}
SCReturn;
}

static void
61 changes: 60 additions & 1 deletion src/detect-parse.c
View file
Original file line number Diff line number Diff line change
@@ -914,6 +914,9 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr,
}
s->init_data->negated = false;

const enum DetectKeywordId idx = SigTableGetIndex(st);
s->init_data->has_possible_prefilter |= de_ctx->sm_types_prefilter[idx];

if (st->flags & SIGMATCH_INFO_DEPRECATED) {
#define URL "https://suricata.io/our-story/deprecation-policy/"
if (st->alternative == 0)
@@ -1021,7 +1024,6 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr,

/* handle 'silent' error case */
if (setup_ret == -2) {
enum DetectKeywordId idx = SigTableGetIndex(st);
if (de_ctx->sm_types_silent_error[idx] == false) {
de_ctx->sm_types_silent_error[idx] = true;
return -1;
@@ -1877,6 +1879,61 @@ SigMatchData* SigMatchList2DataArray(SigMatch *head)
return out;
}

extern int g_skip_prefilter;

static void SigSetupPrefilter(DetectEngineCtx *de_ctx, Signature *s)
{
SCEnter();
if (s->init_data->prefilter_sm != NULL || s->init_data->mpm_sm != NULL) {
SCReturn;
}

SCLogDebug("s %u: set up prefilter/mpm", s->id);
RetrieveFPForSig(de_ctx, s);
if (s->init_data->mpm_sm != NULL) {
s->flags |= SIG_FLAG_PREFILTER;
SCReturn;
}

SCLogDebug("s %u: no mpm; prefilter? de_ctx->prefilter_setting %u "
"s->init_data->has_possible_prefilter %s",
s->id, de_ctx->prefilter_setting, BOOL2STR(s->init_data->has_possible_prefilter));

if (!s->init_data->has_possible_prefilter)
SCReturn;

if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
!(s->flags & SIG_FLAG_PREFILTER)) {
int prefilter_list = DETECT_TBLSIZE;
/* get the keyword supporting prefilter with the lowest type */
for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
if (sigmatch_table[sm->type].SupportsPrefilter(s)) {
prefilter_list = MIN(prefilter_list, sm->type);
}
}
}
}

/* apply that keyword as prefilter */
if (prefilter_list != DETECT_TBLSIZE) {
for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
if (sm->type == prefilter_list) {
s->init_data->prefilter_sm = sm;
s->flags |= SIG_FLAG_PREFILTER;
SCLogConfig("sid %u: prefilter is on \"%s\"", s->id,
sigmatch_table[sm->type].name);
break;
}
}
}
}
}
SCReturn;
}

/**
* \internal
* \brief validate a just parsed signature for internal inconsistencies
@@ -2219,6 +2276,8 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, const char *sigstr,
DetectEngineBufferRunSetupCallback(de_ctx, sig->init_data->buffers[x].id, sig);
}

SigSetupPrefilter(de_ctx, sig);

/* validate signature, SigValidate will report the error reason */
if (SigValidate(de_ctx, sig) == 0) {
goto error;
3 changes: 3 additions & 0 deletions src/detect.h
View file
Original file line number Diff line number Diff line change
@@ -551,6 +551,9 @@ typedef struct SignatureInitData_ {
bool src_contains_negation;
bool dst_contains_negation;

/** see if any of the sigmatches supports an enabled prefilter */
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

only used in debug logging ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm seems so ya

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should it be under #ifdef ? Or used in some other place ? or just removed as a development artifact ?

bool has_possible_prefilter;

/* used to hold flags that are used during init */
uint32_t init_flags;
/* coccinelle: SignatureInitData:init_flags:SIG_FLAG_INIT_ */
Loading
Loading