From b06ef592588442555a61450a036c343115c5f9fb Mon Sep 17 00:00:00 2001 From: Andreas Herz Date: Tue, 11 Jun 2024 17:44:02 +0200 Subject: [PATCH 1/4] tests: add test to verify unmatched signatures does not write to dataset Related to https://redmine.openinfosecfoundation.org/issues/5576 --- tests/datasets-validate-postmatch/README.md | 7 ++++++ tests/datasets-validate-postmatch/input.pcap | Bin 0 -> 1929 bytes tests/datasets-validate-postmatch/test.rules | 4 +++ tests/datasets-validate-postmatch/test.yaml | 25 +++++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 tests/datasets-validate-postmatch/README.md create mode 100644 tests/datasets-validate-postmatch/input.pcap create mode 100644 tests/datasets-validate-postmatch/test.rules create mode 100644 tests/datasets-validate-postmatch/test.yaml diff --git a/tests/datasets-validate-postmatch/README.md b/tests/datasets-validate-postmatch/README.md new file mode 100644 index 000000000..a8ccb5a9a --- /dev/null +++ b/tests/datasets-validate-postmatch/README.md @@ -0,0 +1,7 @@ +# Description + +Test datasets only sets when there is a full signature match. + +# Ticket + +https://redmine.openinfosecfoundation.org/issues/5576 diff --git a/tests/datasets-validate-postmatch/input.pcap b/tests/datasets-validate-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a554c13c8469891db8ed26baf6c4a2625cb99f89 GIT binary patch literal 1929 zcmaKtc~BEq9LL}0Lej1Q2P)8ETn{v$A-hWsvbDt|M1l|uC6U9UlP1fug4vj?fG|@j zvGmZwP)o7xjA*B{cI-^OYKvnlRXpnWhgUu7(GEI9rFLx9X^V{XZBi-*#+|&G&D-}r z@B96}_uD;pc=t9DkbwWM1ppxM58Cyf>l9D`6TBB1NL-G@CxWRD0f?CdEC6VRXDt9t zGJT2s)i~1NFv>fh{cl-L?W6DiJi$2T9_DSMYs{7 zBLsj}# ziNmj}`ZRK36<)#Dq>-kE>=M?VudlO|E~(G0DK2(8*hS8K z6T3varf7|pfg^Qrl)t(}UtC_cvZ2~+F&28O%dC3FZ1vgUqrwVCQ)F}6jA>c9nAPF1 ztBFhkm76)Qmv-|k=cTX$n)a!THEb=7%8hO}?c*sdL!E)jodMdfGBUImcGy_Ap9^r6 zyvpof>*G12Jzq`WnK&x9asl|9a|e8x9+qL^RSgH z?U`sToWcqCi;OOz{k628!Yt+zOzEz2sj~@#Asv;Mslt{i%$g4s3C0C~g|wI9t0+vP zhvx>BAa7-~H&~F;(OGgsNLHtpnYqUu{gj+HARd`os_Q z^|a-*hEKR}{l3zDv}$TkZ>T`C{f+$w(a*2V|Lx#O#ftkG)th&&B7e-jL}_k{+qUMg zpEv#^Px)eZ>vuV+-<-LoRurFGFTH*5kh|0L(x7GFoofR((|7frH9b3|yZ5|r(*y8k zCVihyn0~eI^2#PdSD!vtasHh0dh;Vjksq+{t3BFzr0o6nwik>9*%3PS_oIf=tPOEj z(>C3D*t_06UntdGbC9RKDCdM$COSS#*Xu)40A{66_F_h>Ji*i^Q*0S_zTGl`hiloTCm}aFjkR%26M5bip1Ef>$ylln+|QP{wP?2x}>i(sBW&dnBChZUFuTp_xv{ literal 0 HcmV?d00001 diff --git a/tests/datasets-validate-postmatch/test.rules b/tests/datasets-validate-postmatch/test.rules new file mode 100644 index 000000000..a028fa089 --- /dev/null +++ b/tests/datasets-validate-postmatch/test.rules @@ -0,0 +1,4 @@ +# this will not match, and thus not write to the dataset +alert http any any -> $HOME_NET any (msg:"HTTP learning"; flow:established,to_client; http.content_type; content:"noone"; http.server; content:"ECS"; fast_pattern; dataset:set,http_none,type string,state output/none.intel; sid:2; rev:1; priority:2;) +# this will match, and thus write to the dataset +alert http any any -> $HOME_NET any (msg:"HTTP learning"; flow:established,to_client; http.content_type; content:"ocsp"; http.server; content:"ECS"; fast_pattern; dataset:set,http_match,type string,state output/match.intel; sid:3; rev:1; priority:2;) diff --git a/tests/datasets-validate-postmatch/test.yaml b/tests/datasets-validate-postmatch/test.yaml new file mode 100644 index 000000000..4f4e4b73e --- /dev/null +++ b/tests/datasets-validate-postmatch/test.yaml @@ -0,0 +1,25 @@ +requires: + min-version: 8 + +args: + - -k none + +checks: + # only sid 3 should trigger + - filter: + count: 1 + match: + event_type: alert + alert.signature_id: 3 + - filter: + count: 0 + match: + event_type: alert + alert.signature_id: 2 + # data should only be written to one dataset + - shell: + args: cat none.intel | wc -l | xargs + expect: 0 + - shell: + args: cat match.intel | wc -l | xargs + expect: 1 From a1994b186c51ac244f1a1fc666596059b47f1dbe Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 15 Oct 2024 09:46:40 +0200 Subject: [PATCH 2/4] datasets: test with multibuffer and set postmatch Ticket: 5576 --- tests/datasets-multibuf-postmatch/README.md | 8 ++++++++ .../expected/http_match.csv | 2 ++ tests/datasets-multibuf-postmatch/input.pcap | Bin 0 -> 1708 bytes tests/datasets-multibuf-postmatch/test.rules | 2 ++ tests/datasets-multibuf-postmatch/test.yaml | 15 +++++++++++++++ 5 files changed, 27 insertions(+) create mode 100644 tests/datasets-multibuf-postmatch/README.md create mode 100644 tests/datasets-multibuf-postmatch/expected/http_match.csv create mode 100644 tests/datasets-multibuf-postmatch/input.pcap create mode 100644 tests/datasets-multibuf-postmatch/test.rules create mode 100644 tests/datasets-multibuf-postmatch/test.yaml diff --git a/tests/datasets-multibuf-postmatch/README.md b/tests/datasets-multibuf-postmatch/README.md new file mode 100644 index 000000000..c0a25a38b --- /dev/null +++ b/tests/datasets-multibuf-postmatch/README.md @@ -0,0 +1,8 @@ +# Description + +Test datasets only sets when there is a full signature match. +Test is with multibuffer where some are matching, but not other. + +# Ticket + +https://redmine.openinfosecfoundation.org/issues/5576 diff --git a/tests/datasets-multibuf-postmatch/expected/http_match.csv b/tests/datasets-multibuf-postmatch/expected/http_match.csv new file mode 100644 index 000000000..752b88213 --- /dev/null +++ b/tests/datasets-multibuf-postmatch/expected/http_match.csv @@ -0,0 +1,2 @@ +WC10aXRpOiB0b3RvMg== +WC10b3RvOiB2YWx1ZTE= diff --git a/tests/datasets-multibuf-postmatch/input.pcap b/tests/datasets-multibuf-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..1c4ed0f5760eb42e59bcc4ea4cd750e4876136f4 GIT binary patch literal 1708 zcmbW1(QDH{9LLkH%yAHLzF2UwgZUuanxtL3)*!fH%LZ<2tPyo*%%J= zFWB}6%m-n(7s2V1;KoE>6@2s`5EP$8olkcDu4(8ttE{{B;4b&O?|pxt{4Uw|FCNE* zgwSr`_)YR`%^XMwMOffHYH=8p1Yv^zCwc|pfgr?qZ;TACZ*A<{=W*NdgM&s^5aNmJ zA7ec|y|Gvyx-kCXB40~@0M~xjh4$cjV`Q^$;V0;?Y@h#I`%dclcdj34zXI(7oW*<8 zTE4+mN8c-;zyI)D@@wM?B-8TCb-CBKUFfyE?sKn)^&*d44_!(KJ7A3WsI~mP{I9$%CJlqQ@Nifj7SZh#bC{#zMcoPrXu(`(x{6uAg651e3Nwhwh)6^{r)!YY zNtw49?Fzu?OT_#zqm{t~ya%3ZLjDn0~Fv3zk^N3z=$ITGC#!MBY@*hYZ-rL%M7so;W zItqFS7m06p)a3vW(umJ+1uDlU%*T#xE#u~B$s{&rCh-WSKGtzHuq^D;nys?{p4D|` zm{PbK)|V(VaXqjYy&F`k#KW*1qI{=LupWg|yEcmN1`OK{!w%P$5E{Z7HSQ6Hkja?q ztFr8oN}bw7a$I6lyXyGFs5*fS9>K8Wcr}^&en5OVm&=bg>-x#5aBHi8`%@>}ue;C- z>sniF;_mN!Pu_K**DeQb?O*;b`W5>P=KcHj>{s8uY BlAZtn literal 0 HcmV?d00001 diff --git a/tests/datasets-multibuf-postmatch/test.rules b/tests/datasets-multibuf-postmatch/test.rules new file mode 100644 index 000000000..9636b1e5b --- /dev/null +++ b/tests/datasets-multibuf-postmatch/test.rules @@ -0,0 +1,2 @@ +# this will match, and thus write some headers to the dataset +alert http any any -> any any (msg:"HTTP learning"; http.request_header; content:"toto"; dataset:set,http_match,type string,save http_match.csv; sid:1;) diff --git a/tests/datasets-multibuf-postmatch/test.yaml b/tests/datasets-multibuf-postmatch/test.yaml new file mode 100644 index 000000000..cc30d2721 --- /dev/null +++ b/tests/datasets-multibuf-postmatch/test.yaml @@ -0,0 +1,15 @@ +requires: + min-version: 8 + +args: + - -k none --no-random --data-dir=${OUTPUT_DIR} + +checks: + - filter: + count: 1 + match: + event_type: alert + alert.signature_id: 1 + - file-compare: + filename: http_match.csv + expected: expected/http_match.csv From f69561b7c9318dfbd64ebd030b98ec67d6cb8f93 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 1 Aug 2024 21:21:13 +0200 Subject: [PATCH 3/4] datasets: test with delayed set postmatch Ticket: 5576 Signature full match does not happen on first packet inspected, but signature gets stored as partially matching so far, and then postmatch retreieves the buffer to set in the dataset. --- tests/datasets-delayed-postmatch/README.md | 14 ++++++++++++ .../expected/http_match.csv | 1 + tests/datasets-delayed-postmatch/input.pcap | Bin 0 -> 1182 bytes tests/datasets-delayed-postmatch/test.rules | 2 ++ tests/datasets-delayed-postmatch/test.yaml | 21 ++++++++++++++++++ 5 files changed, 38 insertions(+) create mode 100644 tests/datasets-delayed-postmatch/README.md create mode 100644 tests/datasets-delayed-postmatch/expected/http_match.csv create mode 100644 tests/datasets-delayed-postmatch/input.pcap create mode 100644 tests/datasets-delayed-postmatch/test.rules create mode 100644 tests/datasets-delayed-postmatch/test.yaml diff --git a/tests/datasets-delayed-postmatch/README.md b/tests/datasets-delayed-postmatch/README.md new file mode 100644 index 000000000..a073a6dd2 --- /dev/null +++ b/tests/datasets-delayed-postmatch/README.md @@ -0,0 +1,14 @@ +Test +==== + +Test datasets only sets when there is a full signature match. +Test is with a signature using different keywords matching at different stages, +and pcap having different packets making the transaction progress step by step. + +https://redmine.openinfosecfoundation.org/issues/5576 + +PCAP +==== + +Pcap crafted with some http server and some python client that delays or not the writing of the headers + diff --git a/tests/datasets-delayed-postmatch/expected/http_match.csv b/tests/datasets-delayed-postmatch/expected/http_match.csv new file mode 100644 index 000000000..352340277 --- /dev/null +++ b/tests/datasets-delayed-postmatch/expected/http_match.csv @@ -0,0 +1 @@ +L3BhZ2U/cGFyYW09dmFsdWU= diff --git a/tests/datasets-delayed-postmatch/input.pcap b/tests/datasets-delayed-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..41fc99473f59ea6b92a2b1c74e4157aaa7c365a8 GIT binary patch literal 1182 zcmb7^L2DC17>2*3q3S?DS`UE=otse-lT90vu7qZ5OIM;;%AyAkGOW|AuqJ7`5rv>V zOalC4eGb#c_a#03pim)U>g856+iQMhY zld+%sagz;mox4?AxDMVjdhD^sqig8XUqJmIK@EX^962|i0hp*xQTjLm z#bL+3(5sX5v3tMHLX%jg6t`qu(|ONZW5j)q+)>BW=eIiOr)}k0h2~l0Tzd;}3Ee2l zuQ!=P#&Y$y-(28lINgr6<@;rnUxqOPw-Edvg7%NEYY6bB|BAy#s>fM5g`Cp>zX0SpB}o7P literal 0 HcmV?d00001 diff --git a/tests/datasets-delayed-postmatch/test.rules b/tests/datasets-delayed-postmatch/test.rules new file mode 100644 index 000000000..d9706cc67 --- /dev/null +++ b/tests/datasets-delayed-postmatch/test.rules @@ -0,0 +1,2 @@ +alert http any any -> any any ( sid: 1; http.uri; content: "param"; fast_pattern; pcre: "/param=(.*)/,flow:paramval"; http.request_header; content: "Header1";) +alert http any any -> any any ( sid: 2; http.uri; content: "param"; fast_pattern; dataset:set,http_match,type string,save http_match.csv; http.method; content: "GET"; http.request_header; content: "Header1";) diff --git a/tests/datasets-delayed-postmatch/test.yaml b/tests/datasets-delayed-postmatch/test.yaml new file mode 100644 index 000000000..f7317888e --- /dev/null +++ b/tests/datasets-delayed-postmatch/test.yaml @@ -0,0 +1,21 @@ +requires: + min-version: 8 + +args: +- -k none --no-random --data-dir=${OUTPUT_DIR} + +checks: +- filter: + count: 1 + match: + event_type: alert + alert.signature_id: 1 + #TODO ticket 7197 metadata.flowvars[0].paramval: value +- filter: + count: 1 + match: + event_type: alert + alert.signature_id: 2 +- file-compare: + filename: http_match.csv + expected: expected/http_match.csv \ No newline at end of file From cceb6342fe34bfd5b8f74acc8e742e1eb7a6db49 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 15 Oct 2024 14:41:16 +0200 Subject: [PATCH 4/4] datasets: test with multi-buffer and occurences in different packets Ticket: 5576 --- tests/datasets-delayed-multi-postmatch/README.md | 14 ++++++++++++++ .../expected/http_match.csv | 2 ++ .../datasets-delayed-multi-postmatch/input.pcap | Bin 0 -> 1845 bytes .../datasets-delayed-multi-postmatch/test.rules | 1 + tests/datasets-delayed-multi-postmatch/test.yaml | 15 +++++++++++++++ 5 files changed, 32 insertions(+) create mode 100644 tests/datasets-delayed-multi-postmatch/README.md create mode 100644 tests/datasets-delayed-multi-postmatch/expected/http_match.csv create mode 100644 tests/datasets-delayed-multi-postmatch/input.pcap create mode 100644 tests/datasets-delayed-multi-postmatch/test.rules create mode 100644 tests/datasets-delayed-multi-postmatch/test.yaml diff --git a/tests/datasets-delayed-multi-postmatch/README.md b/tests/datasets-delayed-multi-postmatch/README.md new file mode 100644 index 000000000..b794e7276 --- /dev/null +++ b/tests/datasets-delayed-multi-postmatch/README.md @@ -0,0 +1,14 @@ +Test +==== + +Test datasets only sets when there is a full signature match. +Test is with a signature using different keywords matching at different stages, +and pcap having different packets making the transaction progress step by step. +And test is using a multi-buffer to test that we only save the right occurences. + +https://redmine.openinfosecfoundation.org/issues/5576 + +PCAP +==== + +Pcap crafted with some http server and some python client that delays or not the writing of the headers diff --git a/tests/datasets-delayed-multi-postmatch/expected/http_match.csv b/tests/datasets-delayed-multi-postmatch/expected/http_match.csv new file mode 100644 index 000000000..9a2e45ed0 --- /dev/null +++ b/tests/datasets-delayed-multi-postmatch/expected/http_match.csv @@ -0,0 +1,2 @@ +WC1maXJzdDogc2VjcmV0 +SGVhZGVyMTogZmlyc3Q= diff --git a/tests/datasets-delayed-multi-postmatch/input.pcap b/tests/datasets-delayed-multi-postmatch/input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..fcbd9a32917c3bdb70db8025c7046ddbc0ef4dc0 GIT binary patch literal 1845 zcmbW1&u`LT7{^~2fu$LX^TdrBFJ=m~fbz@CBFrrrn~Y)Hfsict%9b!#+D-=-J(;jS z!J8+qi-~{05(%f>y|~EHnRxUd+2-?<_l1IBQXY7crfoj`e4oB=!)Kp9*olJ-s&{<1 zFkk+>?;>fqfX&Me2g`&g_*a~SydlJfecJRU+YR*(-gjcZzP<_*Vs||HYMYsH+HA8z zEAj9*UUL8d*F>kOM>)i$Z`P&cpl)b^b&I(%PEOLNfMu~30qWsMODnC z>9u`B|GNeK-on+^VF6s4Ul=YZz5p(O2Np(;=%*&=U$z@eKQf|EcV19cj*Bn%R82Q@ ze@&$unln$E>)wO6_vOe%*1!{+7n?SuP=@wxZ`z@`zHWl+W^>3))v7uy8J}K1pm9;B zN4kKIaMrDEF`RJ7JFKj9V#RC@dAEUA09R~YOnG~PvbV3CF)J&bZJ#mPxTw*RN_Bxp z$k1w8--V447K{x3=`y6oW$;0ULeoW_fIK!Ymf`V)45@yG2RuW5sg+PV8l_^rb|_YM zbgJG$`g@>T`=}Wc-vrwqd`q?x>9bfdzl1#J@szxU@w}u39$R#qkbc8FO>{#?^s7wE XTQGlu2E30Jc any any ( sid: 1; http.stat_code; content: "200"; fast_pattern; http.response_header; content: "first"; dataset:set,http_match,type string,save http_match.csv; file.data; content: "later";) diff --git a/tests/datasets-delayed-multi-postmatch/test.yaml b/tests/datasets-delayed-multi-postmatch/test.yaml new file mode 100644 index 000000000..c83451c21 --- /dev/null +++ b/tests/datasets-delayed-multi-postmatch/test.yaml @@ -0,0 +1,15 @@ +requires: + min-version: 8 + +args: +- -k none --no-random --data-dir=${OUTPUT_DIR} + +checks: +- filter: + count: 1 + match: + event_type: alert + alert.signature_id: 1 +- file-compare: + filename: http_match.csv + expected: expected/http_match.csv \ No newline at end of file