Skip to content

Commit 86374ac

Browse files
toddbrunertoddbruner
committed
4.4.2 release
1 parent d9f5c77 commit 86374ac

File tree

17 files changed

+221
-22
lines changed

17 files changed

+221
-22
lines changed

‎etc/testlog.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,5 @@ log4perl.appender.FlairLog = Log::Log4perl::Appender::File
33
log4perl.appender.FlairLog.mode = append
44
log4perl.appender.FlairLog.filename = /var/log/flair/test.log
55
log4perl.appender.FlairLog.layout = Log::Log4perl::Layout::PatternLayout
6-
log4perl.appender.FlairLog.layout.ConversionPattern = %d %7p %10F{1}:%4L %m%n
6+
log4perl.appender.FlairLog.layout.ConversionPattern = %d %7p %15F{1}:%4L %m%n
77

‎lib/Flair/Config.pm‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ log4perl.appender.FlairLog = Log::Log4perl::Appender::File
8989
log4perl.appender.FlairLog.mode = append
9090
log4perl.appender.FlairLog.filename = $logfile
9191
log4perl.appender.FlairLog.layout = Log::Log4perl::Layout::PatternLayout
92-
log4perl.appender.FlairLog.layout.ConversionPattern = %d %5p [%12F:%4L] %m%n
92+
log4perl.appender.FlairLog.layout.ConversionPattern = %d %5p %15F{1}:%4L %m%n
9393
},
9494
},
9595
hypnotoad => {

‎lib/Flair/Parser.pm‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ sub post_match_actions ($self, $match, $et, $edb, $falsepos) {
167167
return $self->domain_action($match, $edb, $falsepos) if $et eq "domain";
168168
return $self->ipaddr_action($match, $edb, $falsepos) if $et eq "ipaddr";
169169
return $self->ipv6_action($match, $edb, $falsepos) if $et eq "ipv6";
170+
return $self->ipv6mixed_action($match, $edb, $falsepos) if $et eq "ipv6mixed";
170171
return $self->email_action($match, $edb, $falsepos) if $et eq "email";
171172
return $self->cve_action($match, $edb, $falsepos) if $et eq "cve";
172173
return $self->internal_link($match, $edb, $falsepos) if $et eq "internal_link";
@@ -344,6 +345,30 @@ sub ipv6_action ($self, $match, $edb, $falsepos) {
344345
return $self->create_span($standardized, 'ipv6');
345346
}
346347

348+
sub ipv6mixed_action ($self, $match, $edb, $falsepos) {
349+
# see if Net::IPv6Addr things this is a valid ipv6 address
350+
$self->log->debug("Checking $match for IPv6-ness");
351+
my $ipobj = try {
352+
return Net::IPv6Addr->new($match);
353+
}
354+
catch {
355+
$self->log->warn("invalid IPv6");
356+
return undef;
357+
};
358+
359+
if (not defined $ipobj or ref($ipobj) ne "Net::IPv6Addr") {
360+
$self->log->warn("failed to validate potential ipv6: $match: $_");
361+
return undef;
362+
}
363+
364+
# it is!
365+
my $standardized = $ipobj->to_string_ipv4();
366+
$self->log->debug("Standardized as $standardized");
367+
$self->add_entity($edb, $standardized, 'ipv6');
368+
return $self->create_span($standardized, 'ipv6');
369+
370+
}
371+
347372
sub suricata_ipv6_action ($self, $match, $edb, $falsepos) {
348373
# suricata puts a :portnum on the end of the ipv6 addr
349374
my @parts = split(/:/, $match);
@@ -437,13 +462,19 @@ sub message_id_action ($self, $match, $edb,$falsepos) {
437462
sub create_span ($self, $match, $et) {
438463
# wrap match string in a span
439464
$self->log->debug("creating entity span for $match type $et");
465+
my $text = $match;
466+
if ($et eq "message_id") {
467+
# scot4/scot4 issue #602: <[email protected]> needs to be &lt;[email protected]%gt;
468+
$text =~ s/</&lt;/;
469+
$text =~ s/>/&gt;/;
470+
}
440471
my $element = HTML::Element->new(
441472
'span',
442473
'class' => "entity $et",
443474
'data-entity-type' => $et,
444475
'data-entity-value' => lc($match),
445476
);
446-
$element->push_content($match);
477+
$element->push_content($text);
447478
return $element;
448479
}
449480

‎lib/Flair/Processor.pm‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ sub class_is ($self, $node, $match) {
524524
}
525525

526526
sub is_flair_override_span ($self, $node) {
527-
# ideas is to detect <span class="flairoverride" data-entity-type="type">sting</span>
527+
# ideas is to detect <span class="flairoverride" data-entity-type="type">string</span>
528528
# and create an entity of type "type" without actually doing the parsing
529529
if ($self->node_is_span($node)) {
530530
my $class = $node->attr('class');
@@ -544,6 +544,7 @@ sub extract_flair_override($self, $node, $edb) {
544544
my $text = pop @content;
545545
# get type
546546
my $type = $node->attr('data-entity-type');
547+
$self->log->debug("Detected Flair Override for $type = $text");
547548
# add it to edb
548549
$edb->{$type}->{$text}++;
549550
# rewrite <span> to be a standard flair span

‎lib/Flair/Regex.pm‎

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,11 @@ sub core_regex_names ($self) {
1111
my @list = (qw(
1212
cve
1313
cidr
14+
ipv6_mixed
1415
ipv6_suricata
1516
ipv6_standard
1617
ipv6_compressed
1718
ipv6_cmp8colons
18-
ipv6_mixed
1919
ipv4
2020
uuid1
2121
clsid
@@ -37,6 +37,9 @@ sub core_regex_names ($self) {
3737
sid
3838
useragent
3939
snumber
40+
suser
41+
snlserver1
42+
snlserver2
4043
));
4144
# omitted uri on purpose, not ready for prime time
4245
return wantarray ? @list : \@list;
@@ -151,7 +154,7 @@ sub ipv6_compressed ($self) {
151154
}xims,
152155
entity_type => 'ipv6',
153156
regex_type => 'core',
154-
re_order => 33,
157+
re_order => 34,
155158
multiword => 0,
156159
};
157160
}
@@ -165,7 +168,7 @@ sub ipv6_cmp8colons ($self ) {
165168
}xims,
166169
entity_type => 'ipv6',
167170
regex_type => 'core',
168-
re_order => 33,
171+
re_order => 35,
169172
multiword => 0,
170173
};
171174
}
@@ -193,7 +196,7 @@ sub ipv6_mixed ($self ) {
193196
# 255
194197
(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])
195198
}xims,
196-
entity_type => 'ipv6',
199+
entity_type => 'ipv6mixed',
197200
regex_type => 'core',
198201
re_order => 33,
199202
multiword => 0,
@@ -593,6 +596,54 @@ sub snumber ($self) {
593596
};
594597
}
595598

599+
sub suser ($self) {
600+
return {
601+
name => 'suser',
602+
description => 'Sandia Username',
603+
regex => qr{
604+
\b
605+
SANDIA\\\S+
606+
\b
607+
}xims,
608+
entity_type => 'suser',
609+
regex_type => 'core',
610+
re_order => 210,
611+
multiword => 1,
612+
};
613+
}
614+
615+
sub snlserver1 ($self) {
616+
return {
617+
name => 'snlserver1',
618+
description => 'Sandia Server Name',
619+
regex => qr{
620+
\b
621+
as\d+snllx
622+
\b
623+
}xims,
624+
entity_type => 'sandiaserver',
625+
regex_type => 'core',
626+
re_order => 300,
627+
multiword => 0,
628+
};
629+
}
630+
631+
sub snlserver2 ($self) {
632+
return {
633+
name => 'snlserver2',
634+
description => 'Sandia Server Name',
635+
regex => qr{
636+
\b
637+
as\d+mcslx
638+
\b
639+
}xims,
640+
entity_type => 'sandiaserver',
641+
regex_type => 'core',
642+
re_order => 300,
643+
multiword => 0,
644+
};
645+
}
646+
596647
# RE{URI} from REGEX::COMMON can not handle URI's with # in them
597648
# which renders it kind of useless.
598649
#sub uri ($self) {

‎lib/Flair/Util/HTML.pm‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,23 @@ sub build_html_tree ($text) {
3737
$tree ->ignore_unknown(0); # allow stuff like <figure/>
3838
$tree ->parse_content($text);
3939
$tree ->elementify;
40+
##
41+
## when dumping the tree as plain text, any tables will have cells
42+
## smashed together. This function will add some spaces to tree
43+
## and will fix this
44+
insert_table_element_spaces($tree);
4045
return $tree;
4146
}
4247

48+
sub insert_table_element_spaces ($tree) {
49+
foreach my $td ($tree->look_down('_tag', 'td')) {
50+
$td->preinsert(" ");
51+
}
52+
foreach my $th ($tree->look_down('_tag', 'th')) {
53+
$th->preinsert(" ");
54+
}
55+
}
56+
4357
sub output_tree_html ($tree) {
4458
# a $tree is full document, we really only want body
4559
my $body = $tree->look_down('_tag', 'body');

‎t/parser.t‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ ok ($db->dbh->migrations->from_file($migfile)->migrate(0)->migrate,
5050

5151
# need to populate flair table
5252
# only if we want to test user defined flair
53-
# $db->regex->populate_regex_table($config->{database}->{udef_file});
53+
$db->regex->populate_regex_table($config->{database}->{udef_file});
5454

5555
my $parser = Flair::Parser->new(log => $log, db => $db, scot_external_hostname => 'scot.watermelon.com');
5656
ok(defined $parser, "Parser instantiated") or die "Failed to instantiate parser";
@@ -83,7 +83,7 @@ foreach my $df (@data_files) {
8383
delete $edb->{cache};
8484

8585
is ($result, $test->{expect}, "Flaired Text Correctly in $df")
86-
or hdiff($result, $test->{expect});
86+
or xdiff($result, $test->{expect});
8787

8888
cmp_deeply($edb, $test->{entities}, "EDB Correct in $df")
8989
or die "EDB Differs: ".Dumper($edb, $test);
@@ -108,6 +108,28 @@ sub hdiff {
108108
die "Produced HTML differs";
109109
}
110110

111+
sub xdiff {
112+
my $g = shift;
113+
my $e = shift;
114+
115+
for (my $i = 0; $i < length($g); $i++) {
116+
my $gchar = substr($g, $i, 1);
117+
my $echar = substr($e, $i, 1);
118+
119+
if ($gchar eq $echar) {
120+
print $gchar;
121+
next;
122+
}
123+
124+
print "\n at char $i, test output differs from expected output\n";
125+
print "[test] = $gchar\n";
126+
print "[exp ] = $echar\n";
127+
last;
128+
}
129+
die "Produced HTML differs";
130+
}
131+
132+
111133

112134

113135

‎t/parser_test_data/cap_1‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
text => <<~'EOF',
3+
This APT5 looks weird.
4+
EOF
5+
6+
expect => <<~'EOF',
7+
This <span class="entity threat-actor" data-entity-type="threat-actor" data-entity-value="apt5">APT5</span> looks weird.
8+
EOF
9+
10+
entities => {
11+
"threat-actor" => {
12+
"apt5" => 1,
13+
},
14+
},
15+
}

‎t/parser_test_data/email_header_1‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,17 @@
1010
<CC6PR09MB885690CE7881A6F17EDB745EBCB1A@CO6PR09MB8856.namprd0.prod.outlook.com>
1111
References:
1212
EOF
13-
# note: these should be emails, but I'm doing message-ids because I have
13+
# note: these should be emails, but I'm doing message-ids because I have not
1414
# fixed this yet but need this test to pass
1515
expect => <<~'EOF',
16-
From: "Benz, Mercedes" <span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>"><[email protected]></span>
17-
To: "Qzzzz, Alll A." <span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>"><[email protected]></span>, "Blue, Bird"
18-
<span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>"><[email protected]></span>
19-
CC: HoneyBear <span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>"><[email protected]></span>
16+
From: "Benz, Mercedes" <span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>">&lt;[email protected]&gt;</span>
17+
To: "Qzzzz, Alll A." <span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>">&lt;[email protected]&gt;</span>, "Blue, Bird"
18+
<span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>">&lt;[email protected]&gt;</span>
19+
CC: HoneyBear <span class="entity message_id" data-entity-type="message_id" data-entity-value="<[email protected]>">&lt;[email protected]&gt;</span>
2020
Subject: Re: Cyber Incident Reported
2121
Thread-Topic: Cyber Incident Reported
2222
Message-ID:
23-
<span class="entity message_id" data-entity-type="message_id" data-entity-value="<cc6pr09mb885690ce7881a6f17edb745ebcb1a@co6pr09mb8856.namprd0.prod.outlook.com>"><CC6PR09MB885690CE7881A6F17EDB745EBCB1A@CO6PR09MB8856.namprd0.prod.outlook.com></span>
23+
<span class="entity message_id" data-entity-type="message_id" data-entity-value="<cc6pr09mb885690ce7881a6f17edb745ebcb1a@co6pr09mb8856.namprd0.prod.outlook.com>">&lt;CC6PR09MB885690CE7881A6F17EDB745EBCB1A@CO6PR09MB8856.namprd0.prod.outlook.com&gt;</span>
2424
References:
2525
EOF
2626

‎t/parser_test_data/file_3‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,15 @@
44
EOF
55

66
expect => <<~'EOF',
7-
Sep 10, 2018 07:33:38 AM Error [ajp-nio-8016-exec-6] - Error Executing Database Query.[Macromedia][SQLServer JDBC Driver][SQLServer]Incorrect syntax near '='. The specific sequence of files included or processed is: /mnt/gfs/cfdocs/eCATT/templates/<span class="entity file" data-entity-type="file" data-entity-value="pgas_rslts.cfm">pgas_rslts.cfm</span>, line: 235
7+
Sep 10, 2018 07:33:38 AM Error [ajp-nio-8016-exec-6] - Error Executing Database Query.[Macromedia][SQLServer JDBC Driver][SQLServer]Incorrect syntax near '='. The specific sequence of files included or processed is: /mnt/gfs<span class="entity cf_site" data-entity-type="cf_site" data-entity-value="/cfdocs/ecatt/">/cfdocs/eCATT/</span>templates/<span class="entity file" data-entity-type="file" data-entity-value="pgas_rslts.cfm">pgas_rslts.cfm</span>, line: 235
88
EOF
99

1010
entities => {
1111
file => {
1212
'pgas_rslts.cfm' => 1,
1313
},
14+
cf_site => {
15+
'/cfdocs/ecatt/' => 1,
16+
},
1417
},
1518
}

0 commit comments

Comments
 (0)