+
Malcolm README |
-+
CyberChef |
++
CyberChef |
Home Page |
FAQ |
Wiki |
diff --git a/moloch/patch/improve-need-quotes-check-hopefully-fixes-1146-4de1686.patch b/moloch/patch/improve-need-quotes-check-hopefully-fixes-1146-4de1686.patch
deleted file mode 100644
index db82c6c42..000000000
--- a/moloch/patch/improve-need-quotes-check-hopefully-fixes-1146-4de1686.patch
+++ /dev/null
@@ -1,31 +0,0 @@
-From 4de16862943fb826a90c2ae967f9c90f6a89323a Mon Sep 17 00:00:00 2001
-From: Andy Wick
-Date: Fri, 30 Aug 2019 09:43:00 -0400
-Subject: [PATCH] improve need quotes check hopefully, fixes #1146
-
----
- CHANGELOG | 3 +++
- viewer/vueapp/src/filters.js | 4 ++--
- 2 files changed, 5 insertions(+), 2 deletions(-)
-
-diff --git a/viewer/vueapp/src/filters.js b/viewer/vueapp/src/filters.js
-index 090049dd..9004fe72 100644
---- a/viewer/vueapp/src/filters.js
-+++ b/viewer/vueapp/src/filters.js
-@@ -299,11 +299,11 @@ Vue.filter('searchFields', function (searchTerm, fields, excludeTokens) {
- */
- Vue.filter('buildExpression', function (field, value, op) {
- // for values required to be strings in the search expression
-- const str = /[^(EXISTS!)-+a-zA-Z0-9_.@:*?/,[]]+/.test(value);
-+ const needQuotes = value !== 'EXISTS!' && /[^-+a-zA-Z0-9_.@:*?/,\[\]]+/.test(value);
-
- // escape unescaped quotes
- value = value.toString().replace(/\\([\s\S])|(")/g, '\\$1$2');
-- if (str) { value = `"${value}"`; }
-+ if (needQuotes) { value = `"${value}"`; }
-
- return `${field} ${op} ${value}`;
- });
---
-2.20.1
-
diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index e4b386acc..131cbb050 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -8,6 +8,7 @@ events { worker_connections 1024; }
http {
+ include /etc/nginx/mime.types;
sendfile on;
client_max_body_size 20m;
fastcgi_buffers 16 16k;
@@ -84,13 +85,6 @@ http {
proxy_set_header Host moloch.malcolm.local;
}
- # Direct inline CyberChef
- location /chef {
- proxy_pass http://docker-malcolm-readme/cyberchef.htm;
- proxy_redirect off;
- proxy_set_header Host moloch.malcolm.local;
- }
-
# Logstash statistics
location ~* ^/logstash\b(.*) {
proxy_pass http://docker-logstash-stats/_node/stats$1;
@@ -99,16 +93,35 @@ http {
}
# Kibana shortcut
+ location ~* ^/idmol2kib(.*) {
+
+ set $filter_start_time now-1d;
+ if ($arg_start != '') {
+ set $filter_start_time \'$arg_start\';
+ }
+
+ set $filter_stop_time now;
+ if ($arg_stop != '') {
+ set $filter_stop_time \'$arg_stop\';
+ }
- location ~* ^/idmol2kib\b(.*) {
- # not working, work in progress
- # see https://discuss.elastic.co/t/nginx-reverse-proxy-with-rewrite-and-app-kibana-discover/198461/1
- rewrite ^/idmol2kib(.*) /kibana/app/kibana#/discover;
+ set $filter_field undefined;
+ if ($arg_field != '') {
+ set $filter_field $arg_field;
+ }
+
+ set $filter_value undefined;
+ if ($arg_value != '') {
+ set $filter_value $arg_value;
+ }
+
+ rewrite ^/idmol2kib/(.*) /kibana/app/kibana#/discover?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'sessions2-*',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'sessions2-*',interval:auto,query:(language:lucene,query:''),sort:!(firstPacket,desc)) redirect;
proxy_pass http://docker-kibana;
proxy_redirect off;
proxy_set_header Host kibana.malcolm.local;
}
+
# already prepended /kibana to match the server.basePath in kibana's YML config file
location /kibana {
proxy_pass http://docker-kibana;
@@ -116,7 +129,7 @@ http {
proxy_set_header Host kibana.malcolm.local;
}
- # otherwise (from old bookmarks), prepend /kibana to match the server.basePath in kibana's YML config file
+ # otherwise (from old kibana bookmarks), prepend /kibana to match the server.basePath in kibana's YML config file
location /app {
proxy_pass http://docker-kibana/kibana/app;
proxy_redirect off;
@@ -127,6 +140,21 @@ http {
alias /etc/nginx/favicon.ico;
}
+ # Fix cyberchef JS module(s)
+ # https://localhost/moloch/session/190924-KgO9H30qhdREw7ltsDXn1Rgp/modules/Regex.js
+ location ~* ^/moloch/session/.*/(modules/.*\.js) {
+ proxy_hide_header Content-Type;
+ proxy_set_header Content-Type "application/javascript";
+ add_header Content-Type "application/javascript";
+ default_type application/javascript;
+ add_header X-Content-Type-Options 'nosniff';
+ proxy_pass http://docker-moloch/cyberchef/$1;
+ proxy_redirect off;
+ proxy_set_header Host moloch.malcolm.local;
+ proxy_set_header http_auth_http_user $remote_user;
+ proxy_set_header Authorization "";
+ }
+
location / {
proxy_pass http://docker-moloch;
proxy_redirect off;
@@ -177,16 +205,10 @@ http {
proxy_set_header Host kibana.malcolm.local;
}
- # otherwise (from old bookmarks), prepend /kibana to match the server.basePath in kibana's YML config file
- location /app {
- proxy_pass http://docker-kibana/kibana/app;
- proxy_redirect off;
- proxy_set_header Host kibana.malcolm.local;
- }
-
# otherwise prepend /kibana to match the server.basePath in kibana's YML config file
location / {
- proxy_pass http://docker-kibana/kibana;
+ rewrite ^/(.*) /kibana/$1;
+ proxy_pass http://docker-kibana;
proxy_redirect off;
proxy_set_header Host kibana.malcolm.local;
}
diff --git a/scripts/auth_setup.sh b/scripts/auth_setup.sh
index a0a6b8a9c..0bd37feca 100755
--- a/scripts/auth_setup.sh
+++ b/scripts/auth_setup.sh
@@ -96,6 +96,17 @@ EOF
touch metadata
popd >/dev/null 2>&1
+if [[ ! -f ./elastalert/config/smtp-auth.yaml ]]; then
+ # create a sample smtp-auth.yaml for if/when we want to do elastalert email
+ pushd ./elastalert/config/ >/dev/null 2>&1
+ cat < smtp-auth.yaml
+user: "user@gmail.com"
+password: "abcdefg1234567"
+EOF
+ chmod 600 ./smtp-auth.yaml
+ popd >/dev/null 2>&1
+fi
+
unset CONFIRMATION
echo ""
read -p "(Re)generate self-signed certificates for HTTPS access [Y/n]? " CONFIRMATION
diff --git a/scripts/start.sh b/scripts/start.sh
index dddcb49d9..dd3ac5eba 100755
--- a/scripts/start.sh
+++ b/scripts/start.sh
@@ -49,6 +49,17 @@ fi
[[ -f ./htadmin/metadata ]] || touch ./htadmin/metadata
+if [[ ! -f ./elastalert/config/smtp-auth.yaml ]]; then
+ # create a sample smtp-auth.yaml for if/when we want to do elastalert email
+ pushd ./elastalert/config/ >/dev/null 2>&1
+ cat < smtp-auth.yaml
+user: "user@gmail.com"
+password: "abcdefg1234567"
+EOF
+ chmod 600 ./smtp-auth.yaml
+ popd >/dev/null 2>&1
+fi
+
# make sure a read permission is set correctly for the nginx worker processes
chmod 644 ./nginx/htpasswd ./htadmin/config.ini ./htadmin/metadata >/dev/null 2>&1
diff --git a/shared/bin/configure-capture.py b/shared/bin/configure-capture.py
index fb227e525..46a390325 100755
--- a/shared/bin/configure-capture.py
+++ b/shared/bin/configure-capture.py
@@ -326,7 +326,7 @@ def input_elasticsearch_connection_info(forwarder,
retcode, message, output = test_connection(protocol=return_dict[Constants.BEAT_KIBANA_PROTOCOL],
host=return_dict[Constants.BEAT_KIBANA_HOST],
port=return_dict[Constants.BEAT_KIBANA_PORT],
- uri="status",
+ uri="api/status",
username=return_dict[Constants.BEAT_HTTP_USERNAME] if (len(return_dict[Constants.BEAT_HTTP_USERNAME]) > 0) else None,
password=return_dict[Constants.BEAT_HTTP_PASSWORD] if (len(return_dict[Constants.BEAT_HTTP_PASSWORD]) > 0) else None,
ssl_verify=return_dict[Constants.BEAT_KIBANA_SSL_VERIFY])
diff --git a/shared/bin/zeek-carve-monitor.py b/shared/bin/zeek-carve-monitor.py
index 6cddde80c..2a6d649b0 100755
--- a/shared/bin/zeek-carve-monitor.py
+++ b/shared/bin/zeek-carve-monitor.py
@@ -45,6 +45,7 @@
###################################################################################################
debug = False
debugToggled = False
+pdbFlagged = False
args = None
scriptName = os.path.basename(__file__)
scriptPath = os.path.dirname(os.path.realpath(__file__))
@@ -60,8 +61,8 @@ def shutdown_handler(signum, frame):
###################################################################################################
# handle sigusr1 for a pdb breakpoint
def pdb_handler(sig, frame):
- import pdb
- pdb.Pdb().set_trace(frame)
+ global pdbFlagged
+ pdbFlagged = True
###################################################################################################
# handle sigusr2 for toggling debug
@@ -201,6 +202,7 @@ def main():
global args
global debug
global debugToggled
+ global pdbFlagged
global shuttingDown
parser = argparse.ArgumentParser(description=scriptName, add_help=False, usage='{} '.format(scriptName))
@@ -354,10 +356,19 @@ def main():
while (not shuttingDown):
+ if pdbFlagged:
+ pdbFlagged = False
+ breakpoint()
+
processedEvents = 0
# processed files for which checking is finished
while (not shuttingDown) and (processedEvents < (MAX_PROCESSED_BATCH_SIZE // 2)):
+
+ if pdbFlagged:
+ pdbFlagged = False
+ breakpoint()
+
try:
fileEvent = finishedFileQueue.popleft()
except IndexError:
@@ -420,6 +431,11 @@ def main():
# process new hashed files to be checked
queuedDupes = deque()
while (not shuttingDown) and (processedEvents < MAX_PROCESSED_BATCH_SIZE):
+
+ if pdbFlagged:
+ pdbFlagged = False
+ breakpoint()
+
try:
fileEvent = hashedFileQueue.popleft()
except IndexError:
@@ -491,6 +507,11 @@ def main():
# put duplicated processing events back into the hashedFileQueue to check again in a bit
dupeEvents = 0
while (len(queuedDupes) > 0):
+
+ if pdbFlagged:
+ pdbFlagged = False
+ breakpoint()
+
dupeEvents += 1
hashedFileQueue.append(queuedDupes.popleft())