From d9fc014a1db4a624cdf2a6016dfd04043601fd22 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 7 Oct 2024 14:53:13 +0200 Subject: [PATCH] fix: [crawler] filter lookup tags --- bin/lib/crawlers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 357b6423..7f6ff157 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -87,6 +87,11 @@ def api_get_onion_lookup(domain): # TODO check if object process done ??? meta['first_seen'] = meta['first_seen'].replace('/', '-') meta['last_seen'] = meta['last_check'].replace('/', '-') meta['languages'] = list(meta['languages']) + tags = [] + for tag in meta['tags']: + if tag.startswith('dark-web:') or tag.startswith('infoleak:'): + tags.append(tag) + meta['tags'] = tags del meta['domain'] del meta['last_check'] del meta['type']