diff --git a/Makefile b/Makefile index 42d80a1b..73ab7702 100644 --- a/Makefile +++ b/Makefile @@ -132,6 +132,7 @@ pylama: ${pysrc} ${app} # check code quality ${env} pylama ${pysrcdirs} --skip "**/migrations/*" + shellcheck: ${shsrc} # shell script checks (if installed) if command -v shellcheck &>/dev/null && ! test -z "${shsrc}";then ${env} shellcheck ${shsrc}; fi @@ -143,6 +144,7 @@ autofix fix: ${pysrc} ${app} ## automatic fix of trivial code quality issues ${env} autoflake -ri --remove-all-unused-imports ${pysrcdirs} # sort imports ${env} isort -rc ${pysrcdirs} + black . # do a check after autofixing to show remaining problems ${MAKE} check diff --git a/dashboard/celery/__init__.py b/dashboard/celery/__init__.py index 8a167736..95a86af3 100644 --- a/dashboard/celery/__init__.py +++ b/dashboard/celery/__init__.py @@ -21,8 +21,9 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings") # autodiscover all celery tasks in tasks.py files inside websecmap modules -app.autodiscover_tasks([app for app in settings.INSTALLED_APPS - if app.startswith('dashboard') or app.startswith('websecmap')]) +app.autodiscover_tasks( + [app for app in settings.INSTALLED_APPS if app.startswith("dashboard") or app.startswith("websecmap")] +) # http://docs.celeryproject.org/en/master/whatsnew-4.0.html?highlight=priority#redis-priorities-reversed # http://docs.celeryproject.org/en/master/history/whatsnew-3.0.html?highlight=priority @@ -31,9 +32,9 @@ # https://github.com/celery/celery/blob/f83b072fba7831f60106c81472e3477608baf289/docs/whatsnew-4.0.rst#redis-priorities-reversed # contrary to 'documentation' in release notes the redis priorities do not seem aligned with rabbitmq app.conf.broker_transport_options = { - 'priority_steps': [1, 5, 9], + "priority_steps": [1, 5, 9], } -if 'redis://' in app.conf.broker_url: +if "redis://" in app.conf.broker_url: PRIO_HIGH = 1 PRIO_NORMAL = 5 PRIO_LOW = 9 @@ -44,8 +45,8 @@ # lookup table for routing keys for different IP versions IP_VERSION_QUEUE = { - 4: 'scanners.ipv4', - 6: 'scanners.ipv6', + 4: "scanners.ipv4", + 6: "scanners.ipv6", } @@ -77,21 +78,24 @@ def status(): active = inspect.active() reserved = inspect.reserved() active_queues = inspect.active_queues() - workers = [{ - 'name': worker_name, - 'queues': [q['name'] for q in active_queues.get(worker_name, [])], - 'tasks_processed': sum(worker_stats['total'].values()), - 'tasks_active': len(active.get(worker_name, [])), - 'tasks_reserved': len(reserved.get(worker_name, [])), - 'prefetch_count': worker_stats['prefetch_count'], - 'concurrency': worker_stats['pool']['max-concurrency'], - } for worker_name, worker_stats in stats.items()] + workers = [ + { + "name": worker_name, + "queues": [q["name"] for q in active_queues.get(worker_name, [])], + "tasks_processed": sum(worker_stats["total"].values()), + "tasks_active": len(active.get(worker_name, [])), + "tasks_reserved": len(reserved.get(worker_name, [])), + "prefetch_count": worker_stats["prefetch_count"], + "concurrency": worker_stats["pool"]["max-concurrency"], + } + for worker_name, worker_stats in stats.items() + ] # todo: fix Returning Any from function declared to return "SupportsLessThan" - workers = sorted(workers, key=lambda k: (k['name']), reverse=False) # type: ignore + workers = sorted(workers, key=lambda k: (k["name"]), reverse=False) # type: ignore - if 'redis://' in app.conf.broker_url: - queue_names = [q.name for q in QUEUES_MATCHING_ROLES['queuemonitor']] + if "redis://" in app.conf.broker_url: + queue_names = [q.name for q in QUEUES_MATCHING_ROLES["queuemonitor"]] # on localhost and remote workers there is no event loop. This causes an exception. # Inspired on https://github.com/tornadoweb/tornado/issues/2352 and @@ -103,6 +107,7 @@ def status(): # 'solves': RuntimeError: There is no current event loop in thread 'Thread-3'. try: import asyncio # pylint: disable=import-outside-toplevel + asyncio.set_event_loop(asyncio.new_event_loop()) except BaseException: # pylint: disable=broad-except # an eventloop already exists. @@ -118,21 +123,17 @@ def status(): log.error("Could not connect to flower to retrieve queue stats.") log.exception(runtime_error) - queues = [{'name': x['name'], 'tasks_pending': x['messages']} for x in queue_stats] + queues = [{"name": x["name"], "tasks_pending": x["messages"]} for x in queue_stats] else: - raise NotImplementedError('Currently only Redis is supported!') + raise NotImplementedError("Currently only Redis is supported!") # todo: fix Returning Any from function declared to return "SupportsLessThan" - queues = sorted(queues, key=lambda k: (k['name']), reverse=False) # type: ignore + queues = sorted(queues, key=lambda k: (k["name"]), reverse=False) # type: ignore alerts = [] if not workers: - alerts.append('No active workers!') + alerts.append("No active workers!") if len(workers) > 9000: - alerts.append('Number of workers is OVER 9000!!!!1111') + alerts.append("Number of workers is OVER 9000!!!!1111") - return { - 'alerts': alerts, - 'workers': workers, - 'queues': queues - } + return {"alerts": alerts, "workers": workers, "queues": queues} diff --git a/dashboard/internet_nl_dashboard/__init__.py b/dashboard/internet_nl_dashboard/__init__.py index 08dc2c7b..6d91bbe0 100644 --- a/dashboard/internet_nl_dashboard/__init__.py +++ b/dashboard/internet_nl_dashboard/__init__.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 import logging -default_app_config = 'dashboard.internet_nl_dashboard.apps.DashboardConfig' # pylint: disable=invalid-name +default_app_config = "dashboard.internet_nl_dashboard.apps.DashboardConfig" # pylint: disable=invalid-name log = logging.getLogger(__package__) diff --git a/dashboard/internet_nl_dashboard/admin.py b/dashboard/internet_nl_dashboard/admin.py index 146495d7..bb4e8524 100644 --- a/dashboard/internet_nl_dashboard/admin.py +++ b/dashboard/internet_nl_dashboard/admin.py @@ -27,11 +27,20 @@ from dashboard.internet_nl_dashboard.forms import CustomAccountModelForm from dashboard.internet_nl_dashboard.logic.domains import scan_urllist_now_ignoring_business_rules from dashboard.internet_nl_dashboard.logic.mail import send_scan_finished_mails -from dashboard.internet_nl_dashboard.models import (Account, AccountInternetNLScan, AccountInternetNLScanLog, - DashboardUser, TaggedUrlInUrllist, UploadLog, UrlList) -from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import (creating_report, - progress_running_scan, - recover_and_retry) +from dashboard.internet_nl_dashboard.models import ( + Account, + AccountInternetNLScan, + AccountInternetNLScanLog, + DashboardUser, + TaggedUrlInUrllist, + UploadLog, + UrlList, +) +from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import ( + creating_report, + progress_running_scan, + recover_and_retry, +) log = logging.getLogger(__package__) @@ -46,7 +55,7 @@ def count(self): # pylint: disable=invalid-overridden-method def only_alphanumeric(data: str) -> str: - return re.sub(r'[^A-Za-z0-9 ]+', '', data) + return re.sub(r"[^A-Za-z0-9 ]+", "", data) class MyPeriodicTaskForm(PeriodicTaskForm): @@ -66,7 +75,7 @@ class MyPeriodicTaskForm(PeriodicTaskForm): """ def clean(self): - print('cleaning') + print("cleaning") cleaned_data = super().clean() # if not self.cleaned_data['last_run_at']: @@ -78,12 +87,25 @@ def clean(self): class IEPeriodicTaskAdmin(PeriodicTaskAdmin, ImportExportModelAdmin): # most / all time schedule functions in celery beat are moot. So the code below likely makes no sense. - list_display = ('name_safe', 'enabled', 'interval', 'crontab', 'next', 'due', - 'precise', 'last_run_at', 'queue', 'task', 'args', 'last_run', 'runs') + list_display = ( + "name_safe", + "enabled", + "interval", + "crontab", + "next", + "due", + "precise", + "last_run_at", + "queue", + "task", + "args", + "last_run", + "runs", + ) - list_filter = ('enabled', 'queue', 'crontab') + list_filter = ("enabled", "queue", "crontab") - search_fields = ('name', 'queue', 'args') + search_fields = ("name", "queue", "args") form = MyPeriodicTaskForm @@ -150,7 +172,7 @@ class IECrontabSchedule(ImportExportModelAdmin): class DashboardUserInline(CompactInline): model = DashboardUser can_delete = False - verbose_name_plural = 'Dashboard Users' + verbose_name_plural = "Dashboard Users" # Thank you: @@ -168,21 +190,30 @@ class Meta: # pylint: disable=too-few-public-methods class UserAdmin(BaseUserAdmin, ImportExportModelAdmin): resource_class = UserResource - inlines = (DashboardUserInline, ) + inlines = (DashboardUserInline,) - list_display = ('username', 'in_account', 'first_name', 'last_name', - 'email', 'is_active', 'is_staff', 'is_superuser', 'last_login') + list_display = ( + "username", + "in_account", + "first_name", + "last_name", + "email", + "is_active", + "is_staff", + "is_superuser", + "last_login", + ) - list_filter = ['is_active', 'is_staff', 'is_superuser'][::-1] + list_filter = ["is_active", "is_staff", "is_superuser"][::-1] - search_fields = ['username', 'dashboarduser__account__name', 'dashboarduser__account__internet_nl_api_username'] + search_fields = ["username", "dashboarduser__account__name", "dashboarduser__account__internet_nl_api_username"] @staticmethod def in_account(obj): user = DashboardUser.objects.all().filter(user=obj).first() if not user: - return '-' + return "-" return user.account @@ -207,7 +238,7 @@ def __init__(self, *args, **kwargs): class ConfigAdmin(ConstanceAdmin): change_list_form = CustomConfigForm - change_list_template = 'admin/config/settings.html' + change_list_template = "admin/config/settings.html" admin.site.unregister([Config]) @@ -218,10 +249,10 @@ class ConfigAdmin(ConstanceAdmin): class AccountAdmin(ImportExportModelAdmin, admin.ModelAdmin): form = CustomAccountModelForm - list_display = ('name', 'internet_nl_api_username', 'can_connect_to_internet_nl_api', 'no_of_users') - search_fields = ('name', 'can_connect_to_internet_nl_api') + list_display = ("name", "internet_nl_api_username", "can_connect_to_internet_nl_api", "no_of_users") + search_fields = ("name", "can_connect_to_internet_nl_api") # list_filter = [][::-1] - fields = ('name', 'report_settings', 'internet_nl_api_username', 'new_password') + fields = ("name", "report_settings", "internet_nl_api_username", "new_password") # cannot use the DashboardUserInline, it acts like there are three un-assigned users and it breaks the 1 to 1 # relation with the DashboardUser to user. Perhaps because Jet doesn't understands that type of relationship @@ -236,15 +267,17 @@ class AccountAdmin(ImportExportModelAdmin, admin.ModelAdmin): @staticmethod def no_of_users(obj): - return mark_safe(f"" # nosec - f"🔎 {DashboardUser.objects.all().filter(account=obj).count()}") + return mark_safe( + f"" # nosec + f"🔎 {DashboardUser.objects.all().filter(account=obj).count()}" + ) def save_model(self, request, obj, form, change): # If the internet_nl_api_password changed, encrypt the new value. # Example usage and docs: https://github.com/pyca/cryptography - if 'new_password' in form.changed_data: - obj.internet_nl_api_password = Account.encrypt_password(form.cleaned_data.get('new_password')) + if "new_password" in form.changed_data: + obj.internet_nl_api_password = Account.encrypt_password(form.cleaned_data.get("new_password")) # check if the username / password combination is valid @@ -262,23 +295,50 @@ def check_api_connectivity(self, request, queryset): # suppressing error: "Callable[[Any, Any, Any], Any]" has no attribute "short_description" # This comes from the manual... so, well. - check_api_connectivity.short_description = "Check API credentials" # type: ignore - actions.append('check_api_connectivity') + check_api_connectivity.short_description = "Check API credentials" # type: ignore + actions.append("check_api_connectivity") @admin.register(UrlList) class UrlListAdmin(ImportExportModelAdmin, admin.ModelAdmin): - list_display = ('pk', 'name', 'account', 'scan_type', 'no_of_urls', 'no_of_endpoints', - 'automated_scan_frequency', 'last_manual_scan', 'is_deleted', 'is_scan_now_available') - search_fields = ('name', 'account__name') - list_filter = ['account', 'is_deleted', 'scan_type', 'enable_scans', 'automated_scan_frequency', - 'last_manual_scan'][::-1] + list_display = ( + "pk", + "name", + "account", + "scan_type", + "no_of_urls", + "no_of_endpoints", + "automated_scan_frequency", + "last_manual_scan", + "is_deleted", + "is_scan_now_available", + ) + search_fields = ("name", "account__name") + list_filter = [ + "account", + "is_deleted", + "scan_type", + "enable_scans", + "automated_scan_frequency", + "last_manual_scan", + ][::-1] # we don't add the urls as that might cause a deletion by mistake - fields = ('name', 'account', 'scan_type', 'enable_scans', 'automated_scan_frequency', 'scheduled_next_scan', - 'last_manual_scan', 'is_deleted', 'deleted_on', 'enable_report_sharing_page', - 'automatically_share_new_reports', 'default_public_share_code_for_new_reports') + fields = ( + "name", + "account", + "scan_type", + "enable_scans", + "automated_scan_frequency", + "scheduled_next_scan", + "last_manual_scan", + "is_deleted", + "deleted_on", + "enable_report_sharing_page", + "automatically_share_new_reports", + "default_public_share_code_for_new_reports", + ) @staticmethod def no_of_urls(obj): @@ -286,8 +346,11 @@ def no_of_urls(obj): @staticmethod def no_of_endpoints(obj): - return Endpoint.objects.all().filter(url__urls_in_dashboard_list_2=obj, is_dead=False, - url__is_dead=False, url__not_resolvable=False).count() + return ( + Endpoint.objects.all() + .filter(url__urls_in_dashboard_list_2=obj, is_dead=False, url__is_dead=False, url__not_resolvable=False) + .count() + ) actions = [] @@ -300,22 +363,22 @@ def scan_urllist_now(self, request, queryset): # suppressing error: "Callable[[Any, Any, Any], Any]" has no attribute "short_description" # This comes from the manual... so, well. - scan_urllist_now.short_description = "Scan now (bypassing quota and business rules)" # type: ignore - actions.append('scan_urllist_now') + scan_urllist_now.short_description = "Scan now (bypassing quota and business rules)" # type: ignore + actions.append("scan_urllist_now") @admin.register(TaggedUrlInUrllist) class TaggedUrlInUrllistAdmin(ImportExportModelAdmin, admin.ModelAdmin): - list_display = ('url', 'urllist') - fields = ('url', 'urllist', 'tags') + list_display = ("url", "urllist") + fields = ("url", "urllist", "tags") class InternetNLV2StateLogInline(nested_admin.NestedTabularInline): # pylint: disable=no-member too-few-public-methods model = InternetNLV2StateLog can_delete = False - readonly_fields = ('state', 'state_message', 'last_state_check', 'at_when') + readonly_fields = ("state", "state_message", "last_state_check", "at_when") def has_add_permission(self, *args, **kwargs): return False @@ -325,7 +388,7 @@ class AccountInternetNLScanLogInl(nested_admin.NestedTabularInline): # pylint: model = AccountInternetNLScanLog can_delete = False # readonly_fields = ('scan', 'account', 'urllist', 'log') - readonly_fields = ('state', 'at_when') + readonly_fields = ("state", "at_when") # todo: restrict any form of editing and creating things here. def has_add_permission(self, *args, **kwargs): @@ -337,7 +400,7 @@ class AccountInternetNLScanInline(nested_admin.NestedTabularInline): # pylint: can_delete = False inlines = [AccountInternetNLScanLogInl] - readonly_fields = ('account', 'urllist', 'report', 'state', 'started_on', 'finished_on', 'state_changed_on') + readonly_fields = ("account", "urllist", "report", "state", "started_on", "finished_on", "state_changed_on") def has_add_permission(self, *args, **kwargs): return False @@ -350,24 +413,33 @@ def get_queryset(self, request): qs = super().get_queryset(request) # prefetch account, scan and urllist, as loading this takes ages because of subqueries. - qs = qs.select_related('account', 'urllist', 'scan') + qs = qs.select_related("account", "urllist", "scan") # this column prevents loading of long lists - qs = qs.defer('scan__subject_urls', 'scan__retrieved_scan_report', 'scan__retrieved_scan_report_technical') + qs = qs.defer("scan__subject_urls", "scan__retrieved_scan_report", "scan__retrieved_scan_report_technical") return qs inlines = [AccountInternetNLScanLogInl] - list_display = ('id', 'account', 'account__name', 'state', 'internetnl_scan', 'internetnl_scan_type', - 'urllist', 'started_on', 'finished_on') + list_display = ( + "id", + "account", + "account__name", + "state", + "internetnl_scan", + "internetnl_scan_type", + "urllist", + "started_on", + "finished_on", + ) - list_filter = ['account', 'urllist', 'state', 'started_on', 'finished_on'][::-1] - search_fields = ('urllist__name', 'account__name') + list_filter = ["account", "urllist", "state", "started_on", "finished_on"][::-1] + search_fields = ("urllist__name", "account__name") - fields = ('state', 'state_changed_on', 'account', 'scan', 'urllist', 'started_on', 'finished_on') + fields = ("state", "state_changed_on", "account", "scan", "urllist", "started_on", "finished_on") - readonly_fields = ('account', 'scan', 'urllist') + readonly_fields = ("account", "scan", "urllist") @staticmethod def account__name(obj): @@ -387,8 +459,9 @@ def attempt_rollback(self, request, queryset): for scan in queryset: recover_and_retry.apply_async([scan.id]) self.message_user(request, "Rolling back asynchronously. May take a while.") - attempt_rollback.short_description = "Attempt rollback (async)" # type: ignore - actions.append('attempt_rollback') + + attempt_rollback.short_description = "Attempt rollback (async)" # type: ignore + actions.append("attempt_rollback") def progress_scan(self, request, queryset): log.debug("Attempting to progress scan.") @@ -398,8 +471,9 @@ def progress_scan(self, request, queryset): log.debug(f"Created task {tasks}.") tasks.apply_async() self.message_user(request, "Attempting to progress scans (async).") - progress_scan.short_description = "Progress scan (async)" # type: ignore - actions.append('progress_scan') + + progress_scan.short_description = "Progress scan (async)" # type: ignore + actions.append("progress_scan") def send_finish_mail(self, request, queryset): sent = 0 @@ -408,8 +482,9 @@ def send_finish_mail(self, request, queryset): sent += 1 send_scan_finished_mails(scan.id) self.message_user(request, f"A total of {sent} mails have been sent.") - send_finish_mail.short_description = "Queue finished mail (finished only)" # type: ignore - actions.append('send_finish_mail') + + send_finish_mail.short_description = "Queue finished mail (finished only)" # type: ignore + actions.append("send_finish_mail") # This is used to create ad-hoc reports for testing the send_finish_mail function. def create_extra_report(self, request, queryset): @@ -419,8 +494,8 @@ def create_extra_report(self, request, queryset): group(tasks).apply_async() self.message_user(request, "Creating additional reports (async).") - create_extra_report.short_description = "Create additional report (async) (finished only)" # type: ignore - actions.append('create_extra_report') + create_extra_report.short_description = "Create additional report (async) (finished only)" # type: ignore + actions.append("create_extra_report") class InternetNLV2ScanAdminNew(InternetNLV2ScanAdmin, nested_admin.NestedModelAdmin): # pylint: disable=no-member @@ -430,7 +505,7 @@ def get_queryset(self, request): qs = super().get_queryset(request) # this column prevents loading of long lists - qs = qs.defer('subject_urls', 'retrieved_scan_report', 'retrieved_scan_report_technical') + qs = qs.defer("subject_urls", "retrieved_scan_report", "retrieved_scan_report_technical") return qs @@ -443,16 +518,23 @@ def get_queryset(self, request): "state_message", "last_state_check", "last_state_change", - "account_scan" + "account_scan", ) list_filter = InternetNLV2ScanAdmin.list_filter + () def account_scan(self, obj): - account = AccountInternetNLScan.objects.all().filter(scan=obj).only( - 'id', 'account__internet_nl_api_username', 'urllist__name').first() - return f"{account.id} - {account.account.internet_nl_api_username} - " \ - f"{account.urllist.name}" if account else None + account = ( + AccountInternetNLScan.objects.all() + .filter(scan=obj) + .only("id", "account__internet_nl_api_username", "urllist__name") + .first() + ) + return ( + f"{account.id} - {account.account.internet_nl_api_username} - " f"{account.urllist.name}" + if account + else None + ) inlines = [InternetNLV2StateLogInline, AccountInternetNLScanInline] @@ -462,7 +544,7 @@ def create_modeladmin(modeladmin, model, name=None): # In this case we can create WhoisOrganization which only allows easy overview and record editing # of registrar information. # https://stackoverflow.com/questions/2223375/multiple-modeladmins-views-for-same-model-in-django-admin - class Meta: # pylint: disable=too-few-public-methods + class Meta: # pylint: disable=too-few-public-methods proxy = True app_label = model._meta.app_label @@ -481,26 +563,26 @@ class Meta: # pylint: disable=too-few-public-methods class AccountInternetNLScanLogAdmin(ImportExportModelAdmin, admin.ModelAdmin): paginator = TooManyRecordsPaginator - list_display = ('id', 'scan', 'state', 'at_when') - list_filter = ['scan', 'state', 'at_when'][::-1] - search_fields = ('scan__urllist__name', 'scan__account__name') + list_display = ("id", "scan", "state", "at_when") + list_filter = ["scan", "state", "at_when"][::-1] + search_fields = ("scan__urllist__name", "scan__account__name") fields = list_display @admin.register(UploadLog) class UploadLogAdmin(ImportExportModelAdmin, admin.ModelAdmin): - list_display = ('original_filename', 'internal_filename', 'status', 'message', 'user', 'upload_date', 'filesize') - search_fields = ('internal_filename', 'orginal_filename', 'status') - list_filter = ['message', 'upload_date', 'user'][::-1] + list_display = ("original_filename", "internal_filename", "status", "message", "user", "upload_date", "filesize") + search_fields = ("internal_filename", "orginal_filename", "status") + list_filter = ["message", "upload_date", "user"][::-1] - fields = ('original_filename', 'internal_filename', 'status', 'message', 'user', 'upload_date', 'filesize') + fields = ("original_filename", "internal_filename", "status", "message", "user", "upload_date", "filesize") @admin.register(models.SubdomainDiscoveryScan) class SubdomainDiscoveryScanAdmin(ImportExportModelAdmin, admin.ModelAdmin): - list_display = ('urllist', 'state', 'state_changed_on', 'state_message') + list_display = ("urllist", "state", "state_changed_on", "state_message") fields = ("urllist", "state", "state_changed_on", "state_message", "domains_discovered") - list_filter = ('state', 'state_changed_on', "state_message") + list_filter = ("state", "state_changed_on", "state_message") @admin.register(models.UrlListReport) @@ -508,8 +590,7 @@ class UrlListReportAdmin(ImportExportModelAdmin, admin.ModelAdmin): @staticmethod def inspect_list(obj): - return format_html('inspect', - id=format(obj.id)) + return format_html('inspect', id=format(obj.id)) # do NOT load the calculation field, as that will be slow. # https://stackoverflow.com/questions/34774028/how-to-ignore-loading-huge-fields-in-django-admin-list-display @@ -520,60 +601,69 @@ def get_queryset(self, request): qs = qs.defer("calculation") return qs - list_display = ('urllist', 'average_internet_nl_score', 'high', 'medium', 'low', 'ok', 'total_endpoints', - 'ok_endpoints', 'is_publicly_shared', 'is_shared_on_homepage', 'at_when', 'inspect_list') - search_fields = ['at_when'] - list_filter = ['urllist', 'at_when', 'is_publicly_shared'][::-1] - fields = ('urllist', - - 'report_type', - 'is_publicly_shared', - 'is_shared_on_homepage', - 'public_report_code', - 'public_share_code', - - 'at_when', - 'calculation', - 'average_internet_nl_score', - - 'total_endpoints', - 'total_issues', - - 'high', - 'medium', - 'low', - 'ok', - 'high_endpoints', - 'medium_endpoints', - 'low_endpoints', - 'ok_endpoints', - 'total_url_issues', - 'url_issues_high', - 'url_issues_medium', - 'url_issues_low', - 'url_ok', - 'total_endpoint_issues', - 'endpoint_issues_high', - 'endpoint_issues_medium', - 'endpoint_issues_low', - 'endpoint_ok', - 'explained_high', - 'explained_medium', - 'explained_low', - 'explained_high_endpoints', - 'explained_medium_endpoints', - 'explained_low_endpoints', - 'explained_total_url_issues', - 'explained_url_issues_high', - 'explained_url_issues_medium', - 'explained_url_issues_low', - 'explained_total_endpoint_issues', - 'explained_endpoint_issues_high', - 'explained_endpoint_issues_medium', - 'explained_endpoint_issues_low', - ) + list_display = ( + "urllist", + "average_internet_nl_score", + "high", + "medium", + "low", + "ok", + "total_endpoints", + "ok_endpoints", + "is_publicly_shared", + "is_shared_on_homepage", + "at_when", + "inspect_list", + ) + search_fields = ["at_when"] + list_filter = ["urllist", "at_when", "is_publicly_shared"][::-1] + fields = ( + "urllist", + "report_type", + "is_publicly_shared", + "is_shared_on_homepage", + "public_report_code", + "public_share_code", + "at_when", + "calculation", + "average_internet_nl_score", + "total_endpoints", + "total_issues", + "high", + "medium", + "low", + "ok", + "high_endpoints", + "medium_endpoints", + "low_endpoints", + "ok_endpoints", + "total_url_issues", + "url_issues_high", + "url_issues_medium", + "url_issues_low", + "url_ok", + "total_endpoint_issues", + "endpoint_issues_high", + "endpoint_issues_medium", + "endpoint_issues_low", + "endpoint_ok", + "explained_high", + "explained_medium", + "explained_low", + "explained_high_endpoints", + "explained_medium_endpoints", + "explained_low_endpoints", + "explained_total_url_issues", + "explained_url_issues_high", + "explained_url_issues_medium", + "explained_url_issues_low", + "explained_total_endpoint_issues", + "explained_endpoint_issues_high", + "explained_endpoint_issues_medium", + "explained_endpoint_issues_low", + ) ordering = ["-at_when"] - readonly_fields = ['calculation'] + readonly_fields = ["calculation"] save_as = True diff --git a/dashboard/internet_nl_dashboard/apps.py b/dashboard/internet_nl_dashboard/apps.py index 2ef651ff..720a000b 100644 --- a/dashboard/internet_nl_dashboard/apps.py +++ b/dashboard/internet_nl_dashboard/apps.py @@ -3,16 +3,16 @@ class DashboardConfig(AppConfig): - name = 'dashboard.internet_nl_dashboard' + name = "dashboard.internet_nl_dashboard" # See: https://django-activity-stream.readthedocs.io/en/latest/configuration.html def ready(self): # Loading actstream is not possible yet, as the apps aren't loaded. Django will crash. from actstream import registry # pylint: disable=import-outside-toplevel - registry.register(self.get_model('UrlList')) - registry.register(self.get_model('AccountInternetNLScan')) - registry.register(self.get_model('UrlListReport')) - registry.register(self.get_model('Account')) - registry.register(self.get_model('DashboardUser')) - registry.register(self.get_model('UploadLog')) + registry.register(self.get_model("UrlList")) + registry.register(self.get_model("AccountInternetNLScan")) + registry.register(self.get_model("UrlListReport")) + registry.register(self.get_model("Account")) + registry.register(self.get_model("DashboardUser")) + registry.register(self.get_model("UploadLog")) diff --git a/dashboard/internet_nl_dashboard/check_dns.py b/dashboard/internet_nl_dashboard/check_dns.py index e4e17365..0a3b5665 100644 --- a/dashboard/internet_nl_dashboard/check_dns.py +++ b/dashboard/internet_nl_dashboard/check_dns.py @@ -1,4 +1,3 @@ - from dns.resolver import Resolver from websecmap.app.constance import constance_cached_value diff --git a/dashboard/internet_nl_dashboard/context_processors.py b/dashboard/internet_nl_dashboard/context_processors.py index 7e55c58a..004a7247 100644 --- a/dashboard/internet_nl_dashboard/context_processors.py +++ b/dashboard/internet_nl_dashboard/context_processors.py @@ -13,4 +13,4 @@ def template_settings_processor(request): :return: """ - return {'LANGUAGES': settings.LANGUAGES, 'debug': settings.DEBUG} + return {"LANGUAGES": settings.LANGUAGES, "debug": settings.DEBUG} diff --git a/dashboard/internet_nl_dashboard/forms.py b/dashboard/internet_nl_dashboard/forms.py index 7ffcb48a..9b82367c 100644 --- a/dashboard/internet_nl_dashboard/forms.py +++ b/dashboard/internet_nl_dashboard/forms.py @@ -10,8 +10,7 @@ class CustomAccountModelForm(forms.ModelForm): new_password = forms.CharField( - help_text='Changing this value will set a new password for this account.', - required=False + help_text="Changing this value will set a new password for this account.", required=False ) def save(self, commit=True): @@ -21,4 +20,4 @@ def save(self, commit=True): class Meta: model = Account - fields = '__all__' + fields = "__all__" diff --git a/dashboard/internet_nl_dashboard/logic/__init__.py b/dashboard/internet_nl_dashboard/logic/__init__.py index 1af1e2c6..bf3fbaf8 100644 --- a/dashboard/internet_nl_dashboard/logic/__init__.py +++ b/dashboard/internet_nl_dashboard/logic/__init__.py @@ -4,15 +4,16 @@ def operation_response( - error: bool = False, success: bool = False, message: str = "", data: Optional[Dict[Any, Any]] = None + error: bool = False, success: bool = False, message: str = "", data: Optional[Dict[Any, Any]] = None ) -> Dict[str, Any]: - return {'error': error, - 'success': success, - 'message': message, - 'state': "error" if error else "success", - 'data': data, - 'timestamp': datetime.now(timezone.utc) - } + return { + "error": error, + "success": success, + "message": message, + "state": "error" if error else "success", + "data": data, + "timestamp": datetime.now(timezone.utc), + } # WARNING! ORDERING IS RELEVANT! @@ -20,219 +21,209 @@ def operation_response( # The ordering is used in the spreadsheet export. The way the fields are ordered is relevant. WEB_OVERALL_FIELDS = [ - 'internet_nl_score', - 'internet_nl_score_report', + "internet_nl_score", + "internet_nl_score_report", ] -WEB_IPV6_CATEGORY = ['internet_nl_web_ipv6'] +WEB_IPV6_CATEGORY = ["internet_nl_web_ipv6"] WEB_IPV6_FIELDS = [ - 'internet_nl_web_ipv6_ns_address', - 'internet_nl_web_ipv6_ns_reach', - - 'internet_nl_web_ipv6_ws_address', - 'internet_nl_web_ipv6_ws_reach', - 'internet_nl_web_ipv6_ws_similar' + "internet_nl_web_ipv6_ns_address", + "internet_nl_web_ipv6_ns_reach", + "internet_nl_web_ipv6_ws_address", + "internet_nl_web_ipv6_ws_reach", + "internet_nl_web_ipv6_ws_similar", ] -WEB_DNSSEC_CATEGORY = ['internet_nl_web_dnssec'] +WEB_DNSSEC_CATEGORY = ["internet_nl_web_dnssec"] WEB_DNSSEC_FIELDS = [ - 'internet_nl_web_dnssec_exist', - 'internet_nl_web_dnssec_valid', + "internet_nl_web_dnssec_exist", + "internet_nl_web_dnssec_valid", ] -WEB_TLS_CATEGORY = ['internet_nl_web_tls'] +WEB_TLS_CATEGORY = ["internet_nl_web_tls"] WEB_TLS_HTTP_FIELDS = [ - 'internet_nl_web_https_http_available', - 'internet_nl_web_https_http_redirect', - 'internet_nl_web_https_http_compress', - 'internet_nl_web_https_http_hsts', + "internet_nl_web_https_http_available", + "internet_nl_web_https_http_redirect", + "internet_nl_web_https_http_compress", + "internet_nl_web_https_http_hsts", ] WEB_TLS_TLS_FIELDS = [ - 'internet_nl_web_https_tls_version', - 'internet_nl_web_https_tls_ciphers', + "internet_nl_web_https_tls_version", + "internet_nl_web_https_tls_ciphers", # api 2.0 tls 1.3 fields, may 2020 - 'internet_nl_web_https_tls_cipherorder', - 'internet_nl_web_https_tls_keyexchange', + "internet_nl_web_https_tls_cipherorder", + "internet_nl_web_https_tls_keyexchange", # api 2.0 tls 1.3 fields, may 2020 - 'internet_nl_web_https_tls_keyexchangehash', - 'internet_nl_web_https_tls_compress', - 'internet_nl_web_https_tls_secreneg', - 'internet_nl_web_https_tls_clientreneg', + "internet_nl_web_https_tls_keyexchangehash", + "internet_nl_web_https_tls_compress", + "internet_nl_web_https_tls_secreneg", + "internet_nl_web_https_tls_clientreneg", # api 2.0 tls 1.3 fields, may 2020 - 'internet_nl_web_https_tls_0rtt', + "internet_nl_web_https_tls_0rtt", # api 2.0 tls 1.3 fields, may 2020 - 'internet_nl_web_https_tls_ocsp', + "internet_nl_web_https_tls_ocsp", ] WEB_TLS_CERTIFICATE_FIELDS = [ - 'internet_nl_web_https_cert_chain', - 'internet_nl_web_https_cert_pubkey', - 'internet_nl_web_https_cert_sig', - 'internet_nl_web_https_cert_domain', + "internet_nl_web_https_cert_chain", + "internet_nl_web_https_cert_pubkey", + "internet_nl_web_https_cert_sig", + "internet_nl_web_https_cert_domain", ] WEB_TLS_DANE_FIELDS = [ - 'internet_nl_web_https_dane_exist', - 'internet_nl_web_https_dane_valid', + "internet_nl_web_https_dane_exist", + "internet_nl_web_https_dane_valid", ] -WEB_APPSECPRIV_CATEGORY = ['internet_nl_web_appsecpriv'] +WEB_APPSECPRIV_CATEGORY = ["internet_nl_web_appsecpriv"] WEB_APPSECPRIV_FIELDS = [ - 'internet_nl_web_appsecpriv_x_frame_options', - 'internet_nl_web_appsecpriv_x_content_type_options', - 'internet_nl_web_appsecpriv_csp', - 'internet_nl_web_appsecpriv_referrer_policy', - 'internet_nl_web_appsecpriv_securitytxt', + "internet_nl_web_appsecpriv_x_frame_options", + "internet_nl_web_appsecpriv_x_content_type_options", + "internet_nl_web_appsecpriv_csp", + "internet_nl_web_appsecpriv_referrer_policy", + "internet_nl_web_appsecpriv_securitytxt", ] -WEB_RPKI_CATEGORY = ['internet_nl_web_rpki'] +WEB_RPKI_CATEGORY = ["internet_nl_web_rpki"] WEB_RPKI_FIELDS = [ - 'internet_nl_web_rpki_exists', - 'internet_nl_web_rpki_valid', - 'internet_nl_web_ns_rpki_exists', - 'internet_nl_web_ns_rpki_valid', + "internet_nl_web_rpki_exists", + "internet_nl_web_rpki_valid", + "internet_nl_web_ns_rpki_exists", + "internet_nl_web_ns_rpki_valid", ] WEB_LEGACY_CATEGORY = ["internet_nl_web_legacy_category"] WEB_LEGACY_FIELDS = [ - 'internet_nl_web_legacy_dnssec', - 'internet_nl_web_legacy_tls_available', - 'internet_nl_web_legacy_tls_ncsc_web', - 'internet_nl_web_legacy_https_enforced', - 'internet_nl_web_legacy_hsts', + "internet_nl_web_legacy_dnssec", + "internet_nl_web_legacy_tls_available", + "internet_nl_web_legacy_tls_ncsc_web", + "internet_nl_web_legacy_https_enforced", + "internet_nl_web_legacy_hsts", # api 2.0 extra fields - 'internet_nl_web_legacy_category_ipv6', - 'internet_nl_web_legacy_ipv6_nameserver', - 'internet_nl_web_legacy_ipv6_webserver', + "internet_nl_web_legacy_category_ipv6", + "internet_nl_web_legacy_ipv6_nameserver", + "internet_nl_web_legacy_ipv6_webserver", # Deleted on request # 'internet_nl_web_legacy_dane', - # added may 2020, api v2 - 'internet_nl_web_legacy_tls_1_3', + "internet_nl_web_legacy_tls_1_3", ] MAIL_OVERALL_FIELDS = WEB_OVERALL_FIELDS -MAIL_IPV6_CATEGORY = ['internet_nl_mail_dashboard_ipv6'] +MAIL_IPV6_CATEGORY = ["internet_nl_mail_dashboard_ipv6"] MAIL_IPV6_FIELDS = [ - # name servers - 'internet_nl_mail_ipv6_ns_address', - 'internet_nl_mail_ipv6_ns_reach', - + "internet_nl_mail_ipv6_ns_address", + "internet_nl_mail_ipv6_ns_reach", # mail server(s) - 'internet_nl_mail_ipv6_mx_address', - 'internet_nl_mail_ipv6_mx_reach', + "internet_nl_mail_ipv6_mx_address", + "internet_nl_mail_ipv6_mx_reach", ] -MAIL_DNSSEC_CATEGORY = ['internet_nl_mail_dashboard_dnssec'] +MAIL_DNSSEC_CATEGORY = ["internet_nl_mail_dashboard_dnssec"] MAIL_DNSSEC_FIELDS = [ # email address domain - 'internet_nl_mail_dnssec_mailto_exist', - 'internet_nl_mail_dnssec_mailto_valid', - + "internet_nl_mail_dnssec_mailto_exist", + "internet_nl_mail_dnssec_mailto_valid", # mail server domain(s) - 'internet_nl_mail_dnssec_mx_exist', - 'internet_nl_mail_dnssec_mx_valid', + "internet_nl_mail_dnssec_mx_exist", + "internet_nl_mail_dnssec_mx_valid", ] -MAIL_AUTH_CATEGORY = ['internet_nl_mail_dashboard_auth'] +MAIL_AUTH_CATEGORY = ["internet_nl_mail_dashboard_auth"] MAIL_AUTH_FIELDS = [ - # DMARC - 'internet_nl_mail_auth_dmarc_exist', - 'internet_nl_mail_auth_dmarc_policy', + "internet_nl_mail_auth_dmarc_exist", + "internet_nl_mail_auth_dmarc_policy", # 'internet_nl_mail_auth_dmarc_policy_only', # Added 24th of May 2019 # 'internet_nl_mail_auth_dmarc_ext_destination', # Added 24th of May 2019 - # DKIM - 'internet_nl_mail_auth_dkim_exist', - + "internet_nl_mail_auth_dkim_exist", # SPF - 'internet_nl_mail_auth_spf_exist', - 'internet_nl_mail_auth_spf_policy', - + "internet_nl_mail_auth_spf_exist", + "internet_nl_mail_auth_spf_policy", ] -MAIL_TLS_CATEGORY = ['internet_nl_mail_dashboard_tls'] +MAIL_TLS_CATEGORY = ["internet_nl_mail_dashboard_tls"] MAIL_TLS_TLS_FIELDS = [ - 'internet_nl_mail_starttls_tls_available', - 'internet_nl_mail_starttls_tls_version', - 'internet_nl_mail_starttls_tls_ciphers', + "internet_nl_mail_starttls_tls_available", + "internet_nl_mail_starttls_tls_version", + "internet_nl_mail_starttls_tls_ciphers", # api 2.0 tls 1.3 fields, may 2020 - 'internet_nl_mail_starttls_tls_cipherorder', - 'internet_nl_mail_starttls_tls_keyexchange', + "internet_nl_mail_starttls_tls_cipherorder", + "internet_nl_mail_starttls_tls_keyexchange", # api 2.0 tls 1.3 fields, may 2020 - 'internet_nl_mail_starttls_tls_keyexchangehash', - 'internet_nl_mail_starttls_tls_compress', - 'internet_nl_mail_starttls_tls_secreneg', - 'internet_nl_mail_starttls_tls_clientreneg', + "internet_nl_mail_starttls_tls_keyexchangehash", + "internet_nl_mail_starttls_tls_compress", + "internet_nl_mail_starttls_tls_secreneg", + "internet_nl_mail_starttls_tls_clientreneg", # api 2.0 tls 1.3 fields, may 2020 - 'internet_nl_mail_starttls_tls_0rtt', + "internet_nl_mail_starttls_tls_0rtt", ] MAIL_TLS_CERTIFICATE_FIELDS = [ - 'internet_nl_mail_starttls_cert_chain', - 'internet_nl_mail_starttls_cert_pubkey', - 'internet_nl_mail_starttls_cert_sig', - 'internet_nl_mail_starttls_cert_domain', + "internet_nl_mail_starttls_cert_chain", + "internet_nl_mail_starttls_cert_pubkey", + "internet_nl_mail_starttls_cert_sig", + "internet_nl_mail_starttls_cert_domain", ] MAIL_TLS_DANE_FIELDS = [ - 'internet_nl_mail_starttls_dane_exist', - 'internet_nl_mail_starttls_dane_valid', - 'internet_nl_mail_starttls_dane_rollover', + "internet_nl_mail_starttls_dane_exist", + "internet_nl_mail_starttls_dane_valid", + "internet_nl_mail_starttls_dane_rollover", ] -MAIL_RPKI_CATEGORY = ['internet_nl_mail_dashboard_rpki'] +MAIL_RPKI_CATEGORY = ["internet_nl_mail_dashboard_rpki"] MAIL_RPKI_FIELDS = [ - 'internet_nl_mail_rpki_exists', - 'internet_nl_mail_rpki_valid', - 'internet_nl_mail_ns_rpki_exists', - 'internet_nl_mail_ns_rpki_valid', - 'internet_nl_mail_mx_ns_rpki_exists', - 'internet_nl_mail_mx_ns_rpki_valid', + "internet_nl_mail_rpki_exists", + "internet_nl_mail_rpki_valid", + "internet_nl_mail_ns_rpki_exists", + "internet_nl_mail_ns_rpki_valid", + "internet_nl_mail_mx_ns_rpki_exists", + "internet_nl_mail_mx_ns_rpki_valid", ] MAIL_LEGACY_CATEGORY = ["internet_nl_mail_legacy_category"] MAIL_LEGACY_FIELDS = [ - 'internet_nl_mail_legacy_dmarc', - 'internet_nl_mail_legacy_dkim', - 'internet_nl_mail_legacy_spf', - 'internet_nl_mail_legacy_dmarc_policy', - 'internet_nl_mail_legacy_spf_policy', - 'internet_nl_mail_legacy_start_tls', - 'internet_nl_mail_legacy_start_tls_ncsc', - 'internet_nl_mail_legacy_dnssec_email_domain', - 'internet_nl_mail_legacy_dnssec_mx', - 'internet_nl_mail_legacy_dane', - 'internet_nl_mail_legacy_category_ipv6', - 'internet_nl_mail_legacy_ipv6_nameserver', - 'internet_nl_mail_legacy_ipv6_mailserver', - + "internet_nl_mail_legacy_dmarc", + "internet_nl_mail_legacy_dkim", + "internet_nl_mail_legacy_spf", + "internet_nl_mail_legacy_dmarc_policy", + "internet_nl_mail_legacy_spf_policy", + "internet_nl_mail_legacy_start_tls", + "internet_nl_mail_legacy_start_tls_ncsc", + "internet_nl_mail_legacy_dnssec_email_domain", + "internet_nl_mail_legacy_dnssec_mx", + "internet_nl_mail_legacy_dane", + "internet_nl_mail_legacy_category_ipv6", + "internet_nl_mail_legacy_ipv6_nameserver", + "internet_nl_mail_legacy_ipv6_mailserver", # Added may 2020 internet.nl api v2 # 'internet_nl_mail_legacy_mail_non_sending_domain', # non mail sending domain has been replaced with sending domain, to prevent double negatives. - 'internet_nl_mail_legacy_mail_sending_domain', - 'internet_nl_mail_legacy_mail_server_testable', - 'internet_nl_mail_legacy_mail_server_reachable', - 'internet_nl_mail_legacy_domain_has_mx', - 'internet_nl_mail_legacy_tls_1_3', + "internet_nl_mail_legacy_mail_sending_domain", + "internet_nl_mail_legacy_mail_server_testable", + "internet_nl_mail_legacy_mail_server_reachable", + "internet_nl_mail_legacy_domain_has_mx", + "internet_nl_mail_legacy_tls_1_3", ] # Obsoleted metrics: @@ -243,8 +234,14 @@ def operation_response( # 'internet_nl_mail_auth_dmarc_policy_only', # Added 24th of May 2019 # 'internet_nl_mail_auth_dmarc_ext_destination', # Added 24th of May 2019 -MAIL_CATEGORIES = MAIL_IPV6_CATEGORY + MAIL_DNSSEC_CATEGORY + MAIL_AUTH_CATEGORY + MAIL_TLS_CATEGORY + \ - MAIL_RPKI_CATEGORY + MAIL_LEGACY_CATEGORY +MAIL_CATEGORIES = ( + MAIL_IPV6_CATEGORY + + MAIL_DNSSEC_CATEGORY + + MAIL_AUTH_CATEGORY + + MAIL_TLS_CATEGORY + + MAIL_RPKI_CATEGORY + + MAIL_LEGACY_CATEGORY +) # When exporting, it also needs to be clear what field belongs to what category, otherwise duplicate # field names will make it unclear what is meant. See examples in issue: @@ -252,94 +249,78 @@ def operation_response( # https://github.com/internetstandards/Internet.nl-dashboard/issues/397 # perhaps at a later state the mapping from the frontend can be copied 1 to 1, but that's more mental luggage FIELD_TO_CATEGORY_MAP = { - 'internet_nl_web_ipv6_ns_address': 'category_web_ipv6_name_server', - 'internet_nl_web_ipv6_ns_reach': 'category_web_ipv6_name_server', - 'internet_nl_web_ipv6_ws_address': 'category_web_ipv6_web_server', - 'internet_nl_web_ipv6_ws_reach': 'category_web_ipv6_web_server', - 'internet_nl_web_ipv6_ws_similar': 'category_web_ipv6_web_server', - - 'internet_nl_web_dnssec_exist': 'category_web_dnssec_dnssec', - 'internet_nl_web_dnssec_valid': 'category_web_dnssec_dnssec', - - 'internet_nl_web_https_http_available': 'category_web_tls_http', - 'internet_nl_web_https_http_redirect': 'category_web_tls_http', - 'internet_nl_web_https_http_compress': 'category_web_tls_http', - 'internet_nl_web_https_http_hsts': 'category_web_tls_http', - - 'internet_nl_web_https_tls_version': 'category_web_tls_tls', - 'internet_nl_web_https_tls_ciphers': 'category_web_tls_tls', - 'internet_nl_web_https_tls_cipherorder': 'category_web_tls_tls', - 'internet_nl_web_https_tls_keyexchange': 'category_web_tls_tls', - 'internet_nl_web_https_tls_keyexchangehash': 'category_web_tls_tls', - 'internet_nl_web_https_tls_compress': 'category_web_tls_tls', - 'internet_nl_web_https_tls_secreneg': 'category_web_tls_tls', - 'internet_nl_web_https_tls_clientreneg': 'category_web_tls_tls', - 'internet_nl_web_https_tls_0rtt': 'category_web_tls_tls', - 'internet_nl_web_https_tls_ocsp': 'category_web_tls_tls', - - 'internet_nl_web_https_cert_chain': 'category_web_tls_certificate', - 'internet_nl_web_https_cert_pubkey': 'category_web_tls_certificate', - 'internet_nl_web_https_cert_sig': 'category_web_tls_certificate', - 'internet_nl_web_https_cert_domain': 'category_web_tls_certificate', - - 'internet_nl_web_https_dane_exist': 'category_web_tls_dane', - 'internet_nl_web_https_dane_valid': 'category_web_tls_dane', - - 'internet_nl_web_appsecpriv_x_frame_options': 'category_web_security_options_appsecpriv', - 'internet_nl_web_appsecpriv_x_content_type_options': 'category_web_security_options_appsecpriv', - 'internet_nl_web_appsecpriv_csp': 'category_web_security_options_appsecpriv', - 'internet_nl_web_appsecpriv_referrer_policy': 'category_web_security_options_appsecpriv', - 'internet_nl_web_appsecpriv_securitytxt': 'category_web_security_options_other', - - 'internet_nl_web_rpki_exists': 'category_web_rpki_name_server', - 'internet_nl_web_rpki_valid': 'category_web_rpki_name_server', - 'internet_nl_web_ns_rpki_exists': 'category_web_rpki_web_server', - 'internet_nl_web_ns_rpki_valid': 'category_web_rpki_web_server', - - 'internet_nl_mail_ipv6_ns_address': 'category_mail_ipv6_name_servers', - 'internet_nl_mail_ipv6_ns_reach': 'category_mail_ipv6_name_servers', - - 'internet_nl_mail_ipv6_mx_address': 'category_mail_ipv6_mail_servers', - 'internet_nl_mail_ipv6_mx_reach': 'category_mail_ipv6_mail_servers', - - 'internet_nl_mail_dnssec_mailto_exist': 'category_mail_dnssec_email_address_domain', - 'internet_nl_mail_dnssec_mailto_valid': 'category_mail_dnssec_email_address_domain', - 'internet_nl_mail_dnssec_mx_exist': 'category_mail_dnssec_mail_server_domain', - 'internet_nl_mail_dnssec_mx_valid': 'category_mail_dnssec_mail_server_domain', - - 'internet_nl_mail_auth_dmarc_exist': 'category_mail_dashboard_auth_dmarc', - 'internet_nl_mail_auth_dmarc_policy': 'category_mail_dashboard_auth_dmarc', - 'internet_nl_mail_auth_dkim_exist': 'category_mail_dashboard_aut_dkim', - 'internet_nl_mail_auth_spf_exist': 'category_mail_dashboard_aut_spf', - 'internet_nl_mail_auth_spf_policy': 'category_mail_dashboard_aut_spf', - - 'internet_nl_mail_starttls_tls_available': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_version': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_ciphers': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_cipherorder': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_keyexchange': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_keyexchangehash': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_compress': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_secreneg': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_clientreneg': 'category_mail_starttls_tls', - 'internet_nl_mail_starttls_tls_0rtt': 'category_mail_starttls_tls', - - 'internet_nl_mail_starttls_cert_chain': 'category_mail_starttls_certificate', - 'internet_nl_mail_starttls_cert_pubkey': 'category_mail_starttls_certificate', - 'internet_nl_mail_starttls_cert_sig': 'category_mail_starttls_certificate', - 'internet_nl_mail_starttls_cert_domain': 'category_mail_starttls_certificate', - - 'internet_nl_mail_starttls_dane_exist': 'category_mail_starttls_dane', - 'internet_nl_mail_starttls_dane_valid': 'category_mail_starttls_dane', - 'internet_nl_mail_starttls_dane_rollover': 'category_mail_starttls_dane', - - 'internet_nl_mail_rpki_exists': 'category_mail_rpki_name_server', - 'internet_nl_mail_rpki_valid': 'category_mail_rpki_name_server', - 'internet_nl_mail_ns_rpki_exists': 'category_mail_rpki_name_mail_server', - 'internet_nl_mail_ns_rpki_valid': 'category_mail_rpki_name_mail_server', - 'internet_nl_mail_mx_ns_rpki_exists': 'category_mail_rpki_mail_server', - 'internet_nl_mail_mx_ns_rpki_valid': 'category_mail_rpki_mail_server', - + "internet_nl_web_ipv6_ns_address": "category_web_ipv6_name_server", + "internet_nl_web_ipv6_ns_reach": "category_web_ipv6_name_server", + "internet_nl_web_ipv6_ws_address": "category_web_ipv6_web_server", + "internet_nl_web_ipv6_ws_reach": "category_web_ipv6_web_server", + "internet_nl_web_ipv6_ws_similar": "category_web_ipv6_web_server", + "internet_nl_web_dnssec_exist": "category_web_dnssec_dnssec", + "internet_nl_web_dnssec_valid": "category_web_dnssec_dnssec", + "internet_nl_web_https_http_available": "category_web_tls_http", + "internet_nl_web_https_http_redirect": "category_web_tls_http", + "internet_nl_web_https_http_compress": "category_web_tls_http", + "internet_nl_web_https_http_hsts": "category_web_tls_http", + "internet_nl_web_https_tls_version": "category_web_tls_tls", + "internet_nl_web_https_tls_ciphers": "category_web_tls_tls", + "internet_nl_web_https_tls_cipherorder": "category_web_tls_tls", + "internet_nl_web_https_tls_keyexchange": "category_web_tls_tls", + "internet_nl_web_https_tls_keyexchangehash": "category_web_tls_tls", + "internet_nl_web_https_tls_compress": "category_web_tls_tls", + "internet_nl_web_https_tls_secreneg": "category_web_tls_tls", + "internet_nl_web_https_tls_clientreneg": "category_web_tls_tls", + "internet_nl_web_https_tls_0rtt": "category_web_tls_tls", + "internet_nl_web_https_tls_ocsp": "category_web_tls_tls", + "internet_nl_web_https_cert_chain": "category_web_tls_certificate", + "internet_nl_web_https_cert_pubkey": "category_web_tls_certificate", + "internet_nl_web_https_cert_sig": "category_web_tls_certificate", + "internet_nl_web_https_cert_domain": "category_web_tls_certificate", + "internet_nl_web_https_dane_exist": "category_web_tls_dane", + "internet_nl_web_https_dane_valid": "category_web_tls_dane", + "internet_nl_web_appsecpriv_x_frame_options": "category_web_security_options_appsecpriv", + "internet_nl_web_appsecpriv_x_content_type_options": "category_web_security_options_appsecpriv", + "internet_nl_web_appsecpriv_csp": "category_web_security_options_appsecpriv", + "internet_nl_web_appsecpriv_referrer_policy": "category_web_security_options_appsecpriv", + "internet_nl_web_appsecpriv_securitytxt": "category_web_security_options_other", + "internet_nl_web_rpki_exists": "category_web_rpki_name_server", + "internet_nl_web_rpki_valid": "category_web_rpki_name_server", + "internet_nl_web_ns_rpki_exists": "category_web_rpki_web_server", + "internet_nl_web_ns_rpki_valid": "category_web_rpki_web_server", + "internet_nl_mail_ipv6_ns_address": "category_mail_ipv6_name_servers", + "internet_nl_mail_ipv6_ns_reach": "category_mail_ipv6_name_servers", + "internet_nl_mail_ipv6_mx_address": "category_mail_ipv6_mail_servers", + "internet_nl_mail_ipv6_mx_reach": "category_mail_ipv6_mail_servers", + "internet_nl_mail_dnssec_mailto_exist": "category_mail_dnssec_email_address_domain", + "internet_nl_mail_dnssec_mailto_valid": "category_mail_dnssec_email_address_domain", + "internet_nl_mail_dnssec_mx_exist": "category_mail_dnssec_mail_server_domain", + "internet_nl_mail_dnssec_mx_valid": "category_mail_dnssec_mail_server_domain", + "internet_nl_mail_auth_dmarc_exist": "category_mail_dashboard_auth_dmarc", + "internet_nl_mail_auth_dmarc_policy": "category_mail_dashboard_auth_dmarc", + "internet_nl_mail_auth_dkim_exist": "category_mail_dashboard_aut_dkim", + "internet_nl_mail_auth_spf_exist": "category_mail_dashboard_aut_spf", + "internet_nl_mail_auth_spf_policy": "category_mail_dashboard_aut_spf", + "internet_nl_mail_starttls_tls_available": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_version": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_ciphers": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_cipherorder": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_keyexchange": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_keyexchangehash": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_compress": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_secreneg": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_clientreneg": "category_mail_starttls_tls", + "internet_nl_mail_starttls_tls_0rtt": "category_mail_starttls_tls", + "internet_nl_mail_starttls_cert_chain": "category_mail_starttls_certificate", + "internet_nl_mail_starttls_cert_pubkey": "category_mail_starttls_certificate", + "internet_nl_mail_starttls_cert_sig": "category_mail_starttls_certificate", + "internet_nl_mail_starttls_cert_domain": "category_mail_starttls_certificate", + "internet_nl_mail_starttls_dane_exist": "category_mail_starttls_dane", + "internet_nl_mail_starttls_dane_valid": "category_mail_starttls_dane", + "internet_nl_mail_starttls_dane_rollover": "category_mail_starttls_dane", + "internet_nl_mail_rpki_exists": "category_mail_rpki_name_server", + "internet_nl_mail_rpki_valid": "category_mail_rpki_name_server", + "internet_nl_mail_ns_rpki_exists": "category_mail_rpki_name_mail_server", + "internet_nl_mail_ns_rpki_valid": "category_mail_rpki_name_mail_server", + "internet_nl_mail_mx_ns_rpki_exists": "category_mail_rpki_mail_server", + "internet_nl_mail_mx_ns_rpki_valid": "category_mail_rpki_mail_server", # 'internet_nl_web_legacy_category': 'internet_nl_web_legacy_category', # 'internet_nl_mail_legacy_category': 'internet_nl_mail_legacy_category', } diff --git a/dashboard/internet_nl_dashboard/logic/account.py b/dashboard/internet_nl_dashboard/logic/account.py index cac1ed76..c4e72346 100644 --- a/dashboard/internet_nl_dashboard/logic/account.py +++ b/dashboard/internet_nl_dashboard/logic/account.py @@ -3,7 +3,7 @@ def save_report_settings(account, report_settings): - account.report_settings = report_settings.get('filters', {}) + account.report_settings = report_settings.get("filters", {}) account.save() return operation_response(success=True, message="settings.updated") @@ -13,5 +13,5 @@ def get_report_settings(account): return operation_response( success=True, message="settings.restored_from_database", - data=account.report_settings if account.report_settings else {} + data=account.report_settings if account.report_settings else {}, ) diff --git a/dashboard/internet_nl_dashboard/logic/deduplication.py b/dashboard/internet_nl_dashboard/logic/deduplication.py index 9d4c3d8a..d7c1feed 100644 --- a/dashboard/internet_nl_dashboard/logic/deduplication.py +++ b/dashboard/internet_nl_dashboard/logic/deduplication.py @@ -16,13 +16,13 @@ def dedupe_urls(): for target_url_name in urls_by_name: print(f"Going to deduplicate stuff for domain: {target_url_name}") # Move everything to the oldest url. - target_url = Url.objects.all().filter(url=target_url_name).order_by('created_on').first() + target_url = Url.objects.all().filter(url=target_url_name).order_by("created_on").first() duplicate_urls = list(Url.objects.all().filter(url=target_url.url).exclude(id=target_url.id)) if not duplicate_urls: print("Could not find duplicates...") continue - print(f'Found {len(duplicate_urls)} for target url: {target_url}.') + print(f"Found {len(duplicate_urls)} for target url: {target_url}.") for duplicate_url in duplicate_urls: # transfer all endpoints and endpoints scans @@ -33,21 +33,24 @@ def dedupe_urls(): protocol=duplicate_endpoint.protocol, port=duplicate_endpoint.port, ip_version=duplicate_endpoint.ip_version, - is_dead=duplicate_endpoint.is_dead) + is_dead=duplicate_endpoint.is_dead, + ) if created: - print(f'A similar endpoint has been created at the target url. {duplicate_endpoint}') + print(f"A similar endpoint has been created at the target url. {duplicate_endpoint}") else: - print('A similar endpoint already exists in the target url.') + print("A similar endpoint already exists in the target url.") # duplicate scans in a day is not a problem, those are filtered out and only the latest one is used # so the data is consistent. EndpointGenericScan.objects.all().filter(endpoint=duplicate_endpoint).update(endpoint=endpoint_target) - print(f'Moved all the scans from endpoint {duplicate_endpoint} to {endpoint_target}.') + print(f"Moved all the scans from endpoint {duplicate_endpoint} to {endpoint_target}.") duplicate_endpoint.delete() # replace the duplicate_url with the original url in urllists. urllist_with_duplicate_urls = UrlList.objects.all().filter(urls__id=duplicate_url.id) - print(f'The duplicate url is being used in {len(urllist_with_duplicate_urls)} lists. It will be ' - f'replaced with the target url {target_url}.') + print( + f"The duplicate url is being used in {len(urllist_with_duplicate_urls)} lists. It will be " + f"replaced with the target url {target_url}." + ) for urllist_with_duplicate_url in urllist_with_duplicate_urls: urllist_with_duplicate_url.urls.remove(duplicate_url) urllist_with_duplicate_url.urls.add(target_url) diff --git a/dashboard/internet_nl_dashboard/logic/domains.py b/dashboard/internet_nl_dashboard/logic/domains.py index e1f422bd..3b37c73a 100644 --- a/dashboard/internet_nl_dashboard/logic/domains.py +++ b/dashboard/internet_nl_dashboard/logic/domains.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# pylint: disable=too-many-lines import logging import sys import time @@ -7,7 +8,6 @@ from typing import Any, Dict, List, Set, Tuple, Union import pyexcel as p -import requests import tldextract from actstream import action from celery import group @@ -21,8 +21,15 @@ from dashboard.celery import app from dashboard.internet_nl_dashboard.logic import operation_response -from dashboard.internet_nl_dashboard.models import (Account, AccountInternetNLScan, TaggedUrlInUrllist, UploadLog, - UrlList, UrlListReport, determine_next_scan_moment) +from dashboard.internet_nl_dashboard.models import ( + Account, + AccountInternetNLScan, + TaggedUrlInUrllist, + UploadLog, + UrlList, + UrlListReport, + determine_next_scan_moment, +) from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import initialize_scan, update_state from dashboard.internet_nl_dashboard.views.download_spreadsheet import create_spreadsheet_download @@ -64,35 +71,45 @@ def suggest_subdomains(domain: str, period: int = 370): def alter_url_in_urllist(account, data) -> Dict[str, Any]: # data = {'list_id': list.id, 'url_id': url.id, 'new_url_string': url.url} - expected_keys = ['list_id', 'url_id', 'new_url_string'] + expected_keys = ["list_id", "url_id", "new_url_string"] if not keys_are_present_in_object(expected_keys, data): return operation_response(error=True, message="Missing keys in data.") # what was the old id we're changing? - old_url = Url.objects.all().filter(pk=data['url_id']).first() + old_url = Url.objects.all().filter(pk=data["url_id"]).first() if not old_url: return operation_response(error=True, message="The old url does not exist.") - if old_url.url == data['new_url_string']: + if old_url.url == data["new_url_string"]: # no changes, for c - return operation_response(success=True, message="Saved.", data={ - 'created': {'id': old_url.id, 'url': old_url.url, 'created_on': old_url.created_on, - 'has_mail_endpoint': 'unknown', - 'has_web_endpoint': 'unknown', 'subdomain': old_url.computed_subdomain, - 'domain': old_url.computed_domain, 'suffix': old_url.computed_suffix}, - }) + return operation_response( + success=True, + message="Saved.", + data={ + "created": { + "id": old_url.id, + "url": old_url.url, + "created_on": old_url.created_on, + "has_mail_endpoint": "unknown", + "has_web_endpoint": "unknown", + "subdomain": old_url.computed_subdomain, + "domain": old_url.computed_domain, + "suffix": old_url.computed_suffix, + }, + }, + ) # is this really a list? - urllist = UrlList.objects.all().filter(account=account, pk=data['list_id']).first() + urllist = UrlList.objects.all().filter(account=account, pk=data["list_id"]).first() if not urllist: return operation_response(error=True, message="List does not exist.") # is the url valid? - if not Url.is_valid_url(data['new_url_string']): + if not Url.is_valid_url(data["new_url_string"]): return operation_response(error=True, message="New url does not have the correct format.") # fetch the url, or create it if it doesn't exist. - new_url, created = get_url(data['new_url_string']) + new_url, created = get_url(data["new_url_string"]) # don't throw away the url, only from the list. (don't call delete, as it will delete the record) urllist.urls.remove(old_url) @@ -103,36 +120,58 @@ def alter_url_in_urllist(account, data) -> Dict[str, Any]: urllist.save() # somewhat inefficient to do 4 queries, yet, good enough - old_url_has_mail_endpoint = Endpoint.objects.all().filter(url=old_url, is_dead=False, protocol='dns_soa').exists() - old_url_has_web_endpoint = Endpoint.objects.all().filter(url=old_url, is_dead=False, protocol='dns_a_aaa').exists() + old_url_has_mail_endpoint = Endpoint.objects.all().filter(url=old_url, is_dead=False, protocol="dns_soa").exists() + old_url_has_web_endpoint = Endpoint.objects.all().filter(url=old_url, is_dead=False, protocol="dns_a_aaa").exists() if not created: - new_url_has_mail_endpoint = Endpoint.objects.all().filter( - url=new_url, is_dead=False, protocol='dns_soa').exists() - new_url_has_web_endpoint = Endpoint.objects.all().filter( - url=new_url, is_dead=False, protocol='dns_a_aaa').exists() + new_url_has_mail_endpoint = ( + Endpoint.objects.all().filter(url=new_url, is_dead=False, protocol="dns_soa").exists() + ) + new_url_has_web_endpoint = ( + Endpoint.objects.all().filter(url=new_url, is_dead=False, protocol="dns_a_aaa").exists() + ) else: - new_url_has_mail_endpoint = 'unknown' - new_url_has_web_endpoint = 'unknown' + new_url_has_mail_endpoint = "unknown" + new_url_has_web_endpoint = "unknown" new_fragments = tldextract.extract(new_url.url) old_fragments = tldextract.extract(old_url.url) - return operation_response(success=True, message="Saved.", data={ - 'created': {'id': new_url.id, 'url': new_url.url, 'created_on': new_url.created_on, - 'has_mail_endpoint': new_url_has_mail_endpoint, - 'has_web_endpoint': new_url_has_web_endpoint, 'subdomain': new_fragments.subdomain, - 'domain': new_fragments.domain, 'suffix': new_fragments.suffix}, - 'removed': {'id': old_url.id, 'url': old_url.url, 'created_on': old_url.created_on, - 'has_mail_endpoint': old_url_has_mail_endpoint, - 'has_web_endpoint': old_url_has_web_endpoint, 'subdomain': old_fragments.subdomain, - 'domain': old_fragments.domain, 'suffix': old_fragments.suffix}, - }) + return operation_response( + success=True, + message="Saved.", + data={ + "created": { + "id": new_url.id, + "url": new_url.url, + "created_on": new_url.created_on, + "has_mail_endpoint": new_url_has_mail_endpoint, + "has_web_endpoint": new_url_has_web_endpoint, + "subdomain": new_fragments.subdomain, + "domain": new_fragments.domain, + "suffix": new_fragments.suffix, + }, + "removed": { + "id": old_url.id, + "url": old_url.url, + "created_on": old_url.created_on, + "has_mail_endpoint": old_url_has_mail_endpoint, + "has_web_endpoint": old_url_has_web_endpoint, + "subdomain": old_fragments.subdomain, + "domain": old_fragments.domain, + "suffix": old_fragments.suffix, + }, + }, + ) def scan_now(account, user_input) -> Dict[str, Any]: - urllist = UrlList.objects.all().filter( - account=account, id=user_input.get('id', -1), is_deleted=False).annotate(num_urls=Count('urls')).first() + urllist = ( + UrlList.objects.all() + .filter(account=account, id=user_input.get("id", -1), is_deleted=False) + .annotate(num_urls=Count("urls")) + .first() + ) if not urllist: return operation_response(error=True, message="List could not be found.") @@ -186,43 +225,52 @@ def get_url(new_url_string: str): def create_list(account: Account, user_input: Dict) -> Dict[str, Any]: - expected_keys = ['id', 'name', 'enable_scans', 'scan_type', 'automated_scan_frequency', 'scheduled_next_scan', - 'automatically_share_new_reports', 'default_public_share_code_for_new_reports', - 'enable_report_sharing_page'] + expected_keys = [ + "id", + "name", + "enable_scans", + "scan_type", + "automated_scan_frequency", + "scheduled_next_scan", + "automatically_share_new_reports", + "default_public_share_code_for_new_reports", + "enable_report_sharing_page", + ] if sorted(user_input.keys()) != sorted(expected_keys): return operation_response(error=True, message="Missing settings.") - frequency = validate_list_automated_scan_frequency(user_input['automated_scan_frequency']) + frequency = validate_list_automated_scan_frequency(user_input["automated_scan_frequency"]) data = { - 'account': account, - 'name': validate_list_name(user_input['name']), - 'enable_scans': bool(user_input['enable_scans']), - 'scan_type': validate_list_scan_type(user_input['scan_type']), - 'automated_scan_frequency': frequency, - 'scheduled_next_scan': determine_next_scan_moment(frequency), - 'enable_report_sharing_page': bool(user_input.get('enable_report_sharing_page', '')), - 'automatically_share_new_reports': bool(user_input.get('automatically_share_new_reports', '')), - 'default_public_share_code_for_new_reports': user_input.get( - 'default_public_share_code_for_new_reports', '')[:64] + "account": account, + "name": validate_list_name(user_input["name"]), + "enable_scans": bool(user_input["enable_scans"]), + "scan_type": validate_list_scan_type(user_input["scan_type"]), + "automated_scan_frequency": frequency, + "scheduled_next_scan": determine_next_scan_moment(frequency), + "enable_report_sharing_page": bool(user_input.get("enable_report_sharing_page", "")), + "automatically_share_new_reports": bool(user_input.get("automatically_share_new_reports", "")), + "default_public_share_code_for_new_reports": user_input.get("default_public_share_code_for_new_reports", "")[ + :64 + ], } urllist = UrlList(**data) urllist.save() # make sure the account is serializable. - data['account'] = account.id + data["account"] = account.id # adding the ID makes it possible to add new urls to a new list. - data['id'] = urllist.pk + data["id"] = urllist.pk # empty list, no warnings. - data['list_warnings'] = [] + data["list_warnings"] = [] # give a hint if it can be scanned: - data['scan_now_available'] = urllist.is_scan_now_available() + data["scan_now_available"] = urllist.is_scan_now_available() # Sprinkling an activity stream action. - action.send(account, verb='created list', target=urllist, public=False) + action.send(account, verb="created list", target=urllist, public=False) return operation_response(success=True, message="List created.", data=data) @@ -239,7 +287,7 @@ def delete_list(account: Account, user_input: dict): :param user_input: :return: """ - urllist = UrlList.objects.all().filter(account=account, id=user_input.get('id', -1), is_deleted=False).first() + urllist = UrlList.objects.all().filter(account=account, id=user_input.get("id", -1), is_deleted=False).first() if not urllist: return operation_response(error=True, message="List could not be deleted.") @@ -249,7 +297,7 @@ def delete_list(account: Account, user_input: dict): urllist.save() # Sprinkling an activity stream action. - action.send(account, verb='deleted list', target=urllist, public=False) + action.send(account, verb="deleted list", target=urllist, public=False) return operation_response(success=True, message="List deleted.") @@ -265,41 +313,48 @@ def get_scan_status_of_list(account: Account, list_id: int) -> Dict[str, Any]: """ prefetch_last_scan = Prefetch( - 'accountinternetnlscan_set', - queryset=AccountInternetNLScan.objects.order_by('-id').select_related('scan').only('scan_id', 'id', - 'finished_on', 'state', - 'urllist__id', 'urllist'), - to_attr='last_scan' + "accountinternetnlscan_set", + queryset=AccountInternetNLScan.objects.order_by("-id") + .select_related("scan") + .only("scan_id", "id", "finished_on", "state", "urllist__id", "urllist"), + to_attr="last_scan", ) last_report_prefetch = Prefetch( - 'urllistreport_set', + "urllistreport_set", # filter(pk=UrlListReport.objects.latest('id').pk). - queryset=UrlListReport.objects.order_by('-id').only('id', 'at_when', 'urllist__id', 'urllist'), - to_attr='last_report' + queryset=UrlListReport.objects.order_by("-id").only("id", "at_when", "urllist__id", "urllist"), + to_attr="last_report", ) - urllist = UrlList.objects.all().filter( - account=account, - id=list_id, - is_deleted=False - ).prefetch_related(prefetch_last_scan, last_report_prefetch).first() + urllist = ( + UrlList.objects.all() + .filter(account=account, id=list_id, is_deleted=False) + .prefetch_related(prefetch_last_scan, last_report_prefetch) + .first() + ) if not urllist: return {} - data = {'last_scan_id': None, 'last_scan_state': None, 'last_scan_finished': None, 'last_report_id': None, - 'last_report_date': None, 'scan_now_available': urllist.is_scan_now_available()} + data = { + "last_scan_id": None, + "last_scan_state": None, + "last_scan_finished": None, + "last_report_id": None, + "last_report_date": None, + "scan_now_available": urllist.is_scan_now_available(), + } if len(urllist.last_scan): # type: ignore # Mypy does not understand to_attr. "UrlList" has no attribute "last_scan" - data['last_scan_id'] = urllist.last_scan[0].scan.id # type: ignore - data['last_scan_state'] = urllist.last_scan[0].state # type: ignore - data['last_scan_finished'] = urllist.last_scan[0].state in ["finished", "cancelled"] # type: ignore + data["last_scan_id"] = urllist.last_scan[0].scan.id # type: ignore + data["last_scan_state"] = urllist.last_scan[0].state # type: ignore + data["last_scan_finished"] = urllist.last_scan[0].state in ["finished", "cancelled"] # type: ignore if len(urllist.last_report): # type: ignore - data['last_report_id'] = urllist.last_report[0].id # type: ignore - data['last_report_date'] = urllist.last_report[0].at_when # type: ignore + data["last_report_id"] = urllist.last_report[0].id # type: ignore + data["last_report_date"] = urllist.last_report[0].at_when # type: ignore return data @@ -316,10 +371,10 @@ def cancel_scan(account: Account, scan_id: int): if not scan: return operation_response(error=True, message="scan not found") - if scan.state == 'finished': + if scan.state == "finished": return operation_response(success=True, message="scan already finished") - if scan.state == 'cancelled': + if scan.state == "cancelled": return operation_response(success=True, message="scan already cancelled") scan.finished_on = datetime.now(timezone.utc) @@ -327,7 +382,7 @@ def cancel_scan(account: Account, scan_id: int): update_state("cancelled", scan.id) # Sprinkling an activity stream action. - action.send(account, verb='cancelled scan', target=scan, public=False) + action.send(account, verb="cancelled scan", target=scan, public=False) return operation_response(success=True, message="scan cancelled") @@ -352,28 +407,30 @@ def update_list_settings(account: Account, user_input: Dict) -> Dict[str, Any]: :return: """ - expected_keys = ['id', 'name', 'enable_scans', 'scan_type', 'automated_scan_frequency', 'scheduled_next_scan'] + expected_keys = ["id", "name", "enable_scans", "scan_type", "automated_scan_frequency", "scheduled_next_scan"] if not keys_are_present_in_object(expected_keys, user_input): return operation_response(error=True, message="Missing settings.") prefetch_last_scan = Prefetch( - 'accountinternetnlscan_set', - queryset=AccountInternetNLScan.objects.order_by('-id').select_related('scan'), - to_attr='last_scan' + "accountinternetnlscan_set", + queryset=AccountInternetNLScan.objects.order_by("-id").select_related("scan"), + to_attr="last_scan", ) last_report_prefetch = Prefetch( - 'urllistreport_set', + "urllistreport_set", # filter(pk=UrlListReport.objects.latest('id').pk). - queryset=UrlListReport.objects.order_by('-id').only('id', 'at_when', 'urllist__id'), - to_attr='last_report' + queryset=UrlListReport.objects.order_by("-id").only("id", "at_when", "urllist__id"), + to_attr="last_report", ) - urllist: UrlList = UrlList.objects.all().filter( - account=account, - id=user_input['id'], - is_deleted=False - ).annotate(num_urls=Count('urls')).prefetch_related(prefetch_last_scan, last_report_prefetch).first() + urllist: UrlList = ( + UrlList.objects.all() + .filter(account=account, id=user_input["id"], is_deleted=False) + .annotate(num_urls=Count("urls")) + .prefetch_related(prefetch_last_scan, last_report_prefetch) + .first() + ) if not urllist: return operation_response(error=True, message="No list of urls found.") @@ -381,54 +438,55 @@ def update_list_settings(account: Account, user_input: Dict) -> Dict[str, Any]: # Yes, you can try and set any value. Values that are not recognized do not result in errors / error messages, # instead they will be overwritten with the default. This means less interaction with users / less annoyance over # errors on such simple forms. - frequency = validate_list_automated_scan_frequency(user_input['automated_scan_frequency']) + frequency = validate_list_automated_scan_frequency(user_input["automated_scan_frequency"]) data = { - 'id': urllist.id, - 'account': account, - 'name': validate_list_name(user_input['name']), - 'enable_scans': bool(user_input['enable_scans']), - 'scan_type': validate_list_scan_type(user_input['scan_type']), - 'automated_scan_frequency': frequency, - 'scheduled_next_scan': determine_next_scan_moment(frequency), - 'enable_report_sharing_page': bool(user_input.get('enable_report_sharing_page', '')), - 'automatically_share_new_reports': bool(user_input.get('automatically_share_new_reports', '')), - 'default_public_share_code_for_new_reports': user_input.get( - 'default_public_share_code_for_new_reports', '')[:64] + "id": urllist.id, + "account": account, + "name": validate_list_name(user_input["name"]), + "enable_scans": bool(user_input["enable_scans"]), + "scan_type": validate_list_scan_type(user_input["scan_type"]), + "automated_scan_frequency": frequency, + "scheduled_next_scan": determine_next_scan_moment(frequency), + "enable_report_sharing_page": bool(user_input.get("enable_report_sharing_page", "")), + "automatically_share_new_reports": bool(user_input.get("automatically_share_new_reports", "")), + "default_public_share_code_for_new_reports": user_input.get("default_public_share_code_for_new_reports", "")[ + :64 + ], } updated_urllist = UrlList(**data) updated_urllist.save() # make sure the account is serializable, inject other data. - data['account'] = account.id - data['num_urls'] = urllist.num_urls - data['last_scan_id'] = None - data['last_scan_state'] = None - data['last_scan'] = None - data['last_scan_finished'] = None - data['last_report_id'] = None - data['last_report_date'] = None + data["account"] = account.id + data["num_urls"] = urllist.num_urls + data["last_scan_id"] = None + data["last_scan_state"] = None + data["last_scan"] = None + data["last_scan_finished"] = None + data["last_report_id"] = None + data["last_report_date"] = None if urllist.last_scan: - data['last_scan_id'] = urllist.last_scan[0].scan.id - data['last_scan_state'] = urllist.last_scan[0].state - data['last_scan'] = urllist.last_scan[0].started_on.isoformat() - data['last_scan_finished'] = urllist.last_scan[0].state in ["finished", "cancelled"] + data["last_scan_id"] = urllist.last_scan[0].scan.id + data["last_scan_state"] = urllist.last_scan[0].state + data["last_scan"] = urllist.last_scan[0].started_on.isoformat() + data["last_scan_finished"] = urllist.last_scan[0].state in ["finished", "cancelled"] if urllist.last_report: - data['last_report_id'] = urllist.last_report[0].id - data['last_report_date'] = urllist.last_report[0].at_when + data["last_report_id"] = urllist.last_report[0].id + data["last_report_date"] = urllist.last_report[0].at_when - data['scan_now_available'] = updated_urllist.is_scan_now_available() + data["scan_now_available"] = updated_urllist.is_scan_now_available() # list warnings (might do: make more generic, only if another list warning ever could occur.) list_warnings = [] if urllist.num_urls > config.DASHBOARD_MAXIMUM_DOMAINS_PER_LIST: - list_warnings.append('WARNING_DOMAINS_IN_LIST_EXCEED_MAXIMUM_ALLOWED') - data['list_warnings'] = [] + list_warnings.append("WARNING_DOMAINS_IN_LIST_EXCEED_MAXIMUM_ALLOWED") + data["list_warnings"] = [] # Sprinkling an activity stream action. - action.send(account, verb='updated list', target=updated_urllist, public=False) + action.send(account, verb="updated list", target=updated_urllist, public=False) return operation_response(success=True, message="Updated list settings", data=data) @@ -448,16 +506,17 @@ def validate_list_name(list_name): # todo: this can be a generic tuple check. def validate_list_automated_scan_frequency(automated_scan_frequency): - if (automated_scan_frequency, automated_scan_frequency) not in \ - UrlList._meta.get_field('automated_scan_frequency').choices: - return UrlList._meta.get_field('automated_scan_frequency').default + if (automated_scan_frequency, automated_scan_frequency) not in UrlList._meta.get_field( + "automated_scan_frequency" + ).choices: + return UrlList._meta.get_field("automated_scan_frequency").default return automated_scan_frequency def validate_list_scan_type(scan_type): # if the option doesn't exist, return the first option as the fallback / default. - if (scan_type, scan_type) not in UrlList._meta.get_field('scan_type').choices: - return UrlList._meta.get_field('scan_type').default + if (scan_type, scan_type) not in UrlList._meta.get_field("scan_type").choices: + return UrlList._meta.get_field("scan_type").default return scan_type @@ -494,12 +553,22 @@ def get_urllists_from_account(account: Account) -> Dict: """ # Could that num_urls slows things down. Given that num_urls is overwritten when loading list data... - urllists = UrlList.objects.all().filter( - account=account, - is_deleted=False - ).annotate(num_urls=Count('urls')).order_by('name').only( - 'id', 'name', 'enable_scans', 'scan_type', 'automated_scan_frequency', 'scheduled_next_scan', - 'enable_report_sharing_page', 'default_public_share_code_for_new_reports', 'automatically_share_new_reports', + urllists = ( + UrlList.objects.all() + .filter(account=account, is_deleted=False) + .annotate(num_urls=Count("urls")) + .order_by("name") + .only( + "id", + "name", + "enable_scans", + "scan_type", + "automated_scan_frequency", + "scheduled_next_scan", + "enable_report_sharing_page", + "default_public_share_code_for_new_reports", + "automatically_share_new_reports", + ) ) url_lists = [] @@ -509,57 +578,57 @@ def get_urllists_from_account(account: Account) -> Dict: for urllist in urllists: data = { - 'id': urllist.id, - 'name': urllist.name, - 'enable_scans': urllist.enable_scans, - 'scan_type': urllist.scan_type, - 'automated_scan_frequency': urllist.automated_scan_frequency, - 'scheduled_next_scan': urllist.scheduled_next_scan, - 'scan_now_available': urllist.is_scan_now_available(), - 'enable_report_sharing_page': urllist.enable_report_sharing_page, - 'automatically_share_new_reports': urllist.automatically_share_new_reports, - 'default_public_share_code_for_new_reports': urllist.default_public_share_code_for_new_reports, - - 'last_scan_id': None, - 'last_scan_state': None, - 'last_scan': None, - 'last_scan_finished': None, - - 'last_report_id': None, - 'last_report_date': None, - - 'list_warnings': [], - - 'num_urls': urllist.num_urls, + "id": urllist.id, + "name": urllist.name, + "enable_scans": urllist.enable_scans, + "scan_type": urllist.scan_type, + "automated_scan_frequency": urllist.automated_scan_frequency, + "scheduled_next_scan": urllist.scheduled_next_scan, + "scan_now_available": urllist.is_scan_now_available(), + "enable_report_sharing_page": urllist.enable_report_sharing_page, + "automatically_share_new_reports": urllist.automatically_share_new_reports, + "default_public_share_code_for_new_reports": urllist.default_public_share_code_for_new_reports, + "last_scan_id": None, + "last_scan_state": None, + "last_scan": None, + "last_scan_finished": None, + "last_report_id": None, + "last_report_date": None, + "list_warnings": [], + "num_urls": urllist.num_urls, } # this will create a warning if the number of domains in the list > max_domains # This is placed outside the loop to save a database query per time this is needed. if urllist.num_urls > max_domains: - data['list_warnings'].append('WARNING_DOMAINS_IN_LIST_EXCEED_MAXIMUM_ALLOWED') - - last_scan = AccountInternetNLScan.objects.all().filter(urllist=urllist).select_related('scan').only( - 'scan__id', 'state', 'started_on' - ).last() + data["list_warnings"].append("WARNING_DOMAINS_IN_LIST_EXCEED_MAXIMUM_ALLOWED") + + last_scan = ( + AccountInternetNLScan.objects.all() + .filter(urllist=urllist) + .select_related("scan") + .only("scan__id", "state", "started_on") + .last() + ) if last_scan and last_scan.scan and last_scan.started_on: - data['last_scan_id'] = last_scan.scan.id - data['last_scan_state'] = last_scan.state - data['last_scan'] = last_scan.started_on.isoformat() - data['last_scan_finished'] = last_scan.state in ["finished", "cancelled"] + data["last_scan_id"] = last_scan.scan.id + data["last_scan_state"] = last_scan.state + data["last_scan"] = last_scan.started_on.isoformat() + data["last_scan_finished"] = last_scan.state in ["finished", "cancelled"] # Selecting the whole object is extremely slow as the reports are very large, therefore we use .only to limit # the number of fields returned. Then the prefetch is pretty fast again. - last_report = UrlListReport.objects.all().filter(urllist=urllist).only('id', 'at_when').last() + last_report = UrlListReport.objects.all().filter(urllist=urllist).only("id", "at_when").last() if last_report: - data['last_report_id'] = last_report.id - data['last_report_date'] = last_report.at_when.isoformat() + data["last_report_id"] = last_report.id + data["last_report_date"] = last_report.at_when.isoformat() url_lists.append(data) # Sprinkling an activity stream action. - action.send(account, verb='retrieved domain lists', public=False) + action.send(account, verb="retrieved domain lists", public=False) - return {'lists': url_lists, 'maximum_domains_per_list': max_domains} + return {"lists": url_lists, "maximum_domains_per_list": max_domains} def get_urllist_content(account: Account, urllist_id: int) -> dict: @@ -572,33 +641,35 @@ def get_urllist_content(account: Account, urllist_id: int) -> dict: :return: """ # This prefetch changes a 1000 ms nested query into a 150 ms query. - prefetch = Prefetch('endpoint_set', - queryset=Endpoint.objects.filter(protocol__in=['dns_soa', 'dns_a_aaaa'], is_dead=False), - to_attr='relevant_endpoints') + prefetch = Prefetch( + "endpoint_set", + queryset=Endpoint.objects.filter(protocol__in=["dns_soa", "dns_a_aaaa"], is_dead=False), + to_attr="relevant_endpoints", + ) prefetch_tags = Prefetch( - 'taggedurlinurllist_set', - queryset=TaggedUrlInUrllist.objects.all().filter(urllist=urllist_id).prefetch_related('tags'), - to_attr='url_tags' + "taggedurlinurllist_set", + queryset=TaggedUrlInUrllist.objects.all().filter(urllist=urllist_id).prefetch_related("tags"), + to_attr="url_tags", ) # This ordering makes sure all subdomains are near the domains with the right extension. - urls = Url.objects.all().filter( - urls_in_dashboard_list_2__account=account, - urls_in_dashboard_list_2__id=urllist_id - ).order_by('computed_domain', 'computed_suffix', 'computed_subdomain').prefetch_related( - prefetch, - prefetch_tags - ).all() + urls = ( + Url.objects.all() + .filter(urls_in_dashboard_list_2__account=account, urls_in_dashboard_list_2__id=urllist_id) + .order_by("computed_domain", "computed_suffix", "computed_subdomain") + .prefetch_related(prefetch, prefetch_tags) + .all() + ) """ It's very possible that the urrlist_id is not matching with the account. The query will just return nothing. Only of both matches it will return something we can work with. """ - response: Dict[str, Any] = {'urllist_id': urllist_id, 'urls': []} + response: Dict[str, Any] = {"urllist_id": urllist_id, "urls": []} """ This is just a simple iteration, all sorting and logic is placed in the vue as that is much more flexible. """ for url in urls: - has_mail_endpoint = len([x for x in url.relevant_endpoints if x.protocol == 'dns_soa']) > 0 - has_web_endpoint = len([x for x in url.relevant_endpoints if x.protocol == 'dns_a_aaaa']) > 0 + has_mail_endpoint = len([x for x in url.relevant_endpoints if x.protocol == "dns_soa"]) > 0 + has_web_endpoint = len([x for x in url.relevant_endpoints if x.protocol == "dns_a_aaaa"]) > 0 # this is terrible code... as it appears prefetching tags is bit complex with queries in # queries with n-to-n on n-to-n and such. @@ -608,22 +679,24 @@ def get_urllist_content(account: Account, urllist_id: int) -> dict: # tags = list(has_tags.tags.names()) if has_tags else [] # log.warning(url.url_tags) tags = [] - for tag1 in [x.tags.values_list('name') for x in url.url_tags]: + for tag1 in [x.tags.values_list("name") for x in url.url_tags]: for tag2 in tag1: tags.extend(iter(tag2)) - response['urls'].append({ - 'id': url.id, - 'url': url.url, - 'subdomain': url.computed_subdomain, - 'domain': url.computed_domain, - 'suffix': url.computed_suffix, - 'created_on': url.created_on, - 'resolves': not url.not_resolvable, - 'has_mail_endpoint': has_mail_endpoint, - 'has_web_endpoint': has_web_endpoint, - 'tags': tags, - }) + response["urls"].append( + { + "id": url.id, + "url": url.url, + "subdomain": url.computed_subdomain, + "domain": url.computed_domain, + "suffix": url.computed_suffix, + "created_on": url.created_on, + "resolves": not url.not_resolvable, + "has_mail_endpoint": has_mail_endpoint, + "has_web_endpoint": has_web_endpoint, + "tags": tags, + } + ) return response @@ -706,8 +779,8 @@ def add_domains_from_raw_user_data(account: Account, user_input: Dict[str, Any]) """ # how could we validate user_input a better way? Using a validator object? - list_id: int = int(user_input.get('list_id', -1)) - unfiltered_urls: str = user_input.get('urls', []) + list_id: int = int(user_input.get("list_id", -1)) + unfiltered_urls: str = user_input.get("urls", []) # these are random unfiltered strings and the method expects keys... # in this case we'll run an extra retrieve_possible_urls_from_unfiltered_input so there is already some filtering. @@ -721,8 +794,13 @@ def add_domains_from_raw_user_data(account: Account, user_input: Dict[str, Any]) return operation_response(success=True, message="add_domains_valid_urls_added", data=result) -def save_urllist_content_by_name(account: Account, urllist_name: str, urls: Dict[str, Dict[str, list]], - uploadlog_id: int = None, pending_message: str = None) -> dict: +def save_urllist_content_by_name( + account: Account, + urllist_name: str, + urls: Dict[str, Dict[str, list]], + uploadlog_id: int = None, + pending_message: str = None, +) -> dict: """ This 'by name' variant is a best guess when a spreadsheet upload with list names is used. @@ -742,8 +820,13 @@ def save_urllist_content_by_name(account: Account, urllist_name: str, urls: Dict return save_urllist_content_by_id(account, urllist.id, urls, uploadlog_id, pending_message) -def save_urllist_content_by_id(account: Account, urllist_id: id, unfiltered_urls: Dict[str, Dict[str, list]], - uploadlog_id: int = None, pending_message: str = None) -> dict: +def save_urllist_content_by_id( + account: Account, + urllist_id: id, + unfiltered_urls: Dict[str, Dict[str, list]], + uploadlog_id: int = None, + pending_message: str = None, +) -> dict: urllist = UrlList.objects.all().filter(account=account, id=urllist_id, is_deleted=False).first() if not urllist: @@ -753,33 +836,43 @@ def save_urllist_content_by_id(account: Account, urllist_id: id, unfiltered_urls urls, duplicates_removed = retrieve_possible_urls_from_unfiltered_input(", ".join(unfiltered_urls.keys())) cleaned_urls: Dict[str, List[str]] = clean_urls(urls) # type: ignore - proposed_number_of_urls = urllist.urls.all().count() + len(cleaned_urls['correct']) - if proposed_number_of_urls > int(constance_cached_value('DASHBOARD_MAXIMUM_DOMAINS_PER_LIST')): + proposed_number_of_urls = urllist.urls.all().count() + len(cleaned_urls["correct"]) + if proposed_number_of_urls > int(constance_cached_value("DASHBOARD_MAXIMUM_DOMAINS_PER_LIST")): return operation_response(error=True, message="too_many_domains") - if cleaned_urls['correct']: + if cleaned_urls["correct"]: # this operation takes a while, to speed it up urls are added async. - counters = _add_to_urls_to_urllist(account, urllist, urls=cleaned_urls['correct'], - urls_with_tags_mapping=unfiltered_urls, - uploadlog_id=uploadlog_id, pending_message=pending_message) + counters = _add_to_urls_to_urllist( + account, + urllist, + urls=cleaned_urls["correct"], + urls_with_tags_mapping=unfiltered_urls, + uploadlog_id=uploadlog_id, + pending_message=pending_message, + ) else: - counters = {'added_to_list': 0, 'already_in_list': 0} - - update_spreadsheet_upload_(uploadlog_id, "[2/3] Processing", - f"{pending_message} Added {counters['added_to_list']} to {urllist.name}. " - f"{counters['already_in_list']} already list.", percentage=0) + counters = {"added_to_list": 0, "already_in_list": 0} + + update_spreadsheet_upload_( + uploadlog_id, + "[2/3] Processing", + f"{pending_message} Added {counters['added_to_list']} to {urllist.name}. " + f"{counters['already_in_list']} already list.", + percentage=0, + ) return { - 'incorrect_urls': cleaned_urls['incorrect'], - 'added_to_list': counters['added_to_list'], - 'already_in_list': counters['already_in_list'], - 'duplicates_removed': duplicates_removed, + "incorrect_urls": cleaned_urls["incorrect"], + "added_to_list": counters["added_to_list"], + "already_in_list": counters["already_in_list"], + "duplicates_removed": duplicates_removed, } @app.task(ignore_result=True) -def update_spreadsheet_upload_(upload_id: int, status: str = "pending", message: str = "", - percentage: int = -1) -> None: +def update_spreadsheet_upload_( + upload_id: int, status: str = "pending", message: str = "", percentage: int = -1 +) -> None: # double to prevent circulair import. This is not nice and should be removed. # user feedback is important on large uploads, as it may take a few minutes to hours it's nice to # get some feedback on how much stuff has been processed (if possible). @@ -800,17 +893,19 @@ def sleep(): time.sleep(2) -def _add_to_urls_to_urllist( # pylint: disable=too-many-arguments - account: Account, - current_list: UrlList, - urls: List[str], - urls_with_tags_mapping: Dict[str, Dict[str, list]] = None, - uploadlog_id: int = None, - pending_message: str = None +def _add_to_urls_to_urllist( # pylint: disable=too-many-positional-arguments too-many-arguments + account: Account, + current_list: UrlList, + urls: List[str], + urls_with_tags_mapping: Dict[str, Dict[str, list]] = None, + uploadlog_id: int = None, + pending_message: str = None, ) -> Dict[str, Any]: - already_existing_urls = UrlList.objects.all().filter( - account=account, id=current_list.id, urls__url__in=urls - ).values_list('urls__url', flat=True) + already_existing_urls = ( + UrlList.objects.all() + .filter(account=account, id=current_list.id, urls__url__in=urls) + .values_list("urls__url", flat=True) + ) new_urls = list(set(urls) - set(already_existing_urls)) @@ -822,15 +917,18 @@ def _add_to_urls_to_urllist( # pylint: disable=too-many-arguments tasks = [] for position, url in enumerate(sorted(new_urls)): tasks.append( - group(add_new_url_to_list_async.si(url, current_list.id, urls_with_tags_mapping) - # discovering endpoints is not a hard requirements, this is done before a scan starts anyway - # and will only slow down creating the list. - # | discover_endpoints.s() - | update_spreadsheet_upload_.si(uploadlog_id, "[3/3] Processing", - f"{pending_message}. Added: {url} to {current_list.name}.", - percentage=round(100 * position / len(new_urls), 0) - ) - ) + group( + add_new_url_to_list_async.si(url, current_list.id, urls_with_tags_mapping) + # discovering endpoints is not a hard requirements, this is done before a scan starts anyway + # and will only slow down creating the list. + # | discover_endpoints.s() + | update_spreadsheet_upload_.si( + uploadlog_id, + "[3/3] Processing", + f"{pending_message}. Added: {url} to {current_list.name}.", + percentage=round(100 * position / len(new_urls), 0), + ) + ) ) # This might be performed earlier than above tasks in the queue @@ -838,10 +936,9 @@ def _add_to_urls_to_urllist( # pylint: disable=too-many-arguments tasks.append( group( # simulate the time it takes to add something... - sleep.si() | - update_spreadsheet_upload_.si( - uploadlog_id, "[3/3] Processing", - f"[3/3] Processing completed for list \"{current_list.name}\".", 100 + sleep.si() + | update_spreadsheet_upload_.si( + uploadlog_id, "[3/3] Processing", f'[3/3] Processing completed for list "{current_list.name}".', 100 ) ) ) @@ -857,8 +954,8 @@ def _add_to_urls_to_urllist( # pylint: disable=too-many-arguments task.apply_async() return { - 'added_to_list': len(new_urls), - 'already_in_list': len(already_existing_urls), + "added_to_list": len(new_urls), + "already_in_list": len(already_existing_urls), } @@ -880,7 +977,7 @@ def add_new_url_to_list_async(url: str, current_list_id: int, urls_with_tags_map @app.task(queue="storage", ignore_result=True) def discover_endpoints(url_id): # always try to find a few dns endpoints... - compose_discover_task(urls_filter={'pk': url_id}).apply_async() + compose_discover_task(urls_filter={"pk": url_id}).apply_async() def add_tags_to_urls_in_urllist(existing_url: Url, current_list: UrlList, tags: List[str]) -> None: @@ -891,29 +988,33 @@ def add_tags_to_urls_in_urllist(existing_url: Url, current_list: UrlList, tags: def _add_to_urls_to_urllist_nicer(account: Account, current_list: UrlList, urls: List[str]) -> Dict[str, List[str]]: - counters: Dict[str, List[str]] = {'added_to_list': [], 'already_in_list': [], 'added_to_list_already_in_db': [], - 'added_to_list_new_in_db': []} + counters: Dict[str, List[str]] = { + "added_to_list": [], + "already_in_list": [], + "added_to_list_already_in_db": [], + "added_to_list_new_in_db": [], + } for url in urls: if UrlList.objects.all().filter(account=account, id=current_list.id, urls__url__iexact=url).exists(): - counters['already_in_list'].append(url) + counters["already_in_list"].append(url) continue # if url already in database, we only need to add it to the list: if existing_url := Url.objects.all().filter(url=url).first(): current_list.urls.add(existing_url) - counters['added_to_list'].append(url) - counters['added_to_list_already_in_db'].append(url) + counters["added_to_list"].append(url) + counters["added_to_list_already_in_db"].append(url) else: new_url = Url.add(url) # always try to find a few dns endpoints... - compose_discover_task(urls_filter={'pk': new_url.id}).apply_async() + compose_discover_task(urls_filter={"pk": new_url.id}).apply_async() current_list.urls.add(new_url) - counters['added_to_list'].append(url) - counters['added_to_list_new_in_db'].append(url) + counters["added_to_list"].append(url) + counters["added_to_list_new_in_db"].append(url) return counters @@ -928,16 +1029,16 @@ def clean_urls(urls: List[str]) -> Dict[str, List[str]]: :return: """ - result: Dict[str, List[Union[str, int]]] = {'incorrect': [], 'correct': []} + result: Dict[str, List[Union[str, int]]] = {"incorrect": [], "correct": []} for url in urls: # all urls in the system must be lowercase (if applicable to used character) url = url.lower() if not Url.is_valid_url(url): - result['incorrect'].append(url) + result["incorrect"].append(url) else: - result['correct'].append(url) + result["correct"].append(url) return result @@ -946,7 +1047,7 @@ def get_or_create_list_by_name(account, name: str, scan_type: str = "web") -> Ur if existing_list := UrlList.objects.all().filter(account=account, name=name, is_deleted=False).first(): return existing_list - urllist = UrlList(**{'name': name, 'account': account, 'scan_type': scan_type}) + urllist = UrlList(**{"name": name, "account": account, "scan_type": scan_type}) urllist.save() return urllist @@ -964,10 +1065,11 @@ def delete_url_from_urllist(account: Account, urllist_id: int, url_id: int) -> b # make sure that the url is in this list and for the current account # we don't want other users to be able to delete urls of other lists. - url_is_in_list = Url.objects.all().filter( - urls_in_dashboard_list_2__account=account, - urls_in_dashboard_list_2__id=urllist_id, - id=url_id).first() + url_is_in_list = ( + Url.objects.all() + .filter(urls_in_dashboard_list_2__account=account, urls_in_dashboard_list_2__id=urllist_id, id=url_id) + .first() + ) if not url_is_in_list: return False @@ -979,10 +1081,11 @@ def delete_url_from_urllist(account: Account, urllist_id: int, url_id: int) -> b def download_as_spreadsheet(account: Account, urllist_id: int, file_type: str = "xlsx") -> Any: - urls = TaggedUrlInUrllist.objects.all().filter( - urllist__account=account, - urllist__pk=urllist_id - ).order_by('url__computed_domain', 'url__computed_subdomain') + urls = ( + TaggedUrlInUrllist.objects.all() + .filter(urllist__account=account, urllist__pk=urllist_id) + .order_by("url__computed_domain", "url__computed_subdomain") + ) if not urls: return JsonResponse({}) @@ -992,7 +1095,7 @@ def download_as_spreadsheet(account: Account, urllist_id: int, file_type: str = data += [["List(s)", "Domain(s)", "Tags"]] for url in urls.all(): - data += [[url.urllist.name, url.url.url, ", ".join(url.tags.values_list('name', flat=True))]] + data += [[url.urllist.name, url.url.url, ", ".join(url.tags.values_list("name", flat=True))]] book = p.get_book(bookdict={"Domains": data}) diff --git a/dashboard/internet_nl_dashboard/logic/internet_nl_translations.py b/dashboard/internet_nl_dashboard/logic/internet_nl_translations.py index e96eaa45..9b06b323 100644 --- a/dashboard/internet_nl_dashboard/logic/internet_nl_translations.py +++ b/dashboard/internet_nl_dashboard/logic/internet_nl_translations.py @@ -11,13 +11,13 @@ from django.utils.text import slugify # Todo: refactor this to languages from settings.py -SUPPORTED_LOCALES: List[str] = ['nl', 'en'] +SUPPORTED_LOCALES: List[str] = ["nl", "en"] log = logging.getLogger(__package__) # .parent = logic; .parent.parent = internet_nl_dashboard; DASHBOARD_APP_DIRECTORY = Path(__file__).parent.parent -VUE_I18N_OUTPUT_PATH = f'{DASHBOARD_APP_DIRECTORY}/static/js/translations/' +VUE_I18N_OUTPUT_PATH = f"{DASHBOARD_APP_DIRECTORY}/static/js/translations/" DJANGO_I18N_OUTPUT_PATH = f"{DASHBOARD_APP_DIRECTORY}/locale/" # log.debug(f"VUE_I18N_OUTPUT_PATH: {VUE_I18N_OUTPUT_PATH}") # log.debug(f"DJANGO_I18N_OUTPUT_PATH: {DJANGO_I18N_OUTPUT_PATH}") @@ -59,15 +59,15 @@ def convert_internet_nl_content_to_vue(): raw_content: bytes = get_locale_content(locale) store_as_django_locale(locale, raw_content) structured_content = load_as_po_file(raw_content) - translated_locales.append({'locale': locale, 'content': structured_content}) + translated_locales.append({"locale": locale, "content": structured_content}) # support a per-language kind of file, in case we're going to do dynamic loading of languages. vue_i18n_content: str = convert_vue_i18n_format(locale, structured_content) combined_vue_i18n_content += vue_i18n_content - store_vue_i18n_file(f'internet_nl.{locale}', vue_i18n_content) + store_vue_i18n_file(f"internet_nl.{locale}", vue_i18n_content) # the locales are easiest stored together. This makes language switching a lot easier. - store_vue_i18n_file('internet_nl', combined_vue_i18n_content) + store_vue_i18n_file("internet_nl", combined_vue_i18n_content) def get_locale_content(locale: str) -> bytes: @@ -106,8 +106,8 @@ def store_as_django_locale(locale, content): # If the language does not exist yet, make the folder supporting this language. os.makedirs(Path(filepath).parent, exist_ok=True) - with open(filepath, 'w', encoding="UTF-8") as file: - file.write(content.decode('UTF-8')) + with open(filepath, "w", encoding="UTF-8") as file: + file.write(content.decode("UTF-8")) def load_as_po_file(raw_content: bytes) -> List[Any]: @@ -162,7 +162,7 @@ def convert_vue_i18n_format(locale: str, po_content: Any) -> str: for entry in po_content: # to save a boatload of data, we're not storing the 'content' from the pages of internet.nl # we'll just have to point to this content. - if entry.msgid.endswith('content'): + if entry.msgid.endswith("content"): continue content += f" {_js_safe_msgid(entry.msgid)}: '{_js_safe_msgstr(entry.msgstr)}',\n" @@ -176,14 +176,14 @@ def get_po_as_dictionary(file): structured_content = polib.pofile(file) po_file_as_dictionary = {} for entry in structured_content: - if entry.msgid.endswith('content'): + if entry.msgid.endswith("content"): continue po_file_as_dictionary[_js_safe_msgid(entry.msgid)] = _js_safe_msgstr(entry.msgstr) return po_file_as_dictionary -def get_po_as_dictionary_v2(language='en'): +def get_po_as_dictionary_v2(language="en"): """Much easier to use with only the language parameters and no magic or miserable path stuff.""" po_file_location = f"{DJANGO_I18N_OUTPUT_PATH}{language}/LC_MESSAGES/django.po" @@ -191,9 +191,11 @@ def get_po_as_dictionary_v2(language='en'): log.debug(f"Loading locale file: {po_file_location}") return get_po_as_dictionary(po_file_location) except OSError as error: - raise SystemError(f"Missing PO file 'django.po' at {po_file_location}. Note that an exception about " - f"incorrect syntax may be misleading. This is also given when there is no file. " - f"The exception that is given: {error}. Is this language available?") from error + raise SystemError( + f"Missing PO file 'django.po' at {po_file_location}. Note that an exception about " + f"incorrect syntax may be misleading. This is also given when there is no file. " + f"The exception that is given: {error}. Is this language available?" + ) from error def _vue_format_start() -> str: @@ -218,19 +220,19 @@ def _vue_format_end() -> str: def _js_safe_msgid(text): - return slugify(text).replace('-', '_') + return slugify(text).replace("-", "_") def _js_safe_msgstr(msgstr): # a poor mans escape for single quotes. - msgstr = msgstr.replace("'", "\\\'") + msgstr = msgstr.replace("'", "\\'") html = markdown.markdown(msgstr) - one_line_html = html.replace('\n', '') + one_line_html = html.replace("\n", "") # as happening on internet.nl, if the line is just a single paragraph, the paragraph tags are removed. # this is convenient for use in labels etc, that have their own markup. # see: https://github.com/NLnetLabs/Internet.nl/blob/cece8255ac7f39bded137f67c94a10748970c3c7/bin/pofiles.py - one_line_html = _strip_simple_item(one_line_html, 'p') + one_line_html = _strip_simple_item(one_line_html, "p") return one_line_html @@ -241,8 +243,8 @@ def _strip_simple_item(text, html_tag): # Opening: <> and closing: len_opening_tag = len(html_tag) + 2 len_closing_tag = len_opening_tag + 1 - - return text[len_opening_tag:len(text) - len_closing_tag] + finish = len(text) - len_closing_tag + return text[len_opening_tag:finish] return text @@ -256,7 +258,7 @@ def store_vue_i18n_file(filename: str, content: str) -> None: :param content: :return: """ - with open(f"{VUE_I18N_OUTPUT_PATH}{filename}.js", 'w', encoding="UTF-8") as file: + with open(f"{VUE_I18N_OUTPUT_PATH}{filename}.js", "w", encoding="UTF-8") as file: file.write(content) @@ -271,163 +273,151 @@ def translate_field(field_label, translation_dictionary: Dict[str, str]): """ field_mapping = { # mail fields, see dashboard.js - 'internet_nl_mail_starttls_cert_domain': 'detail_mail_tls_cert_hostmatch_label', - 'internet_nl_mail_starttls_tls_version': 'detail_mail_tls_version_label', - 'internet_nl_mail_starttls_cert_chain': 'detail_mail_tls_cert_trust_label', - 'internet_nl_mail_starttls_tls_available': 'detail_mail_tls_starttls_exists_label', - 'internet_nl_mail_starttls_tls_clientreneg': 'detail_mail_tls_renegotiation_client_label', - 'internet_nl_mail_starttls_tls_ciphers': 'detail_mail_tls_ciphers_label', - 'internet_nl_mail_starttls_dane_valid': 'detail_mail_tls_dane_valid_label', - 'internet_nl_mail_starttls_dane_exist': 'detail_mail_tls_dane_exists_label', - 'internet_nl_mail_starttls_tls_secreneg': 'detail_mail_tls_renegotiation_secure_label', - 'internet_nl_mail_starttls_dane_rollover': 'detail_mail_tls_dane_rollover_label', - 'internet_nl_mail_starttls_cert_pubkey': 'detail_mail_tls_cert_pubkey_label', - 'internet_nl_mail_starttls_cert_sig': 'detail_mail_tls_cert_signature_label', - 'internet_nl_mail_starttls_tls_compress': 'detail_mail_tls_compression_label', - 'internet_nl_mail_starttls_tls_keyexchange': 'detail_mail_tls_fs_params_label', - 'internet_nl_mail_auth_dmarc_policy': 'detail_mail_auth_dmarc_policy_label', - 'internet_nl_mail_auth_dmarc_exist': 'detail_mail_auth_dmarc_label', - 'internet_nl_mail_auth_spf_policy': 'detail_mail_auth_spf_policy_label', - 'internet_nl_mail_auth_dkim_exist': 'detail_mail_auth_dkim_label', - 'internet_nl_mail_auth_spf_exist': 'detail_mail_auth_spf_label', - 'internet_nl_mail_dnssec_mailto_exist': 'detail_mail_dnssec_exists_label', - 'internet_nl_mail_dnssec_mailto_valid': 'detail_mail_dnssec_valid_label', - 'internet_nl_mail_dnssec_mx_valid': 'detail_mail_dnssec_mx_valid_label', - 'internet_nl_mail_dnssec_mx_exist': 'detail_mail_dnssec_mx_exists_label', - 'internet_nl_mail_ipv6_mx_address': 'detail_mail_ipv6_mx_aaaa_label', - 'internet_nl_mail_ipv6_mx_reach': 'detail_mail_ipv6_mx_reach_label', - 'internet_nl_mail_ipv6_ns_reach': 'detail_web_mail_ipv6_ns_reach_label', - 'internet_nl_mail_ipv6_ns_address': 'detail_web_mail_ipv6_ns_aaaa_label', - 'internet_nl_mail_starttls_tls_cipherorder': 'detail_mail_tls_cipher_order_label', - 'internet_nl_mail_starttls_tls_keyexchangehash': 'detail_mail_tls_kex_hash_func_label', - 'internet_nl_mail_starttls_tls_0rtt': 'detail_mail_tls_zero_rtt_label', - 'internet_nl_mail_rpki_exists': 'detail_mail_rpki_exists_label', - 'internet_nl_mail_rpki_valid': 'detail_mail_rpki_valid_label', - 'internet_nl_mail_ns_rpki_exists': 'detail_web_mail_rpki_ns_exists_label', - 'internet_nl_mail_ns_rpki_valid': 'detail_web_mail_rpki_ns_valid_label', - 'internet_nl_mail_mx_ns_rpki_exists': 'detail_mail_rpki_mx_ns_exists_label', - 'internet_nl_mail_mx_ns_rpki_valid': 'detail_mail_rpki_mx_ns_valid_label', - - + "internet_nl_mail_starttls_cert_domain": "detail_mail_tls_cert_hostmatch_label", + "internet_nl_mail_starttls_tls_version": "detail_mail_tls_version_label", + "internet_nl_mail_starttls_cert_chain": "detail_mail_tls_cert_trust_label", + "internet_nl_mail_starttls_tls_available": "detail_mail_tls_starttls_exists_label", + "internet_nl_mail_starttls_tls_clientreneg": "detail_mail_tls_renegotiation_client_label", + "internet_nl_mail_starttls_tls_ciphers": "detail_mail_tls_ciphers_label", + "internet_nl_mail_starttls_dane_valid": "detail_mail_tls_dane_valid_label", + "internet_nl_mail_starttls_dane_exist": "detail_mail_tls_dane_exists_label", + "internet_nl_mail_starttls_tls_secreneg": "detail_mail_tls_renegotiation_secure_label", + "internet_nl_mail_starttls_dane_rollover": "detail_mail_tls_dane_rollover_label", + "internet_nl_mail_starttls_cert_pubkey": "detail_mail_tls_cert_pubkey_label", + "internet_nl_mail_starttls_cert_sig": "detail_mail_tls_cert_signature_label", + "internet_nl_mail_starttls_tls_compress": "detail_mail_tls_compression_label", + "internet_nl_mail_starttls_tls_keyexchange": "detail_mail_tls_fs_params_label", + "internet_nl_mail_auth_dmarc_policy": "detail_mail_auth_dmarc_policy_label", + "internet_nl_mail_auth_dmarc_exist": "detail_mail_auth_dmarc_label", + "internet_nl_mail_auth_spf_policy": "detail_mail_auth_spf_policy_label", + "internet_nl_mail_auth_dkim_exist": "detail_mail_auth_dkim_label", + "internet_nl_mail_auth_spf_exist": "detail_mail_auth_spf_label", + "internet_nl_mail_dnssec_mailto_exist": "detail_mail_dnssec_exists_label", + "internet_nl_mail_dnssec_mailto_valid": "detail_mail_dnssec_valid_label", + "internet_nl_mail_dnssec_mx_valid": "detail_mail_dnssec_mx_valid_label", + "internet_nl_mail_dnssec_mx_exist": "detail_mail_dnssec_mx_exists_label", + "internet_nl_mail_ipv6_mx_address": "detail_mail_ipv6_mx_aaaa_label", + "internet_nl_mail_ipv6_mx_reach": "detail_mail_ipv6_mx_reach_label", + "internet_nl_mail_ipv6_ns_reach": "detail_web_mail_ipv6_ns_reach_label", + "internet_nl_mail_ipv6_ns_address": "detail_web_mail_ipv6_ns_aaaa_label", + "internet_nl_mail_starttls_tls_cipherorder": "detail_mail_tls_cipher_order_label", + "internet_nl_mail_starttls_tls_keyexchangehash": "detail_mail_tls_kex_hash_func_label", + "internet_nl_mail_starttls_tls_0rtt": "detail_mail_tls_zero_rtt_label", + "internet_nl_mail_rpki_exists": "detail_mail_rpki_exists_label", + "internet_nl_mail_rpki_valid": "detail_mail_rpki_valid_label", + "internet_nl_mail_ns_rpki_exists": "detail_web_mail_rpki_ns_exists_label", + "internet_nl_mail_ns_rpki_valid": "detail_web_mail_rpki_ns_valid_label", + "internet_nl_mail_mx_ns_rpki_exists": "detail_mail_rpki_mx_ns_exists_label", + "internet_nl_mail_mx_ns_rpki_valid": "detail_mail_rpki_mx_ns_valid_label", # web fields, see dashboard.js - 'internet_nl_web_appsecpriv': 'results_domain_appsecpriv_http_headers_label', - 'internet_nl_web_appsecpriv_csp': 'detail_web_appsecpriv_http_csp_label', - 'internet_nl_web_appsecpriv_referrer_policy': 'detail_web_appsecpriv_http_referrer_policy_label', - 'internet_nl_web_appsecpriv_x_content_type_options': 'detail_web_appsecpriv_http_x_content_type_label', - 'internet_nl_web_appsecpriv_x_frame_options': 'detail_web_appsecpriv_http_x_frame_label', - 'internet_nl_web_appsecpriv_securitytxt': 'detail_web_appsecpriv_http_securitytxt_label', - 'internet_nl_web_https_cert_domain': 'detail_web_tls_cert_hostmatch_label', - 'internet_nl_web_https_http_redirect': 'detail_web_tls_https_forced_label', - 'internet_nl_web_https_cert_chain': 'detail_web_tls_cert_trust_label', - 'internet_nl_web_https_tls_version': 'detail_web_tls_version_label', - 'internet_nl_web_https_tls_clientreneg': 'detail_web_tls_renegotiation_client_label', - 'internet_nl_web_https_tls_ciphers': 'detail_web_tls_ciphers_label', - 'internet_nl_web_https_http_available': 'detail_web_tls_https_exists_label', - 'internet_nl_web_https_dane_exist': 'detail_web_tls_dane_exists_label', - 'internet_nl_web_https_http_compress': 'detail_web_tls_http_compression_label', - 'internet_nl_web_https_http_hsts': 'detail_web_tls_https_hsts_label', - 'internet_nl_web_https_tls_secreneg': 'detail_web_tls_renegotiation_secure_label', - 'internet_nl_web_https_dane_valid': 'detail_web_tls_dane_valid_label', - 'internet_nl_web_https_cert_pubkey': 'detail_web_tls_cert_pubkey_label', - 'internet_nl_web_https_cert_sig': 'detail_web_tls_cert_signature_label', - 'internet_nl_web_https_tls_compress': 'detail_web_tls_compression_label', - 'internet_nl_web_https_tls_keyexchange': 'detail_web_tls_fs_params_label', - 'internet_nl_web_dnssec_valid': 'detail_web_dnssec_valid_label', - 'internet_nl_web_dnssec_exist': 'detail_web_dnssec_exists_label', - 'internet_nl_web_ipv6_ws_similar': 'detail_web_ipv6_web_ipv46_label', - 'internet_nl_web_ipv6_ws_address': 'detail_web_ipv6_web_aaaa_label', - 'internet_nl_web_ipv6_ns_reach': 'detail_web_mail_ipv6_ns_reach_label', - 'internet_nl_web_ipv6_ws_reach': 'detail_web_ipv6_web_reach_label', - 'internet_nl_web_ipv6_ns_address': 'detail_web_mail_ipv6_ns_aaaa_label', - 'internet_nl_web_https_tls_cipherorder': 'detail_web_tls_cipher_order_label', - 'internet_nl_web_https_tls_0rtt': 'detail_web_tls_zero_rtt_label', - 'internet_nl_web_https_tls_ocsp': 'detail_web_tls_ocsp_stapling_label', - 'internet_nl_web_https_tls_keyexchangehash': 'detail_web_tls_kex_hash_func_label', - 'internet_nl_web_rpki_exists': 'detail_web_rpki_exists_label', - 'internet_nl_web_rpki_valid': 'detail_web_rpki_valid_label', - 'internet_nl_web_ns_rpki_exists': 'detail_web_mail_rpki_ns_exists_label', - 'internet_nl_web_ns_rpki_valid': 'detail_web_mail_rpki_ns_valid_label', - - 'internet_nl_web_rpki': 'test_siterpki_label', - 'internet_nl_web_tls': 'test_sitetls_label', - 'internet_nl_web_dnssec': 'test_sitednssec_label', - 'internet_nl_web_ipv6': 'test_siteipv6_label', - - 'internet_nl_mail_dashboard_tls': 'test_mailtls_label', - 'internet_nl_mail_dashboard_auth': 'test_mailauth_label', - 'internet_nl_mail_dashboard_dnssec': 'test_maildnssec_label', - 'internet_nl_mail_dashboard_ipv6': 'test_mailipv6_label', - 'internet_nl_mail_dashboard_rpki': 'test_mailrpki_label', - - "category_web_ipv6_name_server": 'results_domain_mail_ipv6_name_servers_label', - "category_web_ipv6_web_server": 'results_domain_ipv6_web_server_label', - "category_web_dnssec_dnssec": 'test_sitednssec_label', - "category_web_tls_http": 'results_domain_tls_https_label', - "category_web_tls_tls": 'results_domain_tls_tls_label', - "category_web_tls_certificate": 'results_domain_mail_tls_certificate_label', - "category_web_tls_dane": 'results_domain_mail_tls_dane_label', - "category_web_security_options_appsecpriv": 'results_domain_appsecpriv_http_headers_label', - - 'category_mail_ipv6_name_servers': 'results_domain_mail_ipv6_name_servers_label', - 'category_mail_ipv6_mail_servers': 'results_mail_ipv6_mail_servers_label', - 'category_mail_dnssec_email_address_domain': 'results_mail_dnssec_domain_label', - 'category_mail_dnssec_mail_server_domain': 'results_mail_dnssec_mail_servers_label', - 'category_mail_dashboard_auth_dmarc': 'results_mail_auth_dmarc_label', - 'category_mail_dashboard_aut_dkim': 'results_mail_auth_dkim_label', - 'category_mail_dashboard_aut_spf': 'results_mail_auth_spf_label', - 'category_mail_starttls_tls': 'results_mail_tls_starttls_label', - 'category_mail_starttls_certificate': 'results_domain_mail_tls_certificate_label', - 'category_mail_starttls_dane': 'results_domain_mail_tls_dane_label', - 'category_web_security_options_other': 'results_domain_appsecpriv_other_options_label', - 'category_web_rpki_name_server': 'results_domain_mail_rpki_name_servers_label', - 'category_web_rpki_web_server': 'results_domain_rpki_web_server_label', - 'category_mail_rpki_name_server': 'results_domain_mail_rpki_name_servers_label', - 'category_mail_rpki_name_mail_server': 'results_mail_rpki_mx_name_servers_label', - 'category_mail_rpki_mail_server': 'results_mail_rpki_mail_servers_label', - - 'internet_nl_score': '% Score', - 'internet_nl_score_report': 'Report', - + "internet_nl_web_appsecpriv": "results_domain_appsecpriv_http_headers_label", + "internet_nl_web_appsecpriv_csp": "detail_web_appsecpriv_http_csp_label", + "internet_nl_web_appsecpriv_referrer_policy": "detail_web_appsecpriv_http_referrer_policy_label", + "internet_nl_web_appsecpriv_x_content_type_options": "detail_web_appsecpriv_http_x_content_type_label", + "internet_nl_web_appsecpriv_x_frame_options": "detail_web_appsecpriv_http_x_frame_label", + "internet_nl_web_appsecpriv_securitytxt": "detail_web_appsecpriv_http_securitytxt_label", + "internet_nl_web_https_cert_domain": "detail_web_tls_cert_hostmatch_label", + "internet_nl_web_https_http_redirect": "detail_web_tls_https_forced_label", + "internet_nl_web_https_cert_chain": "detail_web_tls_cert_trust_label", + "internet_nl_web_https_tls_version": "detail_web_tls_version_label", + "internet_nl_web_https_tls_clientreneg": "detail_web_tls_renegotiation_client_label", + "internet_nl_web_https_tls_ciphers": "detail_web_tls_ciphers_label", + "internet_nl_web_https_http_available": "detail_web_tls_https_exists_label", + "internet_nl_web_https_dane_exist": "detail_web_tls_dane_exists_label", + "internet_nl_web_https_http_compress": "detail_web_tls_http_compression_label", + "internet_nl_web_https_http_hsts": "detail_web_tls_https_hsts_label", + "internet_nl_web_https_tls_secreneg": "detail_web_tls_renegotiation_secure_label", + "internet_nl_web_https_dane_valid": "detail_web_tls_dane_valid_label", + "internet_nl_web_https_cert_pubkey": "detail_web_tls_cert_pubkey_label", + "internet_nl_web_https_cert_sig": "detail_web_tls_cert_signature_label", + "internet_nl_web_https_tls_compress": "detail_web_tls_compression_label", + "internet_nl_web_https_tls_keyexchange": "detail_web_tls_fs_params_label", + "internet_nl_web_dnssec_valid": "detail_web_dnssec_valid_label", + "internet_nl_web_dnssec_exist": "detail_web_dnssec_exists_label", + "internet_nl_web_ipv6_ws_similar": "detail_web_ipv6_web_ipv46_label", + "internet_nl_web_ipv6_ws_address": "detail_web_ipv6_web_aaaa_label", + "internet_nl_web_ipv6_ns_reach": "detail_web_mail_ipv6_ns_reach_label", + "internet_nl_web_ipv6_ws_reach": "detail_web_ipv6_web_reach_label", + "internet_nl_web_ipv6_ns_address": "detail_web_mail_ipv6_ns_aaaa_label", + "internet_nl_web_https_tls_cipherorder": "detail_web_tls_cipher_order_label", + "internet_nl_web_https_tls_0rtt": "detail_web_tls_zero_rtt_label", + "internet_nl_web_https_tls_ocsp": "detail_web_tls_ocsp_stapling_label", + "internet_nl_web_https_tls_keyexchangehash": "detail_web_tls_kex_hash_func_label", + "internet_nl_web_rpki_exists": "detail_web_rpki_exists_label", + "internet_nl_web_rpki_valid": "detail_web_rpki_valid_label", + "internet_nl_web_ns_rpki_exists": "detail_web_mail_rpki_ns_exists_label", + "internet_nl_web_ns_rpki_valid": "detail_web_mail_rpki_ns_valid_label", + "internet_nl_web_rpki": "test_siterpki_label", + "internet_nl_web_tls": "test_sitetls_label", + "internet_nl_web_dnssec": "test_sitednssec_label", + "internet_nl_web_ipv6": "test_siteipv6_label", + "internet_nl_mail_dashboard_tls": "test_mailtls_label", + "internet_nl_mail_dashboard_auth": "test_mailauth_label", + "internet_nl_mail_dashboard_dnssec": "test_maildnssec_label", + "internet_nl_mail_dashboard_ipv6": "test_mailipv6_label", + "internet_nl_mail_dashboard_rpki": "test_mailrpki_label", + "category_web_ipv6_name_server": "results_domain_mail_ipv6_name_servers_label", + "category_web_ipv6_web_server": "results_domain_ipv6_web_server_label", + "category_web_dnssec_dnssec": "test_sitednssec_label", + "category_web_tls_http": "results_domain_tls_https_label", + "category_web_tls_tls": "results_domain_tls_tls_label", + "category_web_tls_certificate": "results_domain_mail_tls_certificate_label", + "category_web_tls_dane": "results_domain_mail_tls_dane_label", + "category_web_security_options_appsecpriv": "results_domain_appsecpriv_http_headers_label", + "category_mail_ipv6_name_servers": "results_domain_mail_ipv6_name_servers_label", + "category_mail_ipv6_mail_servers": "results_mail_ipv6_mail_servers_label", + "category_mail_dnssec_email_address_domain": "results_mail_dnssec_domain_label", + "category_mail_dnssec_mail_server_domain": "results_mail_dnssec_mail_servers_label", + "category_mail_dashboard_auth_dmarc": "results_mail_auth_dmarc_label", + "category_mail_dashboard_aut_dkim": "results_mail_auth_dkim_label", + "category_mail_dashboard_aut_spf": "results_mail_auth_spf_label", + "category_mail_starttls_tls": "results_mail_tls_starttls_label", + "category_mail_starttls_certificate": "results_domain_mail_tls_certificate_label", + "category_mail_starttls_dane": "results_domain_mail_tls_dane_label", + "category_web_security_options_other": "results_domain_appsecpriv_other_options_label", + "category_web_rpki_name_server": "results_domain_mail_rpki_name_servers_label", + "category_web_rpki_web_server": "results_domain_rpki_web_server_label", + "category_mail_rpki_name_server": "results_domain_mail_rpki_name_servers_label", + "category_mail_rpki_name_mail_server": "results_mail_rpki_mx_name_servers_label", + "category_mail_rpki_mail_server": "results_mail_rpki_mail_servers_label", + "internet_nl_score": "% Score", + "internet_nl_score_report": "Report", # directly translated fields. - 'internet_nl_mail_legacy_dmarc': 'DMARC', - 'internet_nl_mail_legacy_dkim': 'DKIM', - 'internet_nl_mail_legacy_spf': 'SPF', - 'internet_nl_mail_legacy_dmarc_policy': 'DMARC policy', - 'internet_nl_mail_legacy_spf_policy': 'SPF policy', - 'internet_nl_mail_legacy_start_tls': 'STARTTLS', - 'internet_nl_mail_legacy_start_tls_ncsc': 'STARTTLS NCSC', - 'internet_nl_mail_legacy_dnssec_email_domain': 'DNSSEC e-mail domain', - 'internet_nl_mail_legacy_dnssec_mx': 'DNSSEC MX', - 'internet_nl_mail_legacy_dane': 'DANE', - 'internet_nl_mail_legacy_category_ipv6': 'IPv6', - 'internet_nl_mail_legacy_ipv6_nameserver': 'IPv6 nameserver', - 'internet_nl_mail_legacy_ipv6_mailserver': 'IPv6 mailserver', - 'internet_nl_mail_legacy_category': 'Extra Fields', - - 'internet_nl_web_legacy_dnssec': 'DNSSEC', - 'internet_nl_web_legacy_tls_available': 'TLS available', - 'internet_nl_web_legacy_tls_ncsc_web': 'TLS NCSC web', - 'internet_nl_web_legacy_https_enforced': 'HTTPS redirect', - 'internet_nl_web_legacy_hsts': 'HSTS', - 'internet_nl_web_legacy_category_ipv6': 'IPv6', - 'internet_nl_web_legacy_ipv6_nameserver': 'IPv6 nameserver', - 'internet_nl_web_legacy_ipv6_webserver': 'IPv6 webserver', - 'internet_nl_web_legacy_category': 'Extra Fields', + "internet_nl_mail_legacy_dmarc": "DMARC", + "internet_nl_mail_legacy_dkim": "DKIM", + "internet_nl_mail_legacy_spf": "SPF", + "internet_nl_mail_legacy_dmarc_policy": "DMARC policy", + "internet_nl_mail_legacy_spf_policy": "SPF policy", + "internet_nl_mail_legacy_start_tls": "STARTTLS", + "internet_nl_mail_legacy_start_tls_ncsc": "STARTTLS NCSC", + "internet_nl_mail_legacy_dnssec_email_domain": "DNSSEC e-mail domain", + "internet_nl_mail_legacy_dnssec_mx": "DNSSEC MX", + "internet_nl_mail_legacy_dane": "DANE", + "internet_nl_mail_legacy_category_ipv6": "IPv6", + "internet_nl_mail_legacy_ipv6_nameserver": "IPv6 nameserver", + "internet_nl_mail_legacy_ipv6_mailserver": "IPv6 mailserver", + "internet_nl_mail_legacy_category": "Extra Fields", + "internet_nl_web_legacy_dnssec": "DNSSEC", + "internet_nl_web_legacy_tls_available": "TLS available", + "internet_nl_web_legacy_tls_ncsc_web": "TLS NCSC web", + "internet_nl_web_legacy_https_enforced": "HTTPS redirect", + "internet_nl_web_legacy_hsts": "HSTS", + "internet_nl_web_legacy_category_ipv6": "IPv6", + "internet_nl_web_legacy_ipv6_nameserver": "IPv6 nameserver", + "internet_nl_web_legacy_ipv6_webserver": "IPv6 webserver", + "internet_nl_web_legacy_category": "Extra Fields", # Deleted on request # 'internet_nl_web_legacy_dane': 'DANE', - - 'internet_nl_web_legacy_tls_1_3': 'TLS 1.3 Support', - 'internet_nl_mail_legacy_mail_sending_domain': 'E-mail sending domain', - 'internet_nl_mail_legacy_mail_server_testable': 'Mail server testable', - 'internet_nl_mail_legacy_mail_server_reachable': 'Mail server reachable', - 'internet_nl_mail_legacy_domain_has_mx': 'Mail server has MX record', - 'internet_nl_mail_legacy_tls_1_3': 'TLS 1.3 Support', - - 'legacy': 'Extra Fields', - 'internet_nl_mail_dashboard_overall_score': 'Score', - 'internet_nl_web_overall_score': 'Score', - - 'overall': "Score", + "internet_nl_web_legacy_tls_1_3": "TLS 1.3 Support", + "internet_nl_mail_legacy_mail_sending_domain": "E-mail sending domain", + "internet_nl_mail_legacy_mail_server_testable": "Mail server testable", + "internet_nl_mail_legacy_mail_server_reachable": "Mail server reachable", + "internet_nl_mail_legacy_domain_has_mx": "Mail server has MX record", + "internet_nl_mail_legacy_tls_1_3": "TLS 1.3 Support", + "legacy": "Extra Fields", + "internet_nl_mail_dashboard_overall_score": "Score", + "internet_nl_web_overall_score": "Score", + "overall": "Score", "ipv6": "Modern address (IPv6)", "dnssec": "DNSSEC", "tls": "Secure connection (HTTPS)", diff --git a/dashboard/internet_nl_dashboard/logic/mail.py b/dashboard/internet_nl_dashboard/logic/mail.py index c01cddce..281fec3c 100644 --- a/dashboard/internet_nl_dashboard/logic/mail.py +++ b/dashboard/internet_nl_dashboard/logic/mail.py @@ -15,8 +15,12 @@ from dashboard.celery import app from dashboard.internet_nl_dashboard.logic.mail_admin_templates import xget_template from dashboard.internet_nl_dashboard.logic.report import get_report_directly -from dashboard.internet_nl_dashboard.logic.report_comparison import (compare_report_in_detail, filter_comparison_report, - key_calculation, render_comparison_view) +from dashboard.internet_nl_dashboard.logic.report_comparison import ( + compare_report_in_detail, + filter_comparison_report, + key_calculation, + render_comparison_view, +) from dashboard.internet_nl_dashboard.models import AccountInternetNLScan, DashboardUser, UrlListReport from dashboard.settings import LANGUAGES @@ -64,13 +68,15 @@ def email_configration_is_correct(): def get_users_to_send_mail_to(scan: AccountInternetNLScan): # Only send mail to users that are active and of course have a mail address... - return User.objects.all().filter( - dashboarduser__account=scan.account, - dashboarduser__mail_preferred_mail_address__isnull=False, - dashboarduser__mail_send_mail_after_scan_finished=True, - is_active=True - ).exclude( - dashboarduser__mail_preferred_mail_address="" + return ( + User.objects.all() + .filter( + dashboarduser__account=scan.account, + dashboarduser__mail_preferred_mail_address__isnull=False, + dashboarduser__mail_send_mail_after_scan_finished=True, + is_active=True, + ) + .exclude(dashboarduser__mail_preferred_mail_address="") ) @@ -84,13 +90,13 @@ def send_scan_finished_mails(scan: AccountInternetNLScan) -> int: :return: """ if not scan.report: - log.error(f'Tried to send a finished mail for a report that was not finished. Scan: {scan}') + log.error(f"Tried to send a finished mail for a report that was not finished. Scan: {scan}") return 0 users = get_users_to_send_mail_to(scan) # remove calculation because textfields are slow while filtering. Have to retrieve the textfield later - report = UrlListReport.objects.all().filter(id=scan.report.id).order_by("-id").defer('calculation').first() + report = UrlListReport.objects.all().filter(id=scan.report.id).order_by("-id").defer("calculation").first() for user in users: log.debug("Sending finished mail to user %s", user.id) @@ -121,12 +127,13 @@ def send_scan_finished_mails(scan: AccountInternetNLScan) -> int: # could be None, strictly speaking if scan.scan: - placeholders['scan_type'] = scan.scan.type \ - if scan.scan.type == "web" else "all" if scan.scan.type == "all" else "mail" + placeholders["scan_type"] = ( + scan.scan.type if scan.scan.type == "web" else "all" if scan.scan.type == "all" else "mail" + ) if scan.started_on: - placeholders['scan_started_on'] = scan.started_on.isoformat() - placeholders['scan_duration'] = (datetime.now(timezone.utc) - scan.started_on).seconds + placeholders["scan_started_on"] = scan.started_on.isoformat() + placeholders["scan_duration"] = (datetime.now(timezone.utc) - scan.started_on).seconds previous = values_from_previous_report( report.id, @@ -141,9 +148,9 @@ def send_scan_finished_mails(scan: AccountInternetNLScan) -> int: recipients=user.dashboarduser.mail_preferred_mail_address, # List of email addresses also accepted template=xget_template( template_name="scan_finished", - preferred_language=user.dashboarduser.mail_preferred_language.code.lower() + preferred_language=user.dashboarduser.mail_preferred_language.code.lower(), ), - variable_dict=placeholders + variable_dict=placeholders, ) # number of mails sent is equal to users configured their account to receive mails. @@ -180,38 +187,33 @@ def values_from_previous_report(report_id: int, previous_report: UrlListReport) if "urls" not in first_report_data["calculation"] or "urls" not in second_report_data["calculation"]: return empty_response - comp = compare_report_in_detail( - key_calculation(first_report_data), - key_calculation(second_report_data) - ) + comp = compare_report_in_detail(key_calculation(first_report_data), key_calculation(second_report_data)) difference = datetime.now(timezone.utc) - previous_report.at_when days_between_current_and_previous_report = difference.days - summary = comp['summary'] - comparison_is_empty = summary['neutral'] + summary['improvement'] + summary['regression'] < 1 + summary = comp["summary"] + comparison_is_empty = summary["neutral"] + summary["improvement"] + summary["regression"] < 1 return { # comparison reports: # The template system only knows strings, so the boolean is coded as string here "previous_report_available": True, - "previous_report_average_internet_nl_score": comp['old']['average_internet_nl_score'], - "current_report_average_internet_nl_score": comp['new']['average_internet_nl_score'], + "previous_report_average_internet_nl_score": comp["old"]["average_internet_nl_score"], + "current_report_average_internet_nl_score": comp["new"]["average_internet_nl_score"], "compared_report_id": previous_report.id, - "comparison_is_empty": comparison_is_empty, - "improvement": summary['improvement'], - "regression": summary['regression'], - "neutral": summary['neutral'], + "improvement": summary["improvement"], + "regression": summary["regression"], + "neutral": summary["neutral"], "comparison_report_available": True, - "comparison_report_contains_improvement": summary['improvement'] > 0, - "comparison_report_contains_regression": summary['regression'] > 0, - + "comparison_report_contains_improvement": summary["improvement"] > 0, + "comparison_report_contains_regression": summary["regression"] > 0, "days_between_current_and_previous_report": days_between_current_and_previous_report, "comparison_table_improvement": filter_comparison_report(deepcopy(comp), "improvement"), "comparison_table_regression": filter_comparison_report(deepcopy(comp), "regression"), - "domains_exclusive_in_current_report": sorted(comp['urls_exclusive_in_new_report']), - "domains_exclusive_in_other_report": sorted(comp['urls_exclusive_in_old_report']), + "domains_exclusive_in_current_report": sorted(comp["urls_exclusive_in_new_report"]), + "domains_exclusive_in_other_report": sorted(comp["urls_exclusive_in_old_report"]), } @@ -222,37 +224,36 @@ def convert_to_email_safe_values(values: dict, mail_language: str = "en") -> dic # so we're nice here and try to use a code that we know in case this happens. # see issue INTERNET-NL-DASHBOARD-68 if mail_language not in [language_code for language_code, name in LANGUAGES]: - mail_language = 'en' + mail_language = "en" log.debug("Mail language: %s", mail_language) return { "previous_report_available": str(values["previous_report_available"]), "previous_report_average_internet_nl_score": values["previous_report_average_internet_nl_score"], "compared_report_id": values["compared_report_id"], - "comparison_is_empty": str(values["comparison_is_empty"]), "improvement": values["improvement"], "regression": values["regression"], "neutral": values["neutral"], - "comparison_report_available": str(values["comparison_report_available"]), "comparison_report_contains_improvement": str(values["comparison_report_contains_improvement"]), "comparison_report_contains_regression": str(values["comparison_report_contains_regression"]), - "days_between_current_and_previous_report": values["days_between_current_and_previous_report"], - "comparison_table_improvement": render_comparison_view(values["comparison_table_improvement"], - impact="improvement", language=mail_language), - "comparison_table_regression": render_comparison_view(values["comparison_table_regression"], - impact="regression", language=mail_language), - "domains_exclusive_in_current_report": ",".join(values['domains_exclusive_in_current_report']), - "domains_exclusive_in_other_report": ",".join(values['domains_exclusive_in_other_report']), + "comparison_table_improvement": render_comparison_view( + values["comparison_table_improvement"], impact="improvement", language=mail_language + ), + "comparison_table_regression": render_comparison_view( + values["comparison_table_regression"], impact="regression", language=mail_language + ), + "domains_exclusive_in_current_report": ",".join(values["domains_exclusive_in_current_report"]), + "domains_exclusive_in_other_report": ",".join(values["domains_exclusive_in_other_report"]), } def generate_unsubscribe_code() -> str: # https://pynative.com/python-generate-random-string/ # secure random is not needed, would be ridiculous. A sleep(1) is enough to deter any attack - return ''.join(choice(string.ascii_letters + string.digits) for i in range(128)) # nosec + return "".join(choice(string.ascii_letters + string.digits) for i in range(128)) # nosec def unsubscribe(feed: str = "scan_finished", unsubscribe_code: str = ""): @@ -262,8 +263,7 @@ def unsubscribe(feed: str = "scan_finished", unsubscribe_code: str = ""): if feed == "scan_finished": users = DashboardUser.objects.all().filter( - mail_after_mail_unsubscribe_code=unsubscribe_code, - mail_send_mail_after_scan_finished=True + mail_after_mail_unsubscribe_code=unsubscribe_code, mail_send_mail_after_scan_finished=True ) for user in users: user.mail_send_mail_after_scan_finished = False @@ -272,10 +272,10 @@ def unsubscribe(feed: str = "scan_finished", unsubscribe_code: str = ""): user.save() # always say that the user has been unsubscribed, even if there was no subscription - return {'unsubscribed': True} + return {"unsubscribed": True} -@app.task(queue='storage', ignore_result=True) +@app.task(queue="storage", ignore_result=True) def send_queued_mail(): """ To use this, add a periodic task. The signature is: @@ -284,4 +284,4 @@ def send_queued_mail(): Is added in the list of periodic tasks in the fixtures. :return: """ - call_command('send_queued_mail', processes=1, log_level=2) + call_command("send_queued_mail", processes=1, log_level=2) diff --git a/dashboard/internet_nl_dashboard/logic/mail_admin_templates.py b/dashboard/internet_nl_dashboard/logic/mail_admin_templates.py index eef88f1a..8de1a8f9 100644 --- a/dashboard/internet_nl_dashboard/logic/mail_admin_templates.py +++ b/dashboard/internet_nl_dashboard/logic/mail_admin_templates.py @@ -4,7 +4,7 @@ from django_mail_admin.models import EmailTemplate -def xget_template(template_name: str = "scan_finished", preferred_language: str = 'en') -> EmailTemplate: +def xget_template(template_name: str = "scan_finished", preferred_language: str = "en") -> EmailTemplate: """ :param template_name: @@ -15,19 +15,21 @@ def xget_template(template_name: str = "scan_finished", preferred_language: str # tries to retrieve the preferred language for emails. If that template is not available, # the fallback language (EN) is used. - template = EmailTemplate.objects.filter(name=f'{template_name}_{preferred_language}').first() + template = EmailTemplate.objects.filter(name=f"{template_name}_{preferred_language}").first() if template: return template - template = EmailTemplate.objects.filter(name=f'{template_name}_{config.EMAIL_FALLBACK_LANGUAGE}').first() + template = EmailTemplate.objects.filter(name=f"{template_name}_{config.EMAIL_FALLBACK_LANGUAGE}").first() if template: return template - raise LookupError(f"Could not find e-mail template {template_name}, neither for language {preferred_language} nor" - f"the fallback language {config.EMAIL_FALLBACK_LANGUAGE}.") + raise LookupError( + f"Could not find e-mail template {template_name}, neither for language {preferred_language} nor" + f"the fallback language {config.EMAIL_FALLBACK_LANGUAGE}." + ) -def xget_template_as_string(template_name: str = "scan_finished", preferred_language: str = 'en') -> str: +def xget_template_as_string(template_name: str = "scan_finished", preferred_language: str = "en") -> str: template = xget_template(template_name, preferred_language) # django_mail_admin does not use types, so everything is any return template.email_html_text # type: ignore diff --git a/dashboard/internet_nl_dashboard/logic/report.py b/dashboard/internet_nl_dashboard/logic/report.py index 0c023c46..dc0d199d 100644 --- a/dashboard/internet_nl_dashboard/logic/report.py +++ b/dashboard/internet_nl_dashboard/logic/report.py @@ -17,8 +17,10 @@ from dashboard.celery import app from dashboard.internet_nl_dashboard.logic import operation_response -from dashboard.internet_nl_dashboard.logic.urllist_dashboard_report import (create_calculation_on_urls, - sum_internet_nl_scores_over_rating) +from dashboard.internet_nl_dashboard.logic.urllist_dashboard_report import ( + create_calculation_on_urls, + sum_internet_nl_scores_over_rating, +) from dashboard.internet_nl_dashboard.models import Account, UrlList, UrlListReport log = logging.getLogger(__name__) @@ -27,25 +29,32 @@ def get_recent_reports(account: Account) -> List[Dict[str, Any]]: # loading the calculation takes some time. In this case we don't need the calculation and as such we defer it. # also show the reports from deleted lists... urllist__is_deleted=False - reports = UrlListReport.objects.all().filter( - urllist__account=account).order_by('-pk').select_related( - 'urllist').defer('calculation') + reports = ( + UrlListReport.objects.all() + .filter(urllist__account=account) + .order_by("-pk") + .select_related("urllist") + .defer("calculation") + ) return create_report_response(reports) def create_report_response(reports) -> List[Dict[str, Any]]: - return [{ - 'id': report.id, - 'report': report.id, - # mask that there is a mail_dashboard variant. - 'type': report.report_type, - 'number_of_urls': report.total_urls, - 'list_name': report.urllist.name, - 'at_when': report.at_when.isoformat(), - 'urllist_id': report.urllist.id, - 'urllist_scan_type': report.urllist.scan_type, - } for report in reports] + return [ + { + "id": report.id, + "report": report.id, + # mask that there is a mail_dashboard variant. + "type": report.report_type, + "number_of_urls": report.total_urls, + "list_name": report.urllist.name, + "at_when": report.at_when.isoformat(), + "urllist_id": report.urllist.id, + "urllist_scan_type": report.urllist.scan_type, + } + for report in reports + ] def ad_hoc_report_create(account: Account, report_id: int, tags: List[str], at_when: Optional[datetime]): @@ -58,8 +67,9 @@ def ad_hoc_report_create(account: Account, report_id: int, tags: List[str], at_w # account is used to connect a session to the current report: so the correct user opens the report # tags is a list of tags that must all be present in the list of urls - report = UrlListReport.objects.all().filter( - urllist__account=account, urllist__is_deleted=False, id=report_id).first() + report = ( + UrlListReport.objects.all().filter(urllist__account=account, urllist__is_deleted=False, id=report_id).first() + ) if not report: return {} @@ -105,19 +115,21 @@ def save_ad_hoc_tagged_report(account: Account, report_id: int, tags: List[str], def ad_hoc_tagged_report(account: Account, report_id: int, tags: List[str], at_when: Optional[datetime]): report = ad_hoc_report_create(account, report_id, tags, at_when) - return '{' \ - f'"id": {report.id}, ' \ - f'"urllist_id": {report.urllist.id}, ' \ - f'"urllist_name": "{report.urllist.name}", ' \ - f'"average_internet_nl_score": {report.average_internet_nl_score}, ' \ - f'"total_urls": {len(report.calculation["urls"])}, ' \ - f'"is_publicly_shared": {"true" if report.is_publicly_shared else "false"}, ' \ - f'"at_when": "{report.at_when}", ' \ - f'"calculation": {json.dumps(report.calculation)}, ' \ - f'"report_type": "{report.report_type}", ' \ - f'"public_report_code": "{report.public_report_code}", ' \ - f'"public_share_code": "{report.public_share_code}" ' \ - '}' + return ( + "{" + f'"id": {report.id}, ' + f'"urllist_id": {report.urllist.id}, ' + f'"urllist_name": "{report.urllist.name}", ' + f'"average_internet_nl_score": {report.average_internet_nl_score}, ' + f'"total_urls": {len(report.calculation["urls"])}, ' + f'"is_publicly_shared": {"true" if report.is_publicly_shared else "false"}, ' + f'"at_when": "{report.at_when}", ' + f'"calculation": {json.dumps(report.calculation)}, ' + f'"report_type": "{report.report_type}", ' + f'"public_report_code": "{report.public_report_code}", ' + f'"public_share_code": "{report.public_share_code}" ' + "}" + ) def get_urllist_timeline_graph(account: Account, urllist_ids: str, report_type: str = "web"): @@ -155,17 +167,20 @@ def get_urllist_timeline_graph(account: Account, urllist_ids: str, report_type: casted_list_split = list({int(list_id) for list_id in list_split}) statistics_over_last_years_reports = Prefetch( - 'urllistreport_set', - queryset=UrlListReport.objects.filter(report_type=report_type).order_by('at_when').only( - 'at_when', 'average_internet_nl_score', 'total_urls', 'urllist_id'), - to_attr='reports_from_the_last_year') + "urllistreport_set", + queryset=UrlListReport.objects.filter(report_type=report_type) + .order_by("at_when") + .only("at_when", "average_internet_nl_score", "total_urls", "urllist_id"), + to_attr="reports_from_the_last_year", + ) # The actual query, note that the ordering is asc on ID, whatever order you specify... - urllists = UrlList.objects.all().filter( - account=account, - pk__in=casted_list_split, - is_deleted=False - ).only('id', 'name').prefetch_related(statistics_over_last_years_reports) + urllists = ( + UrlList.objects.all() + .filter(account=account, pk__in=casted_list_split, is_deleted=False) + .only("id", "name") + .prefetch_related(statistics_over_last_years_reports) + ) if not urllists: return [] @@ -177,13 +192,16 @@ def get_urllist_timeline_graph(account: Account, urllist_ids: str, report_type: stats[urllist.id] = { "id": urllist.id, "name": urllist.name, - "data": [{ - 'date': per_report_statistics.at_when.date().isoformat(), - 'urls': per_report_statistics.total_urls, - 'average_internet_nl_score': per_report_statistics.average_internet_nl_score, - 'report': per_report_statistics.id - # mypy does not understand to_attr - } for per_report_statistics in urllist.reports_from_the_last_year] # type: ignore + "data": [ + { + "date": per_report_statistics.at_when.date().isoformat(), + "urls": per_report_statistics.total_urls, + "average_internet_nl_score": per_report_statistics.average_internet_nl_score, + "report": per_report_statistics.id, + # mypy does not understand to_attr + } + for per_report_statistics in urllist.reports_from_the_last_year + ], # type: ignore } # echo the results in the order you got them: @@ -202,12 +220,24 @@ def get_report(account: Account, report_id: int) -> str: log.debug("Retrieve report data") # it's okay if the list is deleted. Still be able to see reports from the past # urllist__is_deleted=False, - report = UrlListReport.objects.all().filter( - urllist__account=account, - pk=report_id - ).values('id', 'urllist_id', 'calculation', 'average_internet_nl_score', 'total_urls', 'at_when', 'report_type', - 'urllist__name', 'is_publicly_shared', 'public_report_code', 'public_share_code' - ).first() + report = ( + UrlListReport.objects.all() + .filter(urllist__account=account, pk=report_id) + .values( + "id", + "urllist_id", + "calculation", + "average_internet_nl_score", + "total_urls", + "at_when", + "report_type", + "urllist__name", + "is_publicly_shared", + "public_report_code", + "public_share_code", + ) + .first() + ) if not report: return "{}" @@ -229,34 +259,52 @@ def log_report_access_async(report_id: int, account_id: int): # Sprinkling an activity stream action. log.debug("Saving activity stream action") - log_report = UrlListReport.objects.all().filter( - urllist__account=account, - urllist__is_deleted=False, - pk=report_id - ).only('id').first() - action.send(account, verb='viewed report', target=log_report, public=False) + log_report = ( + UrlListReport.objects.all() + .filter(urllist__account=account, urllist__is_deleted=False, pk=report_id) + .only("id") + .first() + ) + action.send(account, verb="viewed report", target=log_report, public=False) def get_public_reports(): - return list(UrlListReport.objects.all().filter( - is_publicly_shared=True, is_shared_on_homepage=True - ).values( - 'at_when', 'average_internet_nl_score', 'report_type', 'urllist__name', 'total_urls', 'public_report_code' - ).order_by('-at_when')) + return list( + UrlListReport.objects.all() + .filter(is_publicly_shared=True, is_shared_on_homepage=True) + .values( + "at_when", "average_internet_nl_score", "report_type", "urllist__name", "total_urls", "public_report_code" + ) + .order_by("-at_when") + ) def get_shared_report(report_code: str, share_code: str = ""): # Check if report_code exists. If so see if a share code is required. - report = UrlListReport.objects.all().filter( - # A deleted list means that the report cannot be seen anymore - urllist__is_deleted=False, - - # All other public sharing filters - public_report_code=report_code, - is_publicly_shared=True - ).values('id', 'urllist_id', 'calculation', 'average_internet_nl_score', 'total_urls', 'at_when', 'report_type', - 'urllist__name', 'is_publicly_shared', 'public_report_code', 'public_share_code' - ).first() + report = ( + UrlListReport.objects.all() + .filter( + # A deleted list means that the report cannot be seen anymore + urllist__is_deleted=False, + # All other public sharing filters + public_report_code=report_code, + is_publicly_shared=True, + ) + .values( + "id", + "urllist_id", + "calculation", + "average_internet_nl_score", + "total_urls", + "at_when", + "report_type", + "urllist__name", + "is_publicly_shared", + "public_report_code", + "public_share_code", + ) + .first() + ) if not report: # deter brute forcing @@ -264,15 +312,17 @@ def get_shared_report(report_code: str, share_code: str = ""): sleep(3) return [] - if report['public_share_code'] == share_code: + if report["public_share_code"] == share_code: # todo: prevent loads/dumps with report calculation, it is should be sent without any loading to speed up # large reports calculation = retrieve_report_raw(report["id"], "UrlListReport") return f"{dump_report_to_text_resembling_json(report, calculation)}" # todo: should be a normal REST response - return f'{{"authentication_required": true, "public_report_code": "{report_code}", "id": "{report["id"]}", ' \ - f'"urllist_name": "{report["urllist__name"]}", "at_when": "{report["at_when"]}"}}' + return ( + f'{{"authentication_required": true, "public_report_code": "{report_code}", "id": "{report["id"]}", ' + f'"urllist_name": "{report["urllist__name"]}", "at_when": "{report["at_when"]}"}}' + ) def dump_report_to_text_resembling_json(report, calculation): @@ -285,19 +335,21 @@ def dump_report_to_text_resembling_json(report, calculation): :param report: :return: """ - return '{' \ - f'"id": {report["id"]}, ' \ - f'"urllist_id": {report["urllist_id"]}, ' \ - f'"urllist_name": "{report["urllist__name"]}", ' \ - f'"average_internet_nl_score": {report["average_internet_nl_score"]}, ' \ - f'"total_urls": {report["total_urls"]}, ' \ - f'"is_publicly_shared": {"true" if report["is_publicly_shared"] else "false"}, ' \ - f'"at_when": "{report["at_when"]}", ' \ - f'"calculation": {calculation}, ' \ - f'"report_type": "{report["report_type"]}", ' \ - f'"public_report_code": "{report["public_report_code"]}", ' \ - f'"public_share_code": "{report["public_share_code"]}" ' \ - '}' + return ( + "{" + f'"id": {report["id"]}, ' + f'"urllist_id": {report["urllist_id"]}, ' + f'"urllist_name": "{report["urllist__name"]}", ' + f'"average_internet_nl_score": {report["average_internet_nl_score"]}, ' + f'"total_urls": {report["total_urls"]}, ' + f'"is_publicly_shared": {"true" if report["is_publicly_shared"] else "false"}, ' + f'"at_when": "{report["at_when"]}", ' + f'"calculation": {calculation}, ' + f'"report_type": "{report["report_type"]}", ' + f'"public_report_code": "{report["public_report_code"]}", ' + f'"public_share_code": "{report["public_share_code"]}" ' + "}" + ) def get_report_directly(report_id): @@ -308,9 +360,12 @@ def get_report_directly(report_id): :return: """ - report = UrlListReport.objects.all().filter( - pk=report_id - ).values('id', 'urllist_id', 'average_internet_nl_score', 'total_urls', 'at_when').first() + report = ( + UrlListReport.objects.all() + .filter(pk=report_id) + .values("id", "urllist_id", "average_internet_nl_score", "total_urls", "at_when") + .first() + ) if not report: return {} @@ -329,12 +384,12 @@ def dump_report_to_json(report): :return: """ return { - 'id': report["id"], - 'urllist_id': report["urllist_id"], - 'average_internet_nl_score': report["average_internet_nl_score"], - 'total_urls': report["total_urls"], - 'at_when': report["at_when"], - 'calculation': report["calculation"] + "id": report["id"], + "urllist_id": report["urllist_id"], + "average_internet_nl_score": report["average_internet_nl_score"], + "total_urls": report["total_urls"], + "at_when": report["at_when"], + "calculation": report["calculation"], } @@ -350,11 +405,12 @@ def get_report_differences_compared_to_current_list(account: Account, report_id: :param report_id: :return: """ - report = UrlListReport.objects.all().filter( - urllist__account=account, - urllist__is_deleted=False, - pk=report_id - ).values('urllist_id').first() + report = ( + UrlListReport.objects.all() + .filter(urllist__account=account, urllist__is_deleted=False, pk=report_id) + .values("urllist_id") + .first() + ) if not report: return {} @@ -362,9 +418,9 @@ def get_report_differences_compared_to_current_list(account: Account, report_id: # since django 3.0 it's already retrieved as json calculation = retrieve_report(report_id, "UrlListReport") - urls_in_report: List[str] = [url['url'] for url in calculation['urls']] + urls_in_report: List[str] = [url["url"] for url in calculation["urls"]] - urllist = UrlList.objects.all().filter(id=report['urllist_id']).first() + urllist = UrlList.objects.all().filter(id=report["urllist_id"]).first() # todo: "ManyToManyField[Sequence[Any], RelatedManager[Any]]" of "Union[ManyToManyField[Sequence[Any], # RelatedManager[Any]], Any]" has no attribute "all" urls_in_list_queryset = urllist.urls.all() # type: ignore @@ -387,15 +443,17 @@ def get_report_differences_compared_to_current_list(account: Account, report_id: def get_previous_report(account: Account, urllist_id, at_when): - report = UrlListReport.objects.all().filter( - urllist_id=urllist_id, - urllist__account=account, - urllist__is_deleted=False, - at_when__lt=at_when).order_by('-at_when').values('pk').first() + report = ( + UrlListReport.objects.all() + .filter(urllist_id=urllist_id, urllist__account=account, urllist__is_deleted=False, at_when__lt=at_when) + .order_by("-at_when") + .values("pk") + .first() + ) if not report: return {} - return get_report(account, report['pk'])[0] + return get_report(account, report["pk"])[0] def optimize_calculation_and_add_statistics(calculation: Dict[str, Any]): @@ -424,7 +482,7 @@ def optimize_calculation_and_add_statistics(calculation: Dict[str, Any]): def remove_comply_or_explain(calculation: Dict[str, Any]): # Also remove all comply or explain information as it costs a lot of data/memory on the client - for url in calculation['urls']: + for url in calculation["urls"]: if "explained_total_issues" not in url: # explanations have already been removed. @@ -446,17 +504,17 @@ def remove_comply_or_explain(calculation: Dict[str, Any]): del url["explained_endpoint_issues_medium"] del url["explained_endpoint_issues_low"] - for endpoint in url['endpoints']: - del endpoint['explained_high'] - del endpoint['explained_medium'] - del endpoint['explained_low'] + for endpoint in url["endpoints"]: + del endpoint["explained_high"] + del endpoint["explained_medium"] + del endpoint["explained_low"] - for rating in endpoint['ratings']: - del rating['is_explained'] - del rating['comply_or_explain_explanation'] - del rating['comply_or_explain_explained_on'] - del rating['comply_or_explain_explanation_valid_until'] - del rating['comply_or_explain_valid_at_time_of_report'] + for rating in endpoint["ratings"]: + del rating["is_explained"] + del rating["comply_or_explain_explanation"] + del rating["comply_or_explain_explained_on"] + del rating["comply_or_explain_explanation_valid_until"] + del rating["comply_or_explain_valid_at_time_of_report"] if "explained_high" not in calculation: return calculation @@ -492,11 +550,11 @@ def add_keyed_ratings(calculation: Dict[str, Any]): :return: """ - for url in calculation['urls']: - for endpoint in url['endpoints']: - endpoint['ratings_by_type'] = {} - for rating in endpoint['ratings']: - endpoint['ratings_by_type'][rating['type']] = rating + for url in calculation["urls"]: + for endpoint in url["endpoints"]: + endpoint["ratings_by_type"] = {} + for rating in endpoint["ratings"]: + endpoint["ratings_by_type"][rating["type"]] = rating def clean_up_not_required_data_to_speed_up_report_on_client(calculation: Dict[str, Any]): @@ -508,28 +566,28 @@ def clean_up_not_required_data_to_speed_up_report_on_client(calculation: Dict[st :return: """ - for url in calculation['urls']: - for endpoint in url['endpoints']: - for rating_key in endpoint['ratings_by_type']: + for url in calculation["urls"]: + for endpoint in url["endpoints"]: + for rating_key in endpoint["ratings_by_type"]: # clean up fields we don't need, to make the report show even quicker # a lot of stuff from web sec map is nice, but not really useful for us at this moment. # perhaps later # These values are used in add_statistics_over_ratings and. Only OK is used in the spreadsheet # export (which could also be pre-generated). - del endpoint['ratings_by_type'][rating_key]['high'] # high is used in add_statistics_over_ratings - del endpoint['ratings_by_type'][rating_key]['medium'] # only 'ok' is used in spreadsheet export. - del endpoint['ratings_by_type'][rating_key]['low'] # only 'ok' is used in spreadsheet export. - del endpoint['ratings_by_type'][rating_key]['not_testable'] # only 'ok' is used in spreadsheet export. - del endpoint['ratings_by_type'][rating_key]['not_applicable'] # only 'ok' is used in spreadsheet export - del endpoint['ratings_by_type'][rating_key]['error_in_test'] # only 'ok' is used in spreadsheet export - del endpoint['ratings_by_type'][rating_key]['last_scan'] # not used in front end + del endpoint["ratings_by_type"][rating_key]["high"] # high is used in add_statistics_over_ratings + del endpoint["ratings_by_type"][rating_key]["medium"] # only 'ok' is used in spreadsheet export. + del endpoint["ratings_by_type"][rating_key]["low"] # only 'ok' is used in spreadsheet export. + del endpoint["ratings_by_type"][rating_key]["not_testable"] # only 'ok' is used in spreadsheet export. + del endpoint["ratings_by_type"][rating_key]["not_applicable"] # only 'ok' is used in spreadsheet export + del endpoint["ratings_by_type"][rating_key]["error_in_test"] # only 'ok' is used in spreadsheet export + del endpoint["ratings_by_type"][rating_key]["last_scan"] # not used in front end # the since field can be unix timestamp, which is less data try: - endpoint['ratings_by_type'][rating_key]['since' - ] = datetime.fromisoformat( - endpoint['ratings_by_type'][rating_key]['since']).timestamp() + endpoint["ratings_by_type"][rating_key]["since"] = datetime.fromisoformat( + endpoint["ratings_by_type"][rating_key]["since"] + ).timestamp() except ValueError: # then don't update it. ... @@ -538,31 +596,31 @@ def clean_up_not_required_data_to_speed_up_report_on_client(calculation: Dict[st # as a report can contain metrics from various moments in time (due to re-scans on measurement errors) # del endpoint['ratings_by_type'][rating_key]['since'] # del endpoint['ratings_by_type'][rating_key]['last_scan'] - del endpoint['ratings_by_type'][rating_key]['explanation'] + del endpoint["ratings_by_type"][rating_key]["explanation"] - del endpoint['ratings_by_type'][rating_key]['type'] # is already in the key - del endpoint['ratings_by_type'][rating_key]['scan_type'] # is already in the key + del endpoint["ratings_by_type"][rating_key]["type"] # is already in the key + del endpoint["ratings_by_type"][rating_key]["scan_type"] # is already in the key # remove the original rating, as that slows parsing on the client down significantly. # with significantly == Vue will parse it, and for a 500 url list this will take 5 seconds. - del endpoint['ratings'] + del endpoint["ratings"] - del url['total_endpoints'] - del url['high_endpoints'] - del url['medium_endpoints'] - del url['low_endpoints'] - del url['ok_endpoints'] + del url["total_endpoints"] + del url["high_endpoints"] + del url["medium_endpoints"] + del url["low_endpoints"] + del url["ok_endpoints"] - del url['total_url_issues'] - del url['url_issues_high'] - del url['url_issues_medium'] - del url['url_issues_low'] - del url['url_ok'] + del url["total_url_issues"] + del url["url_issues_high"] + del url["url_issues_medium"] + del url["url_issues_low"] + del url["url_ok"] - del url['total_endpoint_issues'] - del url['endpoint_issues_high'] - del url['endpoint_issues_medium'] - del url['endpoint_issues_low'] + del url["total_endpoint_issues"] + del url["endpoint_issues_high"] + del url["endpoint_issues_medium"] + del url["endpoint_issues_low"] def add_simple_verdicts(calculation: Dict[str, Any]): @@ -582,23 +640,22 @@ def add_simple_verdicts(calculation: Dict[str, Any]): # <50 will not be compared progression_table = { - 'not_applicable': 0, - 'not_testable': 0, - 'error_in_test': 0, - 'no_mx': 0, - 'unreachable': 0, - - 'failed': 100, - 'warning': 200, - 'info': 300, - 'good_not_tested': 380, - 'passed': 400, + "not_applicable": 0, + "not_testable": 0, + "error_in_test": 0, + "no_mx": 0, + "unreachable": 0, + "failed": 100, + "warning": 200, + "info": 300, + "good_not_tested": 380, + "passed": 400, } - for url in calculation['urls']: - for endpoint in url['endpoints']: - for rating in endpoint['ratings']: - rating['simple_progression'] = progression_table.get(rating.get('test_result', ''), 0) + for url in calculation["urls"]: + for endpoint in url["endpoints"]: + for rating in endpoint["ratings"]: + rating["simple_progression"] = progression_table.get(rating.get("test_result", ""), 0) def split_score_and_url(calculation: Dict[str, Any]): @@ -608,36 +665,35 @@ def split_score_and_url(calculation: Dict[str, Any]): :param report: :return: """ - for url in calculation['urls']: - for endpoint in url['endpoints']: + for url in calculation["urls"]: + for endpoint in url["endpoints"]: score: Union[int, str] = 0 url = "" scan = 0 since = "" last_scan = "" - for rating in endpoint['ratings']: - if rating['type'] in ["internet_nl_web_overall_score", "internet_nl_mail_dashboard_overall_score"]: + for rating in endpoint["ratings"]: + if rating["type"] in ["internet_nl_web_overall_score", "internet_nl_mail_dashboard_overall_score"]: # explanation "78 https://batch.interne…zuiderzeeland.nl/886818/" - explanation = rating['explanation'].split(" ") # type: ignore + explanation = rating["explanation"].split(" ") # type: ignore if explanation[0] == "error": - rating['internet_nl_score'] = score = "error" + rating["internet_nl_score"] = score = "error" else: - rating['internet_nl_score'] = score = int(explanation[0]) - rating['internet_nl_url'] = url = explanation[1] - scan = rating['scan'] - since = rating['since'] - last_scan = rating['last_scan'] + rating["internet_nl_score"] = score = int(explanation[0]) + rating["internet_nl_url"] = url = explanation[1] + scan = rating["scan"] + since = rating["since"] + last_scan = rating["last_scan"] # Now that we had all ratings, add a single value for the score, so we don't have to switch between # web or mail, which is severely annoying. # there is only one rating per set endpoint. So this is safe - endpoint['ratings'].append( + endpoint["ratings"].append( { "type": "internet_nl_score", "scan_type": "internet_nl_score", "internet_nl_score": score, "internet_nl_url": url, - # to comply with the rating structure "high": 0, "medium": 1, # make sure to match simple verdicts as defined above. @@ -646,7 +702,7 @@ def split_score_and_url(calculation: Dict[str, Any]): "not_testable": False, "not_applicable": False, "error_in_test": False, - 'test_result': score, + "test_result": score, "scan": scan, "since": since, "last_scan": last_scan, @@ -659,76 +715,93 @@ def add_statistics_over_ratings(calculation: Dict[str, Any]): # works only after ratings by type. # todo: in report section, move statistics_per_issue_type to calculation - calculation['statistics_per_issue_type'] = {} + calculation["statistics_per_issue_type"] = {} possible_issues = [] - for url in calculation['urls']: - for endpoint in url['endpoints']: - possible_issues += endpoint['ratings_by_type'].keys() + for url in calculation["urls"]: + for endpoint in url["endpoints"]: + possible_issues += endpoint["ratings_by_type"].keys() possible_issues = list(set(possible_issues)) # prepare the stats dict to have less expensive operations in the 3x nested loop for issue in possible_issues: # todo: could be a defaultdict. although explicit initialization is somewhat useful. - calculation['statistics_per_issue_type'][issue] = { - 'high': 0, 'medium': 0, 'low': 0, 'ok': 0, 'not_ok': 0, 'not_testable': 0, 'not_applicable': 0, - 'error_in_test': 0} + calculation["statistics_per_issue_type"][issue] = { + "high": 0, + "medium": 0, + "low": 0, + "ok": 0, + "not_ok": 0, + "not_testable": 0, + "not_applicable": 0, + "error_in_test": 0, + } # count the numbers, can we do this with some map/add function that is way faster? for issue in possible_issues: - for url in calculation['urls']: - for endpoint in url['endpoints']: - rating = endpoint['ratings_by_type'].get(issue, None) + for url in calculation["urls"]: + for endpoint in url["endpoints"]: + rating = endpoint["ratings_by_type"].get(issue, None) if not rating: continue - calculation['statistics_per_issue_type'][issue]['high'] += rating['high'] - calculation['statistics_per_issue_type'][issue]['medium'] += rating['medium'] - calculation['statistics_per_issue_type'][issue]['low'] += rating['low'] - calculation['statistics_per_issue_type'][issue]['not_testable'] += rating['not_testable'] - calculation['statistics_per_issue_type'][issue]['not_applicable'] += rating['not_applicable'] - calculation['statistics_per_issue_type'][issue]['error_in_test'] += rating['error_in_test'] + calculation["statistics_per_issue_type"][issue]["high"] += rating["high"] + calculation["statistics_per_issue_type"][issue]["medium"] += rating["medium"] + calculation["statistics_per_issue_type"][issue]["low"] += rating["low"] + calculation["statistics_per_issue_type"][issue]["not_testable"] += rating["not_testable"] + calculation["statistics_per_issue_type"][issue]["not_applicable"] += rating["not_applicable"] + calculation["statistics_per_issue_type"][issue]["error_in_test"] += rating["error_in_test"] # things that are not_testable or not_applicable do not have impact on thigns being OK # see: https://github.com/internetstandards/Internet.nl-dashboard/issues/68 - if not any([rating['not_testable'], rating['not_applicable'], rating['error_in_test']]): - calculation['statistics_per_issue_type'][issue]['ok'] += rating['ok'] + if not any([rating["not_testable"], rating["not_applicable"], rating["error_in_test"]]): + calculation["statistics_per_issue_type"][issue]["ok"] += rating["ok"] # these can be summed because only one of high, med, low is 1 - calculation['statistics_per_issue_type'][issue]['not_ok'] += \ - rating['high'] + rating['medium'] + rating['low'] + calculation["statistics_per_issue_type"][issue]["not_ok"] += ( + rating["high"] + rating["medium"] + rating["low"] + ) def add_percentages_to_statistics(calculation: Dict[str, Any]): - for key, _ in calculation['statistics_per_issue_type'].items(): - issue = calculation['statistics_per_issue_type'][key] + for key, _ in calculation["statistics_per_issue_type"].items(): + issue = calculation["statistics_per_issue_type"][key] # may 2020: we want to see the other issues in the graphs as being gray. - graphs_all = sum([issue['ok'], issue['high'], issue['medium'], issue['low'], - issue['not_testable'], issue['not_applicable'], issue['error_in_test']]) + graphs_all = sum( + [ + issue["ok"], + issue["high"], + issue["medium"], + issue["low"], + issue["not_testable"], + issue["not_applicable"], + issue["error_in_test"], + ] + ) if graphs_all == 0: # This happens when everything tested is not applicable or not testable: thus no stats: - calculation['statistics_per_issue_type'][key]['pct_high'] = 0 - calculation['statistics_per_issue_type'][key]['pct_medium'] = 0 - calculation['statistics_per_issue_type'][key]['pct_low'] = 0 - calculation['statistics_per_issue_type'][key]['pct_ok'] = 0 - calculation['statistics_per_issue_type'][key]['pct_not_ok'] = 0 + calculation["statistics_per_issue_type"][key]["pct_high"] = 0 + calculation["statistics_per_issue_type"][key]["pct_medium"] = 0 + calculation["statistics_per_issue_type"][key]["pct_low"] = 0 + calculation["statistics_per_issue_type"][key]["pct_ok"] = 0 + calculation["statistics_per_issue_type"][key]["pct_not_ok"] = 0 continue - tcskp = calculation['statistics_per_issue_type'][key] - tcskp['pct_high'] = round((issue['high'] / graphs_all) * 100, 2) - tcskp['pct_medium'] = round((issue['medium'] / graphs_all) * 100, 2) - tcskp['pct_low'] = round((issue['low'] / graphs_all) * 100, 2) + tcskp = calculation["statistics_per_issue_type"][key] + tcskp["pct_high"] = round((issue["high"] / graphs_all) * 100, 2) + tcskp["pct_medium"] = round((issue["medium"] / graphs_all) * 100, 2) + tcskp["pct_low"] = round((issue["low"] / graphs_all) * 100, 2) # all other possible stuff. Note that no_mx, unreachable and such have been mapped to one of these. - tcskp['pct_not_applicable'] = round((issue['not_applicable'] / graphs_all) * 100, 2) - tcskp['pct_not_testable'] = round((issue['not_testable'] / graphs_all) * 100, 2) - tcskp['pct_error_in_test'] = round((issue['error_in_test'] / graphs_all) * 100, 2) + tcskp["pct_not_applicable"] = round((issue["not_applicable"] / graphs_all) * 100, 2) + tcskp["pct_not_testable"] = round((issue["not_testable"] / graphs_all) * 100, 2) + tcskp["pct_error_in_test"] = round((issue["error_in_test"] / graphs_all) * 100, 2) # May 2019 warning (=medium) and info(=low) do NOT have a score impact, only high has a score impact. # https://www.internet.nl/faqs/report/ # This has been altered in May 2020 to avoid confusion and show different kinds of values, it's now just OK # instead of including medium and low as ok. - tcskp['pct_ok'] = round(((issue['ok']) / graphs_all) * 100, 2) - tcskp['pct_not_ok'] = round((issue['not_ok'] / graphs_all) * 100, 2) + tcskp["pct_ok"] = round(((issue["ok"]) / graphs_all) * 100, 2) + tcskp["pct_not_ok"] = round((issue["not_ok"] / graphs_all) * 100, 2) def share(account, report_id, share_code): @@ -785,16 +858,21 @@ def update_report_code(account, report_id): def get_report_for_sharing(account: Account, report_id: int, is_publicly_shared: bool) -> Any: - return UrlListReport.objects.all().filter( - urllist__account=account, urllist__is_deleted=False, id=report_id, is_publicly_shared=is_publicly_shared - ).defer('calculation').first() + return ( + UrlListReport.objects.all() + .filter( + urllist__account=account, urllist__is_deleted=False, id=report_id, is_publicly_shared=is_publicly_shared + ) + .defer("calculation") + .first() + ) def report_sharing_data(report: UrlListReport) -> Dict[str, Any]: return { - 'public_report_code': report.public_report_code, - 'public_share_code': report.public_share_code, - 'is_publicly_shared': report.is_publicly_shared + "public_report_code": report.public_report_code, + "public_share_code": report.public_share_code, + "is_publicly_shared": report.is_publicly_shared, } diff --git a/dashboard/internet_nl_dashboard/logic/report_comparison.py b/dashboard/internet_nl_dashboard/logic/report_comparison.py index 7e434395..abe22179 100644 --- a/dashboard/internet_nl_dashboard/logic/report_comparison.py +++ b/dashboard/internet_nl_dashboard/logic/report_comparison.py @@ -52,20 +52,20 @@ def filter_comparison_report(comparison_report: Dict[str, Any], impact: str = "i data = [] # prevent key errors - if impact not in ['regression', 'improvement', 'neutral']: + if impact not in ["regression", "improvement", "neutral"]: return [] # perhaps there is nothing to compare if "comparison" not in comparison_report: return [] - for comparison_url_key in comparison_report['comparison'].keys(): - if comparison_report['comparison'][comparison_url_key]['changes'][impact] > 0: - data.append(comparison_report['comparison'][comparison_url_key]) + for comparison_url_key in comparison_report["comparison"].keys(): + if comparison_report["comparison"][comparison_url_key]["changes"][impact] > 0: + data.append(comparison_report["comparison"][comparison_url_key]) # sort the data on domain and then subdomain. Make sure example.com and example.nl are separate and that # the subdomains of example.com are near example.com - data = sorted(data, key=lambda k: (k['computed_domain_and_suffix'], k['computed_subdomain']), reverse=False) + data = sorted(data, key=lambda k: (k["computed_domain_and_suffix"], k["computed_subdomain"]), reverse=False) return data @@ -85,27 +85,23 @@ def render_comparison_view(data: List[Dict[str, Any]], impact: str = "improvemen """ # prevent an arbitrary file inclusion - if impact not in ['regression', 'improvement', 'neutral']: + if impact not in ["regression", "improvement", "neutral"]: return "" # In case there was nothing to compare, there is no output. if not data: return "" - metrics_keys = { - 'regression': 'regressed_metrics', - 'improvement': 'improved_metrics', - 'neutral': 'neutral_metrics' - } + metrics_keys = {"regression": "regressed_metrics", "improvement": "improved_metrics", "neutral": "neutral_metrics"} # make sure the translations are correct for the fields: translation_dictionary = get_po_as_dictionary_v2(language) for item in data: translated = [] - for metric in item['changes'][metrics_keys[impact]]: + for metric in item["changes"][metrics_keys[impact]]: translated_field = translate_field(metric, translation_dictionary=translation_dictionary) translated.append(translated_field) - item['changes'][metrics_keys[impact]] = translated + item["changes"][metrics_keys[impact]] = translated # Make sure any translations in the template itself is done with the correct language. # Try to _not_ use translations inside this template. @@ -199,78 +195,67 @@ def compare_report_in_detail(new_report, old_report) -> Dict[str, Any]: """ comparison_report: Dict[str, Union[Dict[str, Any], Any]] = { - 'urls_exclusive_in_new_report': list( - set(new_report['calculation']['urls_by_url'].keys()) - - set(old_report['calculation']['urls_by_url'].keys()) + "urls_exclusive_in_new_report": list( + set(new_report["calculation"]["urls_by_url"].keys()) - set(old_report["calculation"]["urls_by_url"].keys()) ), - 'urls_exclusive_in_old_report': list( - set(old_report['calculation']['urls_by_url'].keys()) - - set(new_report['calculation']['urls_by_url'].keys()) + "urls_exclusive_in_old_report": list( + set(old_report["calculation"]["urls_by_url"].keys()) - set(new_report["calculation"]["urls_by_url"].keys()) ), - 'old': { - 'average_internet_nl_score': old_report.get('average_internet_nl_score', 0), - 'number_of_urls': old_report.get('total_urls', 0), - 'data_from': old_report.get('at_when', datetime.now()), - 'report_id': old_report.get('id', 0), - 'urllist_id': old_report.get('urllist_id', 0) + "old": { + "average_internet_nl_score": old_report.get("average_internet_nl_score", 0), + "number_of_urls": old_report.get("total_urls", 0), + "data_from": old_report.get("at_when", datetime.now()), + "report_id": old_report.get("id", 0), + "urllist_id": old_report.get("urllist_id", 0), }, - 'new': { - 'average_internet_nl_score': new_report.get('average_internet_nl_score', 0), - 'number_of_urls': new_report.get('total_urls', 0), - 'data_from': new_report.get('at_when', datetime.now()), - 'report_id': new_report.get('id', 0), - 'urllist_id': new_report.get('urllist_id', 0) + "new": { + "average_internet_nl_score": new_report.get("average_internet_nl_score", 0), + "number_of_urls": new_report.get("total_urls", 0), + "data_from": new_report.get("at_when", datetime.now()), + "report_id": new_report.get("id", 0), + "urllist_id": new_report.get("urllist_id", 0), }, - 'summary': { - 'improvement': 0, - 'regression': 0, - 'neutral': 0, + "summary": { + "improvement": 0, + "regression": 0, + "neutral": 0, }, - 'comparison': {} + "comparison": {}, } - for url_key in new_report['calculation']['urls_by_url'].keys(): - new_url_data = new_report['calculation']['urls_by_url'][url_key] - old_url_data = old_report['calculation']['urls_by_url'].get(url_key, {}) + for url_key in new_report["calculation"]["urls_by_url"].keys(): + new_url_data = new_report["calculation"]["urls_by_url"][url_key] + old_url_data = old_report["calculation"]["urls_by_url"].get(url_key, {}) my_extract = tldextract.extract(url_key) data: Dict[str, Any] = { - 'url': url_key, + "url": url_key, # Added for ordering the comparison results. So it's possible to sort like this: # - hyves.nl # - internet.nl # - acc.internet.nl # - beta.internet.nl # - justeat.nl - 'computed_suffix': my_extract.suffix, - 'computed_domain': my_extract.domain, - 'computed_subdomain': my_extract.subdomain, - 'computed_domain_and_suffix': f'{my_extract.domain}.{my_extract.suffix}', - + "computed_suffix": my_extract.suffix, + "computed_domain": my_extract.domain, + "computed_subdomain": my_extract.subdomain, + "computed_domain_and_suffix": f"{my_extract.domain}.{my_extract.suffix}", # In case there is no test data / no endpoint, this is set to False (the default). # You can see this being set to true with acc.internet.nl - 'test_results_from_internet_nl_available': False, - + "test_results_from_internet_nl_available": False, # in case there was no endpoint, fill the report with some information: # This is done to reduce 'edge cases' and 'flags by values': it allows consistent summary of # the number of total issues, and it will also set a flag this could not be tested. - 'changes': { - 'improvement': 0, - 'neutral': 0, - 'regression': 0, - 'neutral_metrics': [], - 'regressed_metrics': [], - 'improved_metrics': [], + "changes": { + "improvement": 0, + "neutral": 0, + "regression": 0, + "neutral_metrics": [], + "regressed_metrics": [], + "improved_metrics": [], }, - - 'old': { - 'report': '', - 'score': 0 - }, - 'new': { - 'report': '', - 'score': 0 - } + "old": {"report": "", "score": 0}, + "new": {"report": "", "score": 0}, } # Looping through endpoints is not really useful, because we only recognize and can process internet.nl @@ -278,42 +263,42 @@ def compare_report_in_detail(new_report, old_report) -> Dict[str, Any]: # will leave new, old and changes empty endpoint_key = "" # web, the report type will be in the report itself soon, so that will be easier. - if "dns_a_aaaa/0 IPv0" in new_url_data['endpoints_by_key'].keys(): + if "dns_a_aaaa/0 IPv0" in new_url_data["endpoints_by_key"].keys(): endpoint_key = "dns_a_aaaa/0 IPv0" # mail - if "dns_soa/0 IPv0" in new_url_data['endpoints_by_key'].keys(): + if "dns_soa/0 IPv0" in new_url_data["endpoints_by_key"].keys(): endpoint_key = "dns_soa/0 IPv0" if endpoint_key: - data['test_results_from_internet_nl_available'] = True + data["test_results_from_internet_nl_available"] = True - new_endpoint_data = new_url_data['endpoints_by_key'][endpoint_key] - old_endpoint_data = old_url_data.get('endpoints_by_key', {}).get(endpoint_key, {}) + new_endpoint_data = new_url_data["endpoints_by_key"][endpoint_key] + old_endpoint_data = old_url_data.get("endpoints_by_key", {}).get(endpoint_key, {}) # Our base is the 'new' domain. A scan might have failed in either case... - if new_endpoint_data.get('ratings_by_type', None): - data['new'] = { - 'report': new_endpoint_data['ratings_by_type']['internet_nl_score']['internet_nl_url'], - 'score': new_endpoint_data['ratings_by_type']['internet_nl_score']['internet_nl_score'] + if new_endpoint_data.get("ratings_by_type", None): + data["new"] = { + "report": new_endpoint_data["ratings_by_type"]["internet_nl_score"]["internet_nl_url"], + "score": new_endpoint_data["ratings_by_type"]["internet_nl_score"]["internet_nl_score"], } - if old_endpoint_data.get('ratings_by_type', {}).get('internet_nl_score', {}).get('internet_nl_url', {}): - data['old'] = { - 'report': old_endpoint_data['ratings_by_type']['internet_nl_score']['internet_nl_url'], - 'score': old_endpoint_data['ratings_by_type']['internet_nl_score']['internet_nl_score'] + if old_endpoint_data.get("ratings_by_type", {}).get("internet_nl_score", {}).get("internet_nl_url", {}): + data["old"] = { + "report": old_endpoint_data["ratings_by_type"]["internet_nl_score"]["internet_nl_url"], + "score": old_endpoint_data["ratings_by_type"]["internet_nl_score"]["internet_nl_score"], } - data['changes'] = determine_changes_in_ratings( - new_endpoint_data.get('ratings_by_type', {}), - old_endpoint_data.get('ratings_by_type', {}), + data["changes"] = determine_changes_in_ratings( + new_endpoint_data.get("ratings_by_type", {}), + old_endpoint_data.get("ratings_by_type", {}), ) - comparison_report['comparison'][url_key] = data + comparison_report["comparison"][url_key] = data # add the summary to the report, which is just a simple loop with counters - for _, report_per_url in comparison_report['comparison'].items(): - comparison_report['summary']['improvement'] += report_per_url['changes']['improvement'] - comparison_report['summary']['regression'] += report_per_url['changes']['regression'] - comparison_report['summary']['neutral'] += report_per_url['changes']['neutral'] + for _, report_per_url in comparison_report["comparison"].items(): + comparison_report["summary"]["improvement"] += report_per_url["changes"]["improvement"] + comparison_report["summary"]["regression"] += report_per_url["changes"]["regression"] + comparison_report["summary"]["neutral"] += report_per_url["changes"]["neutral"] return comparison_report @@ -324,26 +309,23 @@ def determine_changes_in_ratings(new_ratings_data, old_ratings_data) -> Dict[str # we don't want to have statistics over these fields, they are processed elsewhere. ratings_to_ignore = [ # Scoring can be ignored, it's included in the comparison elsewhere. - 'internet_nl_mail_dashboard_overall_score', - 'internet_nl_web_overall_score', - 'internet_nl_score', - + "internet_nl_mail_dashboard_overall_score", + "internet_nl_web_overall_score", + "internet_nl_score", # Removed metrics that will not be used again, which thus will not have data: - 'internet_nl_web_appsecpriv_x_xss_protection', - + "internet_nl_web_appsecpriv_x_xss_protection", # Categories are just higher levels of metrics, and that will skew the results if included, because # if one subtest fails, the category is also wrong. - 'internet_nl_web_tls', - 'internet_nl_web_ipv6', - 'internet_nl_web_appsecpriv', - 'internet_nl_web_dnssec', - 'internet_nl_web_rpki', - 'internet_nl_mail_dashboard_tls', - 'internet_nl_mail_dashboard_auth', - 'internet_nl_mail_dashboard_dnssec', - 'internet_nl_mail_dashboard_ipv6', - 'internet_nl_mail_dashboard_rpki', - + "internet_nl_web_tls", + "internet_nl_web_ipv6", + "internet_nl_web_appsecpriv", + "internet_nl_web_dnssec", + "internet_nl_web_rpki", + "internet_nl_mail_dashboard_tls", + "internet_nl_mail_dashboard_auth", + "internet_nl_mail_dashboard_dnssec", + "internet_nl_mail_dashboard_ipv6", + "internet_nl_mail_dashboard_rpki", # Some fields that are status indications, old scans and old reports. # They are ignored because they dont have a test_result field. # Fields such as: 'internet_nl_mail_server_configured', and so on... @@ -353,56 +335,64 @@ def determine_changes_in_ratings(new_ratings_data, old_ratings_data) -> Dict[str # 'internet_nl_mail_auth_dmarc_policy_only', # 'internet_nl_mail_auth_dmarc_ext_destination', ] - neutral_test_result_values = \ - ["unknown", "not_applicable", "not_testable", 'no_mx', 'unreachable', 'error_in_test', 'error', 'not_tested'] + neutral_test_result_values = [ + "unknown", + "not_applicable", + "not_testable", + "no_mx", + "unreachable", + "error_in_test", + "error", + "not_tested", + ] # prepare result changes: Dict[str, int] = { - 'improvement': 0, - 'regression': 0, - 'neutral': 0, + "improvement": 0, + "regression": 0, + "neutral": 0, } changed_metrics: Dict[str, List[str]] = { - 'improved_metrics': [], - 'regressed_metrics': [], - 'neutral_metrics': [], + "improved_metrics": [], + "regressed_metrics": [], + "neutral_metrics": [], } for rating_key in new_ratings_data.keys(): # Note that the following logic should be extactly the same as in the dashboard report GUI. # skip internet.nl extra fields and the internet.nl score field handled above: - if '_legacy_' in rating_key or rating_key in ratings_to_ignore: + if "_legacy_" in rating_key or rating_key in ratings_to_ignore: continue - new_test_result = new_ratings_data[rating_key].get('test_result', None) + new_test_result = new_ratings_data[rating_key].get("test_result", None) if new_test_result is None: # fields that are not supported, status fields and all other stuff prior to the 'test_result' field: continue # both the rating_key as well as test_result could be missing - old_test_result = old_ratings_data.get(rating_key, {}).get('test_result', 'unknown') + old_test_result = old_ratings_data.get(rating_key, {}).get("test_result", "unknown") # in case of uncomparable results, a neutral verdict is given: # uncomparable includes all situations where the old_data was not present. if new_test_result in neutral_test_result_values or old_test_result in neutral_test_result_values: - changes['neutral'] += 1 - changed_metrics['neutral_metrics'].append(rating_key) + changes["neutral"] += 1 + changed_metrics["neutral_metrics"].append(rating_key) continue # in other cases there will be simple progression - new_simple_progression = new_ratings_data[rating_key]['simple_progression'] - old_simple_progression = old_ratings_data[rating_key]['simple_progression'] + new_simple_progression = new_ratings_data[rating_key]["simple_progression"] + old_simple_progression = old_ratings_data[rating_key]["simple_progression"] if new_simple_progression == old_simple_progression: - changes['neutral'] += 1 - changed_metrics['neutral_metrics'].append(rating_key) + changes["neutral"] += 1 + changed_metrics["neutral_metrics"].append(rating_key) elif new_simple_progression > old_simple_progression: - changes['improvement'] += 1 - changed_metrics['improved_metrics'].append(rating_key) + changes["improvement"] += 1 + changed_metrics["improved_metrics"].append(rating_key) else: - changes['regression'] += 1 - changed_metrics['regressed_metrics'].append(rating_key) + changes["regression"] += 1 + changed_metrics["regressed_metrics"].append(rating_key) return {**changes, **changed_metrics} @@ -440,19 +430,19 @@ def key_calculation(report_data): :return: """ - if "urls" not in report_data['calculation']: - report_data['calculation']['urls_by_url'] = [] + if "urls" not in report_data["calculation"]: + report_data["calculation"]["urls_by_url"] = [] return report_data urls_by_key = {} - for url in report_data['calculation']['urls']: - urls_by_key[url['url']] = url + for url in report_data["calculation"]["urls"]: + urls_by_key[url["url"]] = url for url, url_by_key in urls_by_key.items(): endpoints_by_key = {} - for endpoint in url_by_key['endpoints']: - endpoints_by_key[endpoint['concat']] = endpoint - urls_by_key[url]['endpoints_by_key'] = endpoints_by_key + for endpoint in url_by_key["endpoints"]: + endpoints_by_key[endpoint["concat"]] = endpoint + urls_by_key[url]["endpoints_by_key"] = endpoints_by_key - report_data['calculation']['urls_by_url'] = urls_by_key + report_data["calculation"]["urls_by_url"] = urls_by_key return report_data diff --git a/dashboard/internet_nl_dashboard/logic/report_to_spreadsheet.py b/dashboard/internet_nl_dashboard/logic/report_to_spreadsheet.py index c2c8eca4..15e89167 100644 --- a/dashboard/internet_nl_dashboard/logic/report_to_spreadsheet.py +++ b/dashboard/internet_nl_dashboard/logic/report_to_spreadsheet.py @@ -15,23 +15,45 @@ from websecmap.reporting.diskreport import retrieve_report from dashboard.internet_nl_dashboard.logic import FIELD_TO_CATEGORY_MAP # pylint: disable=duplicate-code -from dashboard.internet_nl_dashboard.logic import (MAIL_AUTH_CATEGORY, MAIL_AUTH_FIELDS, MAIL_DNSSEC_CATEGORY, - MAIL_DNSSEC_FIELDS, MAIL_IPV6_CATEGORY, MAIL_IPV6_FIELDS, - MAIL_LEGACY_FIELDS, MAIL_OVERALL_FIELDS, MAIL_RPKI_CATEGORY, - MAIL_RPKI_FIELDS, MAIL_TLS_CATEGORY, MAIL_TLS_CERTIFICATE_FIELDS, - MAIL_TLS_DANE_FIELDS, MAIL_TLS_TLS_FIELDS, WEB_APPSECPRIV_CATEGORY, - WEB_APPSECPRIV_FIELDS, WEB_DNSSEC_CATEGORY, WEB_DNSSEC_FIELDS, - WEB_IPV6_CATEGORY, WEB_IPV6_FIELDS, WEB_LEGACY_CATEGORY, - WEB_LEGACY_FIELDS, WEB_OVERALL_FIELDS, WEB_RPKI_CATEGORY, - WEB_RPKI_FIELDS, WEB_TLS_CATEGORY, WEB_TLS_CERTIFICATE_FIELDS, - WEB_TLS_DANE_FIELDS, WEB_TLS_HTTP_FIELDS, WEB_TLS_TLS_FIELDS) +from dashboard.internet_nl_dashboard.logic import ( + MAIL_AUTH_CATEGORY, + MAIL_AUTH_FIELDS, + MAIL_DNSSEC_CATEGORY, + MAIL_DNSSEC_FIELDS, + MAIL_IPV6_CATEGORY, + MAIL_IPV6_FIELDS, + MAIL_LEGACY_FIELDS, + MAIL_OVERALL_FIELDS, + MAIL_RPKI_CATEGORY, + MAIL_RPKI_FIELDS, + MAIL_TLS_CATEGORY, + MAIL_TLS_CERTIFICATE_FIELDS, + MAIL_TLS_DANE_FIELDS, + MAIL_TLS_TLS_FIELDS, + WEB_APPSECPRIV_CATEGORY, + WEB_APPSECPRIV_FIELDS, + WEB_DNSSEC_CATEGORY, + WEB_DNSSEC_FIELDS, + WEB_IPV6_CATEGORY, + WEB_IPV6_FIELDS, + WEB_LEGACY_CATEGORY, + WEB_LEGACY_FIELDS, + WEB_OVERALL_FIELDS, + WEB_RPKI_CATEGORY, + WEB_RPKI_FIELDS, + WEB_TLS_CATEGORY, + WEB_TLS_CERTIFICATE_FIELDS, + WEB_TLS_DANE_FIELDS, + WEB_TLS_HTTP_FIELDS, + WEB_TLS_TLS_FIELDS, +) from dashboard.internet_nl_dashboard.logic.internet_nl_translations import get_po_as_dictionary_v2, translate_field from dashboard.internet_nl_dashboard.models import Account, TaggedUrlInUrllist, Url, UrlListReport log = logging.getLogger(__package__) # todo: read the preferred language of the user and use the translations matching this user... -po_file_as_dictionary = get_po_as_dictionary_v2('en') +po_file_as_dictionary = get_po_as_dictionary_v2("en") """ Creates spreadsheets containing report data. @@ -47,39 +69,31 @@ SANE_COLUMN_ORDER = { # scanner - 'dns_a_aaaa': { - 'overall': WEB_OVERALL_FIELDS, - - 'ipv6': WEB_IPV6_CATEGORY + WEB_IPV6_FIELDS, - - 'dnssec': WEB_DNSSEC_CATEGORY + WEB_DNSSEC_FIELDS, - - 'tls': WEB_TLS_CATEGORY + WEB_TLS_HTTP_FIELDS + WEB_TLS_TLS_FIELDS + WEB_TLS_CERTIFICATE_FIELDS + - WEB_TLS_DANE_FIELDS, - + "dns_a_aaaa": { + "overall": WEB_OVERALL_FIELDS, + "ipv6": WEB_IPV6_CATEGORY + WEB_IPV6_FIELDS, + "dnssec": WEB_DNSSEC_CATEGORY + WEB_DNSSEC_FIELDS, + "tls": WEB_TLS_CATEGORY + + WEB_TLS_HTTP_FIELDS + + WEB_TLS_TLS_FIELDS + + WEB_TLS_CERTIFICATE_FIELDS + + WEB_TLS_DANE_FIELDS, # Added 24th of May 2019 - 'appsecpriv': WEB_APPSECPRIV_CATEGORY + WEB_APPSECPRIV_FIELDS, - - 'rpki': WEB_RPKI_CATEGORY + WEB_RPKI_FIELDS, - - 'legacy': WEB_LEGACY_CATEGORY + WEB_LEGACY_FIELDS + "appsecpriv": WEB_APPSECPRIV_CATEGORY + WEB_APPSECPRIV_FIELDS, + "rpki": WEB_RPKI_CATEGORY + WEB_RPKI_FIELDS, + "legacy": WEB_LEGACY_CATEGORY + WEB_LEGACY_FIELDS, }, - 'dns_soa': { + "dns_soa": { # any grouping, every group has a empty column between them. The label is not used. - 'overall': MAIL_OVERALL_FIELDS, - 'ipv6': MAIL_IPV6_CATEGORY + MAIL_IPV6_FIELDS, - - 'dnssec': MAIL_DNSSEC_CATEGORY + MAIL_DNSSEC_FIELDS, - - 'auth': MAIL_AUTH_CATEGORY + MAIL_AUTH_FIELDS, - + "overall": MAIL_OVERALL_FIELDS, + "ipv6": MAIL_IPV6_CATEGORY + MAIL_IPV6_FIELDS, + "dnssec": MAIL_DNSSEC_CATEGORY + MAIL_DNSSEC_FIELDS, + "auth": MAIL_AUTH_CATEGORY + MAIL_AUTH_FIELDS, # perhaps split these into multiple groups. - 'tls': MAIL_TLS_CATEGORY + MAIL_TLS_TLS_FIELDS + MAIL_TLS_CERTIFICATE_FIELDS + MAIL_TLS_DANE_FIELDS, - - 'rpki': MAIL_RPKI_CATEGORY + MAIL_RPKI_FIELDS, - + "tls": MAIL_TLS_CATEGORY + MAIL_TLS_TLS_FIELDS + MAIL_TLS_CERTIFICATE_FIELDS + MAIL_TLS_DANE_FIELDS, + "rpki": MAIL_RPKI_CATEGORY + MAIL_RPKI_FIELDS, # #358 MAIL_LEGACY_CATEGORY is useless - 'legacy': MAIL_LEGACY_FIELDS + "legacy": MAIL_LEGACY_FIELDS, }, } @@ -97,24 +111,24 @@ def get_tags_from_urllist_urls(account: Account, urllist_id: int) -> Dict[str, L # This might significantly slow down an export, and this might be a separate process is we're going to process # a ton of huge lists in the future. prefetch_tags = Prefetch( - 'taggedurlinurllist_set', - queryset=TaggedUrlInUrllist.objects.all().filter(urllist=urllist_id).prefetch_related('tags'), - to_attr='url_tags' + "taggedurlinurllist_set", + queryset=TaggedUrlInUrllist.objects.all().filter(urllist=urllist_id).prefetch_related("tags"), + to_attr="url_tags", ) # This ordering makes sure all subdomains are near the domains with the right extension. - urls = Url.objects.all().filter( - urls_in_dashboard_list_2__account=account, - urls_in_dashboard_list_2__id=urllist_id - ).prefetch_related( - prefetch_tags - ).all() + urls = ( + Url.objects.all() + .filter(urls_in_dashboard_list_2__account=account, urls_in_dashboard_list_2__id=urllist_id) + .prefetch_related(prefetch_tags) + .all() + ) data = {} for url in urls: tags = [] - for tag1 in [x.tags.values_list('name') for x in url.url_tags]: + for tag1 in [x.tags.values_list("name") for x in url.url_tags]: for tag2 in tag1: tags.extend(iter(tag2)) data[url.url] = tags @@ -124,9 +138,9 @@ def get_tags_from_urllist_urls(account: Account, urllist_id: int) -> Dict[str, L def create_spreadsheet(account: Account, report_id: int): # Fails softly, without exceptions if there is nothing yet. - report = UrlListReport.objects.all().filter( - urllist__account=account, - pk=report_id).select_related('urllist').first() + report = ( + UrlListReport.objects.all().filter(urllist__account=account, pk=report_id).select_related("urllist").first() + ) if not report: return None, None @@ -138,7 +152,7 @@ def create_spreadsheet(account: Account, report_id: int): # so when exporting they need to be retrieved one by one, which takes a lot of time. url_tag_mapping = get_tags_from_urllist_urls(account, report.urllist.id) - protocol = 'dns_soa' if report.report_type == 'mail' else 'dns_a_aaaa' + protocol = "dns_soa" if report.report_type == "mail" else "dns_a_aaaa" # results is a matrix / 2-d array / array with arrays. data: List[List[Any]] = [] @@ -154,11 +168,14 @@ def create_spreadsheet(account: Account, report_id: int): data += [category_headers(protocol)] data += [subcategory_headers(protocol)] data += [headers(protocol)] - data += urllistreport_to_spreadsheet_data(category_name=report.urllist.name, urls=urls, protocol=protocol, - tags=url_tag_mapping) + data += urllistreport_to_spreadsheet_data( + category_name=report.urllist.name, urls=urls, protocol=protocol, tags=url_tag_mapping + ) - filename = "internet nl dashboard report " \ - f"{report.pk} {report.urllist.name} {report.urllist.scan_type} {report.at_when.date()}" + filename = ( + "internet nl dashboard report " + f"{report.pk} {report.urllist.name} {report.urllist.scan_type} {report.at_when.date()}" + ) # The sheet is created into memory and then passed to the caller. They may save it, or serve it, etc... # http://docs.pyexcel.org/en/latest/tutorial06.html?highlight=memory @@ -187,79 +204,144 @@ def upgrade_excel_spreadsheet(spreadsheet_data): # Add statistic rows: worksheet.insert_rows(0, amount=9) - worksheet['B1'] = "Total" - worksheet['B2'] = "Passed" - worksheet['B3'] = "Info" - worksheet['B4'] = "Warning" - worksheet['B5'] = "Failed" - worksheet['B6'] = "Not tested" - worksheet['B7'] = "Error" - worksheet['B8'] = "Test not applicable (mail only)" - worksheet['B9'] = "Percentage passed" + worksheet["B1"] = "Total" + worksheet["B2"] = "Passed" + worksheet["B3"] = "Info" + worksheet["B4"] = "Warning" + worksheet["B5"] = "Failed" + worksheet["B6"] = "Not tested" + worksheet["B7"] = "Error" + worksheet["B8"] = "Test not applicable (mail only)" + worksheet["B9"] = "Percentage passed" # one to one match on the values, so they can be filtered - worksheet['I1'] = "<>" - worksheet['I2'] = "passed" - worksheet['I3'] = "info" - worksheet['I4'] = "warning" - worksheet['I5'] = "failed" - worksheet['I6'] = "not_tested" - worksheet['I7'] = "error" - worksheet['I8'] = "not_applicable" + worksheet["I1"] = "<>" + worksheet["I2"] = "passed" + worksheet["I3"] = "info" + worksheet["I4"] = "warning" + worksheet["I5"] = "failed" + worksheet["I6"] = "not_tested" + worksheet["I7"] = "error" + worksheet["I8"] = "not_applicable" # bold totals: for i in range(1, 10): - worksheet[f'B{i}'].font = Font(bold=True) + worksheet[f"B{i}"].font = Font(bold=True) data_columns = [ - 'J', 'K', 'L', "M", "N", 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'AA', 'AB', 'AC', 'AD', 'AE', 'AF', 'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', 'AO', - 'AP', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AV', 'AW', 'AX', 'AY', 'AZ', 'BA', 'BB', 'BC', 'BD', - 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ', 'BK', "BL", "BM", "BN", "BO", "BP", "BQ", "BR", "BS", - "BT", "BU", "BV", "BW", "BX", "BY", "BZ", "CA", "CB", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "AA", + "AB", + "AC", + "AD", + "AE", + "AF", + "AG", + "AH", + "AI", + "AJ", + "AK", + "AL", + "AM", + "AN", + "AO", + "AP", + "AQ", + "AR", + "AS", + "AT", + "AU", + "AV", + "AW", + "AX", + "AY", + "AZ", + "BA", + "BB", + "BC", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", + "BK", + "BL", + "BM", + "BN", + "BO", + "BP", + "BQ", + "BR", + "BS", + "BT", + "BU", + "BV", + "BW", + "BX", + "BY", + "BZ", + "CA", + "CB", ] # add some statistics for cell in data_columns: # if header, then aggregate - if worksheet[f'{cell}12'].value: + if worksheet[f"{cell}12"].value: _extracted_from_upgrade_excel_spreadsheet_57(cell, lines, worksheet) # make headers bold - worksheet['A12'].font = Font(bold=True) # List - worksheet['B12'].font = Font(bold=True) # Url - worksheet['C12'].font = Font(bold=True) # Subdomain - worksheet['D12'].font = Font(bold=True) # Domain - worksheet['E12'].font = Font(bold=True) # Suffix - worksheet['F12'].font = Font(bold=True) # Tags - worksheet['G12'].font = Font(bold=True) # InStats - worksheet['H11'].font = Font(bold=True) # overall - worksheet['H12'].font = Font(bold=True) # % Score - worksheet['I12'].font = Font(bold=True) # Report + worksheet["A12"].font = Font(bold=True) # List + worksheet["B12"].font = Font(bold=True) # Url + worksheet["C12"].font = Font(bold=True) # Subdomain + worksheet["D12"].font = Font(bold=True) # Domain + worksheet["E12"].font = Font(bold=True) # Suffix + worksheet["F12"].font = Font(bold=True) # Tags + worksheet["G12"].font = Font(bold=True) # InStats + worksheet["H11"].font = Font(bold=True) # overall + worksheet["H12"].font = Font(bold=True) # % Score + worksheet["I12"].font = Font(bold=True) # Report for cell in data_columns: - worksheet[f'{cell}11'].font = Font(bold=True) - worksheet[f'{cell}12'].font = Font(bold=True) - worksheet[f'{cell}13'].font = Font(bold=True) + worksheet[f"{cell}11"].font = Font(bold=True) + worksheet[f"{cell}12"].font = Font(bold=True) + worksheet[f"{cell}13"].font = Font(bold=True) # Freeze pane to make navigation easier. - worksheet.freeze_panes = worksheet['K14'] + worksheet.freeze_panes = worksheet["K14"] # there is probably a feature that puts this in a single conditional value. conditional_rules = { - "passed": PatternFill(start_color='B7FFC8', end_color='B7FFC8', fill_type='solid'), - "failed": PatternFill(start_color='FFB7B7', end_color='FFB7B7', fill_type='solid'), - "warning": PatternFill(start_color='FFD9B7', end_color='FFD9B7', fill_type='solid'), - "info": PatternFill(start_color='B7E3FF', end_color='B7E3FF', fill_type='solid'), - "good_not_tested": PatternFill(start_color='99FFFF', end_color='C0C0C0', fill_type='solid'), - "not_tested": PatternFill(start_color='99FFFF', end_color='DBDBDB', fill_type='solid'), + "passed": PatternFill(start_color="B7FFC8", end_color="B7FFC8", fill_type="solid"), + "failed": PatternFill(start_color="FFB7B7", end_color="FFB7B7", fill_type="solid"), + "warning": PatternFill(start_color="FFD9B7", end_color="FFD9B7", fill_type="solid"), + "info": PatternFill(start_color="B7E3FF", end_color="B7E3FF", fill_type="solid"), + "good_not_tested": PatternFill(start_color="99FFFF", end_color="C0C0C0", fill_type="solid"), + "not_tested": PatternFill(start_color="99FFFF", end_color="DBDBDB", fill_type="solid"), } # Set the measurements to green/red depending on value using conditional formatting. # There is no true/false, but we can color based on value. for grade, pattern in conditional_rules.items(): worksheet.conditional_formatting.add( - f'F13:CD{lines}', - CellIsRule(operator='=', formula=[f'"{grade}"'], stopIfTrue=True, fill=pattern) + f"F13:CD{lines}", CellIsRule(operator="=", formula=[f'"{grade}"'], stopIfTrue=True, fill=pattern) ) workbook.save(tmp.name) @@ -272,69 +354,73 @@ def _extracted_from_upgrade_excel_spreadsheet_57(cell, lines, worksheet): # There is a max of 5000 domains per scan. So we set this to something lower. # There is no good support of headers versus data, which makes working with excel a drama # If you ever read this code, and want a good spreadsheet editor: try Apple Numbers. It's fantastic. - worksheet[f'{cell}1'] = f'=COUNTA({cell}13:{cell}{lines})' + worksheet[f"{cell}1"] = f"=COUNTA({cell}13:{cell}{lines})" # todo: also support other values - worksheet[f'{cell}2'] = f'=COUNTIF({cell}13:{cell}{lines}, "passed")' - worksheet[f'{cell}3'] = f'=COUNTIF({cell}13:{cell}{lines}, "info")' - worksheet[f'{cell}4'] = f'=COUNTIF({cell}13:{cell}{lines}, "warning")' - worksheet[f'{cell}5'] = f'=COUNTIF({cell}13:{cell}{lines}, "failed")' - worksheet[f'{cell}6'] = f'=COUNTIF({cell}13:{cell}{lines}, "not_tested")' - worksheet[f'{cell}7'] = f'=' \ - f'COUNTIF({cell}13:{cell}{lines}, "error")+' \ - f'COUNTIF({cell}13:{cell}{lines}, "unreachable")+' \ - f'COUNTIF({cell}13:{cell}{lines}, "untestable")+' \ + worksheet[f"{cell}2"] = f'=COUNTIF({cell}13:{cell}{lines}, "passed")' + worksheet[f"{cell}3"] = f'=COUNTIF({cell}13:{cell}{lines}, "info")' + worksheet[f"{cell}4"] = f'=COUNTIF({cell}13:{cell}{lines}, "warning")' + worksheet[f"{cell}5"] = f'=COUNTIF({cell}13:{cell}{lines}, "failed")' + worksheet[f"{cell}6"] = f'=COUNTIF({cell}13:{cell}{lines}, "not_tested")' + worksheet[f"{cell}7"] = ( + f"=" + f'COUNTIF({cell}13:{cell}{lines}, "error")+' + f'COUNTIF({cell}13:{cell}{lines}, "unreachable")+' + f'COUNTIF({cell}13:{cell}{lines}, "untestable")+' f'COUNTIF({cell}13:{cell}{lines}, "not_testable")' - worksheet[f'{cell}8'] = f'=' \ - f'COUNTIF({cell}13:{cell}{lines}, "no_mx")+' \ - f'COUNTIF({cell}13:{cell}{lines}, "not_applicable")' + ) + worksheet[f"{cell}8"] = ( + f"=" f'COUNTIF({cell}13:{cell}{lines}, "no_mx")+' f'COUNTIF({cell}13:{cell}{lines}, "not_applicable")' + ) # Not applicable and not testable are subtracted from the total. # See https://github.com/internetstandards/Internet.nl-dashboard/issues/68 # Rounding's num digits is NOT the number of digits behind the comma, but the total number of digits. # todo: we should use the calculations in report.py. And there include the "missing" / empty stuff IF # that is missing. # IF( H1=0,0,ROUND( H2÷ H1, 4)) - worksheet[f'{cell}9'] = f'=IF({cell}1=0,0,ROUND({cell}2/{cell}1, 4))' - worksheet[f'{cell}9'].number_format = '0.00%' + worksheet[f"{cell}9"] = f"=IF({cell}1=0,0,ROUND({cell}2/{cell}1, 4))" + worksheet[f"{cell}9"].number_format = "0.00%" -def category_headers(protocol: str = 'dns_soa'): - sheet_headers: List[str] = ['', '', '', '', '', '', ''] +def category_headers(protocol: str = "dns_soa"): + sheet_headers: List[str] = ["", "", "", "", "", "", ""] for group in SANE_COLUMN_ORDER[protocol]: sheet_headers += [translate_field(group, translation_dictionary=po_file_as_dictionary)] for _ in range(len(SANE_COLUMN_ORDER[protocol][group]) - 1): - sheet_headers += [''] + sheet_headers += [""] # add empty thing after each group to make distinction per group clearer - sheet_headers += [''] + sheet_headers += [""] return sheet_headers -def subcategory_headers(protocol: str = 'dns_soa'): - sheet_headers = ['', '', '', '', '', '', ''] +def subcategory_headers(protocol: str = "dns_soa"): + sheet_headers = ["", "", "", "", "", "", ""] for group in SANE_COLUMN_ORDER[protocol]: sheet_headers += SANE_COLUMN_ORDER[protocol][group] # add empty thing after each group to make distinction per group clearer - sheet_headers += [''] + sheet_headers += [""] # translate them: - return [translate_field(FIELD_TO_CATEGORY_MAP.get(header, ''), - translation_dictionary=po_file_as_dictionary) for header in sheet_headers] + return [ + translate_field(FIELD_TO_CATEGORY_MAP.get(header, ""), translation_dictionary=po_file_as_dictionary) + for header in sheet_headers + ] -def headers(protocol: str = 'dns_soa'): - sheet_headers = ['List', 'Url', "Subdomain", "Domain", "Suffix", 'Tags', 'InStats'] +def headers(protocol: str = "dns_soa"): + sheet_headers = ["List", "Url", "Subdomain", "Domain", "Suffix", "Tags", "InStats"] for group in SANE_COLUMN_ORDER[protocol]: sheet_headers += SANE_COLUMN_ORDER[protocol][group] # add empty thing after each group to make distinction per group clearer - sheet_headers += [''] + sheet_headers += [""] # translate them: return [translate_field(header, translation_dictionary=po_file_as_dictionary) for header in sheet_headers] -def formula_row(function: str, protocol: str = 'dns_soa'): +def formula_row(function: str, protocol: str = "dns_soa"): data = [] my_headers = headers(protocol) @@ -342,7 +428,7 @@ def formula_row(function: str, protocol: str = 'dns_soa'): empty_headers = [] for i in range(total): - if my_headers[i] == '': + if my_headers[i] == "": empty_headers.append(i) # log.debug(empty_headers) @@ -351,9 +437,9 @@ def formula_row(function: str, protocol: str = 'dns_soa'): index = 0 for column_name in itertools.islice(iter_all_strings(), total): if index in empty_headers: - data.append('') + data.append("") else: - data.append(function % {'column_name': column_name}) + data.append(function % {"column_name": column_name}) index += 1 @@ -361,39 +447,40 @@ def formula_row(function: str, protocol: str = 'dns_soa'): def urllistreport_to_spreadsheet_data( - category_name: str, urls: List[Any], protocol: str = 'dns_soa', tags: Dict[str, List[str]] = None + category_name: str, urls: List[Any], protocol: str = "dns_soa", tags: Dict[str, List[str]] = None ): data = [] for url in urls: - extract = tldextract.extract(url['url']) + extract = tldextract.extract(url["url"]) - url_tags = ', '.join(tags.get(url['url'], [])) + url_tags = ", ".join(tags.get(url["url"], [])) - if len(url['endpoints']) == 1: + if len(url["endpoints"]) == 1: # we can just put the whole result in one, which is nicer to look at. - for endpoint in url['endpoints']: - if endpoint['protocol'] != protocol: + for endpoint in url["endpoints"]: + if endpoint["protocol"] != protocol: continue - keyed_ratings = endpoint['ratings_by_type'] + keyed_ratings = endpoint["ratings_by_type"] data.append( - [category_name, url['url'], extract.subdomain, extract.domain, extract.suffix, url_tags, 'TRUE'] + - keyed_values_as_boolean(keyed_ratings, protocol) + [category_name, url["url"], extract.subdomain, extract.domain, extract.suffix, url_tags, "TRUE"] + + keyed_values_as_boolean(keyed_ratings, protocol) ) else: - data.append([category_name, url['url'], extract.subdomain, - extract.domain, extract.suffix, url_tags, 'TRUE']) + data.append( + [category_name, url["url"], extract.subdomain, extract.domain, extract.suffix, url_tags, "TRUE"] + ) - for endpoint in url['endpoints']: - if endpoint['protocol'] != protocol: + for endpoint in url["endpoints"]: + if endpoint["protocol"] != protocol: continue - keyed_ratings = endpoint['ratings_by_type'] - data.append(['', '', '', '', '', '', ''] + keyed_values_as_boolean(keyed_ratings, protocol)) + keyed_ratings = endpoint["ratings_by_type"] + data.append(["", "", "", "", "", "", ""] + keyed_values_as_boolean(keyed_ratings, protocol)) # log.debug(data) return data -def keyed_values_as_boolean(keyed_ratings: Dict[str, Dict[str, Union[str, int]]], protocol: str = 'dns_soa'): +def keyed_values_as_boolean(keyed_ratings: Dict[str, Dict[str, Union[str, int]]], protocol: str = "dns_soa"): """ Keyed rating: {'internet_nl_mail_auth_dkim_exist': {'comply_or_explain_explained_on': '', @@ -427,30 +514,30 @@ def keyed_values_as_boolean(keyed_ratings: Dict[str, Dict[str, Union[str, int]]] for issue_name in SANE_COLUMN_ORDER[protocol][group]: values.append(some_value(issue_name, keyed_ratings)) - if issue_name == 'internet_nl_score_report': + if issue_name == "internet_nl_score_report": # add empty column values.append(" ") # add empty thing after each group to make distinction per group clearer # overall group already adds an extra value (url), so we don't need this. if group != "overall": - values += [''] + values += [""] return values def some_value(issue_name: str, keyed_ratings: Dict[str, Dict[str, Union[str, int]]]) -> Union[str, int]: - if issue_name == 'internet_nl_score': + if issue_name == "internet_nl_score": # Handle the special case of the score column. # explanation":"75 https://batch.internet.nl/mail/portaal.digimelding.nl/289480/", # Not steadily convertable to a percentage, so printing it as an integer instead. - score = keyed_ratings[issue_name]['internet_nl_score'] + score = keyed_ratings[issue_name]["internet_nl_score"] return score if score == "error" else int(score) - if issue_name == 'internet_nl_score_report': + if issue_name == "internet_nl_score_report": # fake column to give the column a title per #205, also makes the report more explicit. - return keyed_ratings['internet_nl_score']['internet_nl_url'] + return keyed_ratings["internet_nl_score"]["internet_nl_url"] # the issue name might not exist, the 'ok' value might not exist. In those cases replace it with a ? value = keyed_ratings.get(issue_name, None) @@ -458,8 +545,8 @@ def some_value(issue_name: str, keyed_ratings: Dict[str, Dict[str, Union[str, in return "?" # api v2, tls1.3 update - if value.get('test_result', False): - test_value = value.get('test_result', '?') + if value.get("test_result", False): + test_value = value.get("test_result", "?") # per 205, translate not_testable to untestable. This is cosmetic as the 'not_testable' is # everywhere in the software and is just renamed, and will probably be renamed a few times # more in the future. @@ -469,12 +556,12 @@ def some_value(issue_name: str, keyed_ratings: Dict[str, Dict[str, Union[str, in # unknown columns and data will be empty. if "simple_verdict" not in value: - return '' + return "" # backward compatible with api v1 reports unreachable mapping: Dict[str, str] = { - 'not_testable': 'untestable', - 'not_applicable': 'not_applicable', - 'error_in_test': 'error' + "not_testable": "untestable", + "not_applicable": "not_applicable", + "error_in_test": "error", } - return mapping.get(str(value['simple_verdict']), "?") + return mapping.get(str(value["simple_verdict"]), "?") diff --git a/dashboard/internet_nl_dashboard/logic/scan_monitor.py b/dashboard/internet_nl_dashboard/logic/scan_monitor.py index dc807f12..6c702ebb 100644 --- a/dashboard/internet_nl_dashboard/logic/scan_monitor.py +++ b/dashboard/internet_nl_dashboard/logic/scan_monitor.py @@ -8,49 +8,49 @@ def get_scan_monitor_data(account: Account) -> List[Dict[str, Union[str, int, bool, None]]]: - latest_30_scans = AccountInternetNLScan.objects.all().filter( - account=account, - urllist__is_deleted=False - ).order_by('-pk')[:30].select_related( - 'urllist', 'scan', 'report' - ).only( - 'id', - 'state', - 'started_on', - 'finished_on', - - 'scan__type', - 'scan__id', - 'scan__scan_id', - 'scan__last_state_check', - - 'urllist_id', - 'urllist__name', - - 'report__id' + latest_30_scans = ( + AccountInternetNLScan.objects.all() + .filter(account=account, urllist__is_deleted=False) + .order_by("-pk")[:30] + .select_related("urllist", "scan", "report") + .only( + "id", + "state", + "started_on", + "finished_on", + "scan__type", + "scan__id", + "scan__scan_id", + "scan__last_state_check", + "urllist_id", + "urllist__name", + "report__id", + ) ) # append all scans that are still running or any state except finished - unfinished_scans = AccountInternetNLScan.objects.all().filter( - account=account, - urllist__is_deleted=False, - ).exclude(state__in=['finished', 'cancelled']).order_by('-pk').select_related( - 'urllist', 'scan', 'report' - ).only( - 'id', - 'state', - 'started_on', - 'finished_on', - - 'scan__type', - 'scan__id', - 'scan__scan_id', - 'scan__last_state_check', - - 'urllist_id', - 'urllist__name', - - 'report__id' + unfinished_scans = ( + AccountInternetNLScan.objects.all() + .filter( + account=account, + urllist__is_deleted=False, + ) + .exclude(state__in=["finished", "cancelled"]) + .order_by("-pk") + .select_related("urllist", "scan", "report") + .only( + "id", + "state", + "started_on", + "finished_on", + "scan__type", + "scan__id", + "scan__scan_id", + "scan__last_state_check", + "urllist_id", + "urllist__name", + "report__id", + ) ) response = [] @@ -61,17 +61,13 @@ def get_scan_monitor_data(account: Account) -> List[Dict[str, Union[str, int, bo response.append(prepare_scan_data_for_display(scan)) # add all scans that have not finished - response.extend( - prepare_scan_data_for_display(scan) - for scan in unfinished_scans - if scan.id not in handled_scans - ) + response.extend(prepare_scan_data_for_display(scan) for scan in unfinished_scans if scan.id not in handled_scans) return response def prepare_scan_data_for_display(scan: Any): last_report_id = None - if scan.state == 'finished' and scan.report is not None: + if scan.state == "finished" and scan.report is not None: last_report_id = scan.report.id if scan.state == "finished" and scan.finished_on: @@ -84,41 +80,37 @@ def prepare_scan_data_for_display(scan: Any): runtime = moment - scan.started_on runtime_seconds = int(runtime.total_seconds() * 1000) - logs = AccountInternetNLScanLog.objects.all().filter(scan=scan).only('at_when', 'state').order_by('-at_when') - log_messages = [{'at_when': log.at_when, 'state': log.state} for log in logs] + logs = AccountInternetNLScanLog.objects.all().filter(scan=scan).only("at_when", "state").order_by("-at_when") + log_messages = [{"at_when": log.at_when, "state": log.state} for log in logs] data = { - 'id': scan.id, - 'state': scan.state, - + "id": scan.id, + "state": scan.state, # mask that there is a mail_dashboard variant. - 'type': "", - 'last_check': None, - - 'started': True, - 'started_on': scan.started_on, - 'finished': scan.finished, - 'finished_on': scan.finished_on, - 'status_url': "", - 'message': scan.state, - 'success': scan.finished, - 'list': "", - 'list_id': 0, - - 'runtime': runtime_seconds, - 'last_report_id': last_report_id, - - 'log': log_messages + "type": "", + "last_check": None, + "started": True, + "started_on": scan.started_on, + "finished": scan.finished, + "finished_on": scan.finished_on, + "status_url": "", + "message": scan.state, + "success": scan.finished, + "list": "", + "list_id": 0, + "runtime": runtime_seconds, + "last_report_id": last_report_id, + "log": log_messages, } if scan.scan: # mask that there is a mail_dashboard variant. - data['type'] = "web" if scan.scan.type == "web" else "all" if scan.scan.type == "all" else "mail" - data['last_check'] = scan.scan.last_state_check - data['status_url'] = f"{config.INTERNET_NL_API_URL}" - data['status_url'] += f"/requests/{scan.scan.scan_id}" + data["type"] = "web" if scan.scan.type == "web" else "all" if scan.scan.type == "all" else "mail" + data["last_check"] = scan.scan.last_state_check + data["status_url"] = f"{config.INTERNET_NL_API_URL}" + data["status_url"] += f"/requests/{scan.scan.scan_id}" if scan.urllist: - data['list'] = scan.urllist.name - data['list_id'] = scan.urllist.id + data["list"] = scan.urllist.name + data["list_id"] = scan.urllist.id return data diff --git a/dashboard/internet_nl_dashboard/logic/shared_report_lists.py b/dashboard/internet_nl_dashboard/logic/shared_report_lists.py index 9960e425..45e9e611 100644 --- a/dashboard/internet_nl_dashboard/logic/shared_report_lists.py +++ b/dashboard/internet_nl_dashboard/logic/shared_report_lists.py @@ -6,7 +6,7 @@ from dashboard.internet_nl_dashboard.models import UrlList, UrlListReport -def get_latest_report_id_from_list_and_type(urllist_id: int, report_type: str = '') -> Dict[str, str]: +def get_latest_report_id_from_list_and_type(urllist_id: int, report_type: str = "") -> Dict[str, str]: report = UrlListReport.objects.filter(urllist=urllist_id, is_publicly_shared=True) if report_type in {"web", "mail"}: @@ -15,9 +15,9 @@ def get_latest_report_id_from_list_and_type(urllist_id: int, report_type: str = found_report = report.last() return ( - {'latest_report_public_report_code': found_report.public_report_code} + {"latest_report_public_report_code": found_report.public_report_code} if found_report - else {'latest_report_public_report_code': ''} + else {"latest_report_public_report_code": ""} ) @@ -30,19 +30,27 @@ def get_publicly_shared_lists_per_account(account_id, urllist_id: Optional[int] log.debug(f"get_publicly_shared_lists_per_account account_id: {account_id}") report_prefetch = Prefetch( - 'urllistreport_set', - queryset=UrlListReport.objects.filter(is_publicly_shared=True).order_by('-id').only( - 'id', 'at_when', 'report_type', 'public_share_code', 'average_internet_nl_score', 'public_report_code', - 'total_urls', 'urllist_id' + "urllistreport_set", + queryset=UrlListReport.objects.filter(is_publicly_shared=True) + .order_by("-id") + .only( + "id", + "at_when", + "report_type", + "public_share_code", + "average_internet_nl_score", + "public_report_code", + "total_urls", + "urllist_id", ), - to_attr='reports' + to_attr="reports", ) - urllists = UrlList.objects.all().filter( - account=account_id, - is_deleted=False, - enable_report_sharing_page=True - ).prefetch_related(report_prefetch) + urllists = ( + UrlList.objects.all() + .filter(account=account_id, is_deleted=False, enable_report_sharing_page=True) + .prefetch_related(report_prefetch) + ) if urllist_id: urllists = urllists.filter(id=urllist_id) @@ -51,30 +59,30 @@ def get_publicly_shared_lists_per_account(account_id, urllist_id: Optional[int] return [ { - 'list': { - 'id': my_list.id, - 'name': my_list.name, - 'scan_type': my_list.scan_type, - 'automatically_share_new_reports': my_list.automatically_share_new_reports, - 'automated_scan_frequency': my_list.automated_scan_frequency + "list": { + "id": my_list.id, + "name": my_list.name, + "scan_type": my_list.scan_type, + "automatically_share_new_reports": my_list.automatically_share_new_reports, + "automated_scan_frequency": my_list.automated_scan_frequency, }, # for future use - 'account': { - 'public_name': '', + "account": { + "public_name": "", }, - 'number_of_reports': len(my_list.reports), - 'reports': [ + "number_of_reports": len(my_list.reports), + "reports": [ { - 'id': report.id, - 'at_when': report.at_when, - 'report_type': report.report_type, + "id": report.id, + "at_when": report.at_when, + "report_type": report.report_type, # don't send the code, only if there is password protection - 'has_public_share_code': bool(report.public_share_code), - 'average_internet_nl_score': report.average_internet_nl_score, - 'public_report_code': report.public_report_code, - 'total_urls': report.total_urls, + "has_public_share_code": bool(report.public_share_code), + "average_internet_nl_score": report.average_internet_nl_score, + "public_report_code": report.public_report_code, + "total_urls": report.total_urls, # be compatible with the frontpage report view: - 'urllist__name': my_list.name + "urllist__name": my_list.name, } for report in my_list.reports ], diff --git a/dashboard/internet_nl_dashboard/logic/spreadsheet.py b/dashboard/internet_nl_dashboard/logic/spreadsheet.py index 5b408934..e63e036c 100644 --- a/dashboard/internet_nl_dashboard/logic/spreadsheet.py +++ b/dashboard/internet_nl_dashboard/logic/spreadsheet.py @@ -34,39 +34,39 @@ from websecmap.celery import app from xlrd import XLRDError -from dashboard.internet_nl_dashboard.logic.domains import (clean_urls, retrieve_possible_urls_from_unfiltered_input, - save_urllist_content_by_id, save_urllist_content_by_name) +from dashboard.internet_nl_dashboard.logic.domains import ( + clean_urls, + retrieve_possible_urls_from_unfiltered_input, + save_urllist_content_by_id, + save_urllist_content_by_name, +) from dashboard.internet_nl_dashboard.models import Account, DashboardUser, UploadLog, UrlList log = logging.getLogger(__package__) SPREADSHEET_MIME_TYPES: List[str] = [ # XLSX - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", # XLS - 'application/vnd.ms-excel', - + "application/vnd.ms-excel", # ODS - 'application/vnd.oasis.opendocument.spreadsheet', - + "application/vnd.oasis.opendocument.spreadsheet", # magic thinks ms spreadsheets are application/octet-stream, which is basically everything... # XLSX / XLS - 'application/octet-stream', - + "application/octet-stream", # csv # https://stackoverflow.com/questions/7076042/what-mime-type-should-i-use-for-csv - 'text/plain', - 'text/x-csv', - 'text/csv', + "text/plain", + "text/x-csv", + "text/csv", ] -ALLOWED_SPREADSHEET_EXTENSIONS: List[str] = ['xlsx', 'xls', 'ods', 'csv'] +ALLOWED_SPREADSHEET_EXTENSIONS: List[str] = ["xlsx", "xls", "ods", "csv"] def is_file(file: str) -> bool: if not os.path.isfile(file): - log.debug('Not a valid file path.') + log.debug("Not a valid file path.") return False return True @@ -76,7 +76,7 @@ def save_file(myfile) -> str: # https://docs.djangoproject.com/en/2.1/ref/files/storage/ file_system_storage = FileSystemStorage(location=settings.MEDIA_ROOT) filename = file_system_storage.save(myfile.name, myfile) - file = settings.MEDIA_ROOT + '/' + filename + file = settings.MEDIA_ROOT + "/" + filename return file @@ -96,7 +96,7 @@ def is_valid_mimetype(file: str) -> bool: if mimetype in SPREADSHEET_MIME_TYPES: return True - log.debug(f'{mimetype} is not a valid mime type.') + log.debug(f"{mimetype} is not a valid mime type.") return False @@ -110,7 +110,7 @@ def is_valid_extension(file: str) -> bool: if file.split(".")[-1] in ALLOWED_SPREADSHEET_EXTENSIONS: return True - log.debug('Not a valid extension.') + log.debug("Not a valid extension.") return False @@ -119,8 +119,15 @@ def get_sheet(file: str) -> List: # perhaps some files have additional columns, which are not needed as part of the imprt # do not try to be smart about types, as everything in the uploads are strings # There is no alternative, it just takes 12 seconds for a 25k line file - sheet = p.get_sheet(file_name=file, name_columns_by_row=0, column_limit=3, skip_empty_rows=True, - auto_detect_int=False, auto_detect_datetime=False, auto_detect_float=False) + sheet = p.get_sheet( + file_name=file, + name_columns_by_row=0, + column_limit=3, + skip_empty_rows=True, + auto_detect_int=False, + auto_detect_datetime=False, + auto_detect_float=False, + ) except XLRDError: # xlrd.biffh.XLRDError: Unsupported format, or corrupt file: Expected BOF record; found b'thisfile' return [] @@ -165,12 +172,12 @@ def get_data(file: str) -> Dict[str, Dict[str, Dict[str, list]]]: continue # Data is parsed to python-like datatype. In this case we only expect strings and cast them as such. - found_categories = str(row[0]).lower().strip().split(',') - found_urls = str(row[1]).lower().strip().split(',') + found_categories = str(row[0]).lower().strip().split(",") + found_urls = str(row[1]).lower().strip().split(",") found_tags = [] # if there is no tag column: if len(row) > 2: - found_tags = str(row[2]).lower().strip().split(',') + found_tags = str(row[2]).lower().strip().split(",") for found_category in found_categories: found_category = found_category.strip() @@ -184,18 +191,18 @@ def get_data(file: str) -> Dict[str, Dict[str, Dict[str, list]]]: # use a list because a set is not json serializable: if found_url not in data[found_category]: - data[found_category][found_url] = {'tags': []} + data[found_category][found_url] = {"tags": []} for tag in found_tags: # not use a set here, while that would save a line of code to support serialization - if tag not in data[found_category][found_url]['tags']: - data[found_category][found_url]['tags'].append(tag) + if tag not in data[found_category][found_url]["tags"]: + data[found_category][found_url]["tags"].append(tag) # During editing, it might happen there are some 'left over' cells that are also added. # These left overs contain no urls. If they do, and something has been attempted to be added to # 'empty', it is discarded. We require urls to be in a list / category. - if '' in data: - data.pop('') + if "" in data: + data.pop("") log.debug("Freeeing resources") p.free_resources() @@ -205,14 +212,14 @@ def get_data(file: str) -> Dict[str, Dict[str, Dict[str, list]]]: def get_upload_history(account: Account) -> List: # limit amount of history due to frequent updates - uploads = UploadLog.objects.all().filter(user__account=account).order_by('-pk')[:30] + uploads = UploadLog.objects.all().filter(user__account=account).order_by("-pk")[:30] return [ { - 'original_filename': upload.original_filename, - 'message': upload.message, - 'upload_date': upload.upload_date, - 'filesize': upload.filesize, - 'percentage': upload.percentage, + "original_filename": upload.original_filename, + "message": upload.message, + "upload_date": upload.upload_date, + "filesize": upload.filesize, + "percentage": upload.percentage, } for upload in uploads ] @@ -240,33 +247,33 @@ def log_spreadsheet_upload(user: DashboardUser, file: str, status: str = "", mes # waterschappen_gucvVcD.xlsx or waterschappen.xlsx # https://regex101.com/, regex = "_[a-zA-Z0-9]{7,7}\." - internal_filename = file.split('/')[-1] + internal_filename = file.split("/")[-1] # the 8 random letters and numbers + possible file extension regex = r"_[a-zA-Z0-9]{7,7}\.[xlsodcsv]{3,4}" if re.findall(regex, internal_filename): - original_filename = re.sub(regex, "", internal_filename) + '.' + file.split('.')[-1] + original_filename = re.sub(regex, "", internal_filename) + "." + file.split(".")[-1] else: original_filename = internal_filename upload = { - 'user': user, - 'original_filename': original_filename[:250], - 'internal_filename': internal_filename[:250], - 'status': status[:250], - 'message': message[:250], - 'upload_date': datetime.now(timezone.utc), - 'filesize': os.path.getsize(file), + "user": user, + "original_filename": original_filename[:250], + "internal_filename": internal_filename[:250], + "status": status[:250], + "message": message[:250], + "upload_date": datetime.now(timezone.utc), + "filesize": os.path.getsize(file), } uploadlog = UploadLog(**upload) uploadlog.save() - upload['id'] = uploadlog.id + upload["id"] = uploadlog.id # Sprinkling an activity stream action. - action.send(user, verb='uploaded spreadsheet', target=uploadlog, public=False) + action.send(user, verb="uploaded spreadsheet", target=uploadlog, public=False) return upload @@ -274,40 +281,50 @@ def log_spreadsheet_upload(user: DashboardUser, file: str, status: str = "", mes # Do not accept partial imports. Or all, or nothing in a single transaction. # Depending on the speed this needs to become a task, as the wait will be too long. # transaction atomic cannot happen on very large lists it seems... -def save_data(account: Account, data: Dict[str, Dict[str, Dict[str, set]]], - uploadlog_id: int = None, pending_message: str = None): +def save_data( + account: Account, data: Dict[str, Dict[str, Dict[str, set]]], uploadlog_id: int = None, pending_message: str = None +): return { - urllist: save_urllist_content_by_name(account, urllist, data[urllist], uploadlog_id, pending_message) + urllist: save_urllist_content_by_name(account, urllist, data[urllist], uploadlog_id, pending_message) for urllist in data } def upload_error(message, user, file) -> Dict[str, Any]: - response: Dict[str, Any] = {'error': True, 'success': False, 'message': message, 'details': {}, 'status': 'error'} - log_spreadsheet_upload(user=user, file=file, status=response['status'], message=response['message']) + response: Dict[str, Any] = {"error": True, "success": False, "message": message, "details": {}, "status": "error"} + log_spreadsheet_upload(user=user, file=file, status=response["status"], message=response["message"]) return response def inspect_upload_file(user: DashboardUser, file: str) -> Optional[Dict[str, Any]]: # use more verbose validation, to give better feedback. if not is_file(file): - return upload_error("Uploaded file was not found. I might be not stored due to a full disk or or has been " - "stored in the wrong location, or has been deleted automatically by a background " - "process like a virus scanner.", user, file) + return upload_error( + "Uploaded file was not found. I might be not stored due to a full disk or or has been " + "stored in the wrong location, or has been deleted automatically by a background " + "process like a virus scanner.", + user, + file, + ) if not is_valid_extension(file): - return upload_error("File does not have a valid extension. " - f"Allowed extensions are: {','.join(ALLOWED_SPREADSHEET_EXTENSIONS)}.", user, file) + return upload_error( + "File does not have a valid extension. " + f"Allowed extensions are: {','.join(ALLOWED_SPREADSHEET_EXTENSIONS)}.", + user, + file, + ) if not is_valid_mimetype(file): - return upload_error("The content of the file could not be established. It might not be a spreadsheet file.", - user, file) + return upload_error( + "The content of the file could not be established. It might not be a spreadsheet file.", user, file + ) return None def get_data_from_spreadsheet( - user: DashboardUser, file: str + user: DashboardUser, file: str ) -> Union[Tuple[Dict[str, Dict[str, Dict[str, list]]], int], Tuple[Dict[str, Any], str]]: has_errors = inspect_upload_file(user, file) if has_errors: @@ -317,8 +334,15 @@ def get_data_from_spreadsheet( log.debug("Getting data from file") domain_lists: Dict[str, Dict[str, Dict[str, list]]] = get_data(file) if not domain_lists: - return upload_error("The uploaded file contained no data. This might happen when the file is not in the " - "correct format. Are you sure it is a correct spreadsheet file?", user, file), "error" + return ( + upload_error( + "The uploaded file contained no data. This might happen when the file is not in the " + "correct format. Are you sure it is a correct spreadsheet file?", + user, + file, + ), + "error", + ) # sanity check on data length and number of lists (this does not prevent anyone from trying to upload the same # file over and over again), it's just a usability feature against mistakes. @@ -326,14 +350,26 @@ def get_data_from_spreadsheet( number_of_urls = sum(len(urls) for _, urls in domain_lists.items()) if len(domain_lists) > config.DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET: - return upload_error(f"The maximum number of new lists is {config.DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET}. " - "The uploaded spreadsheet contains more than this limit. Try again in smaller batches.", - user, file), "error" + return ( + upload_error( + f"The maximum number of new lists is {config.DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET}. " + "The uploaded spreadsheet contains more than this limit. Try again in smaller batches.", + user, + file, + ), + "error", + ) if number_of_urls > config.DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET: - return upload_error(f"The maximum number of new urls is {config.DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET}." - "The uploaded spreadsheet contains more than this limit. Try again in smaller batches.", - user, file), "error" + return ( + upload_error( + f"The maximum number of new urls is {config.DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET}." + "The uploaded spreadsheet contains more than this limit. Try again in smaller batches.", + user, + file, + ), + "error", + ) log.debug("Checking url data per list inside the spreadsheet") for urllist, urls in domain_lists.items(): @@ -341,16 +377,22 @@ def get_data_from_spreadsheet( possible_urls, _ = retrieve_possible_urls_from_unfiltered_input(", ".join(urls)) url_check = clean_urls(possible_urls) - if url_check['incorrect']: - return upload_error("This spreadsheet contains urls that are not in the correct format. Please correct " - "them and try again. The first list that contains an error is " - f"{urllist} with the url(s) {', '.join(url_check['incorrect'])}", - user, file), "error" + if url_check["incorrect"]: + return ( + upload_error( + "This spreadsheet contains urls that are not in the correct format. Please correct " + "them and try again. The first list that contains an error is " + f"{urllist} with the url(s) {', '.join(url_check['incorrect'])}", + user, + file, + ), + "error", + ) return domain_lists, number_of_urls -@app.task(queue='storage', ignore_result=True) +@app.task(queue="storage", ignore_result=True) def import_step_2(user: int, file: str, uploadlog_id: int): user = DashboardUser.objects.all().filter(id=user).first() @@ -360,11 +402,11 @@ def import_step_2(user: int, file: str, uploadlog_id: int): domain_lists, number_of_urls = get_data_from_spreadsheet(user, file) if number_of_urls == "error": - update_spreadsheet_upload(uploadlog_id, status="error", message=domain_lists['message']) + update_spreadsheet_upload(uploadlog_id, status="error", message=domain_lists["message"]) return step_2_message = f"[2/3] Saving {len(domain_lists)} lists and {number_of_urls} urls. This can take a while." - update_spreadsheet_upload(uploadlog_id, status='[2/3] Processing', message=step_2_message) + update_spreadsheet_upload(uploadlog_id, status="[2/3] Processing", message=step_2_message) # File system full, database full. details = save_data(user.account, domain_lists, uploadlog_id=uploadlog_id, pending_message=step_2_message) @@ -381,13 +423,17 @@ def import_step_2(user: int, file: str, uploadlog_id: int): details_str += f"{urllist}: new: {detail['added_to_list']}, existing: {detail['already_in_list']}; " if not error_set: - message = "[3/3] Spreadsheet uploaded successfully. " \ - f"Added {len(domain_lists)} lists and {number_of_urls} urls. Details: {details_str}" + message = ( + "[3/3] Spreadsheet uploaded successfully. " + f"Added {len(domain_lists)} lists and {number_of_urls} urls. Details: {details_str}" + ) else: - message = "[3/3] Spreadsheet upload failed. " \ - f"Might not have added {len(domain_lists)} lists and {number_of_urls} urls. Details: {details_str}" + message = ( + "[3/3] Spreadsheet upload failed. " + f"Might not have added {len(domain_lists)} lists and {number_of_urls} urls. Details: {details_str}" + ) - update_spreadsheet_upload(uploadlog_id, status='[3/3] Finished', message=message) + update_spreadsheet_upload(uploadlog_id, status="[3/3] Finished", message=message) return @@ -402,9 +448,9 @@ def upload_domain_spreadsheet_to_list(account: Account, user: DashboardUser, url urllist = UrlList.objects.all().filter(id=urllist_id, account=account).first() if not urllist: - return {'error': True, 'success': False, 'message': 'list_does_not_exist', 'details': '', 'status': 'error'} + return {"error": True, "success": False, "message": "list_does_not_exist", "details": "", "status": "error"} - log_spreadsheet_upload(user=user, file=file, status='pending', message="Uploading and processing spreadsheet...") + log_spreadsheet_upload(user=user, file=file, status="pending", message="Uploading and processing spreadsheet...") # todo: put this in a separate task, to make sure the upload is fast enough and give this feedback to the user. # the spreadsheet content is leading, this means that anything in the current list, including tags, will @@ -415,16 +461,18 @@ def upload_domain_spreadsheet_to_list(account: Account, user: DashboardUser, url urllist.urls.clear() # domain lists: {'10ksites': {'1.site.nl': {'tags': ['1']}, 'asdasd': {'tags': ['2']}}} - result = {'added_to_list': 0, 'already_in_list': 0} + result = {"added_to_list": 0, "already_in_list": 0} for _, domain_data in domain_lists.items(): result = save_urllist_content_by_id(account, urllist.id, domain_data) # too many domains if "error" in result: - log_spreadsheet_upload(user=user, file=file, status='success', message="Too many domains in list.") + log_spreadsheet_upload(user=user, file=file, status="success", message="Too many domains in list.") return result details_str = f"{urllist.name}: new: {result['added_to_list']}, existing: {result['already_in_list']}; " - message = "Spreadsheet uploaded successfully. " \ - f"Added {len(domain_lists)} lists and {number_of_urls} urls. Details: {details_str}" - log_spreadsheet_upload(user=user, file=file, status='success', message=message) - return {'error': False, 'success': True, 'message': message, 'details': details_str, 'status': 'success'} + message = ( + "Spreadsheet uploaded successfully. " + f"Added {len(domain_lists)} lists and {number_of_urls} urls. Details: {details_str}" + ) + log_spreadsheet_upload(user=user, file=file, status="success", message=message) + return {"error": False, "success": True, "message": message, "details": details_str, "status": "success"} diff --git a/dashboard/internet_nl_dashboard/logic/suggestions.py b/dashboard/internet_nl_dashboard/logic/suggestions.py new file mode 100644 index 00000000..c65e928b --- /dev/null +++ b/dashboard/internet_nl_dashboard/logic/suggestions.py @@ -0,0 +1,26 @@ +import requests +import tldextract +from constance import config + +from dashboard.internet_nl_dashboard.logic.domains import log + + +def suggest_subdomains(domain: str, period: int = 370): + extract = tldextract.extract(domain) + + # ip address or garbage + if not extract.domain or not extract.suffix: + return [] + + # call SUBDOMAIN_SUGGESTION_SERVER_ADDRESS + response = requests.get( + config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS, + params={"domain": extract.domain, "suffix": extract.suffix, "period": period}, + timeout=10, + ) + + if response.status_code != 200: + log.error("Failed to retrieve subdomain suggestions from %s.", config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS) + return [] + + return response.json() diff --git a/dashboard/internet_nl_dashboard/logic/tags.py b/dashboard/internet_nl_dashboard/logic/tags.py index c7a28a2e..78a62b8a 100644 --- a/dashboard/internet_nl_dashboard/logic/tags.py +++ b/dashboard/internet_nl_dashboard/logic/tags.py @@ -38,7 +38,11 @@ def remove_tag(account: Account, url_ids: List[int], urllist_id: int, tag: str) def tags_in_urllist(account: Account, urllist_id: int) -> List[str]: - return list(sorted(Tag.objects.all().filter( - taggedurlinurllist__urllist=urllist_id, - taggedurlinurllist__urllist__account=account - ).values_list('name', flat=True).distinct())) + return list( + sorted( + Tag.objects.all() + .filter(taggedurlinurllist__urllist=urllist_id, taggedurlinurllist__urllist__account=account) + .values_list("name", flat=True) + .distinct() + ) + ) diff --git a/dashboard/internet_nl_dashboard/logic/tests/test_tags.py b/dashboard/internet_nl_dashboard/logic/tests/test_tags.py index 5123ed21..736cec41 100644 --- a/dashboard/internet_nl_dashboard/logic/tests/test_tags.py +++ b/dashboard/internet_nl_dashboard/logic/tests/test_tags.py @@ -22,10 +22,10 @@ def test_tags(db): # pylint: disable=invalid-name, unused-argument validate(url.id, my_list.id, []) add_tag(account, [url.id], my_list.id, "test_tag") - validate(url.id, my_list.id, ['test_tag']) + validate(url.id, my_list.id, ["test_tag"]) add_tag(account, [url.id], my_list.id, "test_tag_2") add_tag(account, [url.id], my_list.id, "test_tag_3") - assert tags_in_urllist(account, my_list.id) == ['test_tag', 'test_tag_2', 'test_tag_3'] + assert tags_in_urllist(account, my_list.id) == ["test_tag", "test_tag_2", "test_tag_3"] remove_tag(account, [url.id], my_list.id, "test_tag") remove_tag(account, [url.id], my_list.id, "test_tag_2") remove_tag(account, [url.id], my_list.id, "test_tag_3") diff --git a/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py b/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py index 52fcec76..f5ce8a37 100644 --- a/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py +++ b/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py @@ -9,36 +9,83 @@ from websecmap.celery import Task, app from websecmap.organizations.models import Url from websecmap.reporting.diskreport import store_report -from websecmap.reporting.report import (add_statistics_to_calculation, aggegrate_url_rating_scores, - get_latest_urlratings_fast, relevant_urls_at_timepoint, - remove_issues_from_calculation, statistics_over_url_calculation) - -from dashboard.internet_nl_dashboard.logic import (MAIL_AUTH_FIELDS, MAIL_CATEGORIES, # pylint: disable=duplicate-code - MAIL_DNSSEC_FIELDS, MAIL_IPV6_FIELDS, MAIL_LEGACY_FIELDS, - MAIL_RPKI_FIELDS, MAIL_TLS_CERTIFICATE_FIELDS, MAIL_TLS_DANE_FIELDS, - MAIL_TLS_TLS_FIELDS, WEB_APPSECPRIV_CATEGORY, WEB_APPSECPRIV_FIELDS, - WEB_DNSSEC_CATEGORY, WEB_DNSSEC_FIELDS, WEB_IPV6_CATEGORY, - WEB_IPV6_FIELDS, WEB_LEGACY_CATEGORY, WEB_LEGACY_FIELDS, - WEB_RPKI_CATEGORY, WEB_RPKI_FIELDS, WEB_TLS_CATEGORY, - WEB_TLS_CERTIFICATE_FIELDS, WEB_TLS_DANE_FIELDS, WEB_TLS_HTTP_FIELDS, - WEB_TLS_TLS_FIELDS) +from websecmap.reporting.report import ( + add_statistics_to_calculation, + aggegrate_url_rating_scores, + get_latest_urlratings_fast, + relevant_urls_at_timepoint, + remove_issues_from_calculation, + statistics_over_url_calculation, +) + +from dashboard.internet_nl_dashboard.logic import MAIL_CATEGORIES # pylint: disable=duplicate-code +from dashboard.internet_nl_dashboard.logic import ( + MAIL_AUTH_FIELDS, + MAIL_DNSSEC_FIELDS, + MAIL_IPV6_FIELDS, + MAIL_LEGACY_FIELDS, + MAIL_RPKI_FIELDS, + MAIL_TLS_CERTIFICATE_FIELDS, + MAIL_TLS_DANE_FIELDS, + MAIL_TLS_TLS_FIELDS, + WEB_APPSECPRIV_CATEGORY, + WEB_APPSECPRIV_FIELDS, + WEB_DNSSEC_CATEGORY, + WEB_DNSSEC_FIELDS, + WEB_IPV6_CATEGORY, + WEB_IPV6_FIELDS, + WEB_LEGACY_CATEGORY, + WEB_LEGACY_FIELDS, + WEB_RPKI_CATEGORY, + WEB_RPKI_FIELDS, + WEB_TLS_CATEGORY, + WEB_TLS_CERTIFICATE_FIELDS, + WEB_TLS_DANE_FIELDS, + WEB_TLS_HTTP_FIELDS, + WEB_TLS_TLS_FIELDS, +) from dashboard.internet_nl_dashboard.models import AccountInternetNLScan, UrlList, UrlListReport log = logging.getLogger(__package__) urllist_report_content = { - 'mail': ['internet_nl_mail_dashboard_overall_score'] + - MAIL_CATEGORIES + MAIL_IPV6_FIELDS + MAIL_DNSSEC_FIELDS + MAIL_TLS_CERTIFICATE_FIELDS + - MAIL_TLS_TLS_FIELDS + MAIL_TLS_DANE_FIELDS + MAIL_AUTH_FIELDS + MAIL_RPKI_FIELDS + MAIL_LEGACY_FIELDS, - - 'mail_dashboard': ['internet_nl_mail_dashboard_overall_score'] + - MAIL_CATEGORIES + MAIL_IPV6_FIELDS + MAIL_DNSSEC_FIELDS + MAIL_TLS_CERTIFICATE_FIELDS + - MAIL_TLS_TLS_FIELDS + MAIL_TLS_DANE_FIELDS + MAIL_AUTH_FIELDS + MAIL_RPKI_FIELDS + MAIL_LEGACY_FIELDS, - - 'web': ['internet_nl_web_overall_score'] + WEB_IPV6_CATEGORY + WEB_IPV6_FIELDS + WEB_DNSSEC_CATEGORY + - WEB_DNSSEC_FIELDS + WEB_TLS_CATEGORY + WEB_TLS_HTTP_FIELDS + WEB_TLS_TLS_FIELDS + - WEB_TLS_CERTIFICATE_FIELDS + WEB_TLS_DANE_FIELDS + WEB_APPSECPRIV_CATEGORY + WEB_APPSECPRIV_FIELDS + - WEB_RPKI_CATEGORY + WEB_RPKI_FIELDS + WEB_LEGACY_FIELDS + WEB_LEGACY_FIELDS + WEB_LEGACY_CATEGORY + "mail": ["internet_nl_mail_dashboard_overall_score"] + + MAIL_CATEGORIES + + MAIL_IPV6_FIELDS + + MAIL_DNSSEC_FIELDS + + MAIL_TLS_CERTIFICATE_FIELDS + + MAIL_TLS_TLS_FIELDS + + MAIL_TLS_DANE_FIELDS + + MAIL_AUTH_FIELDS + + MAIL_RPKI_FIELDS + + MAIL_LEGACY_FIELDS, + "mail_dashboard": ["internet_nl_mail_dashboard_overall_score"] + + MAIL_CATEGORIES + + MAIL_IPV6_FIELDS + + MAIL_DNSSEC_FIELDS + + MAIL_TLS_CERTIFICATE_FIELDS + + MAIL_TLS_TLS_FIELDS + + MAIL_TLS_DANE_FIELDS + + MAIL_AUTH_FIELDS + + MAIL_RPKI_FIELDS + + MAIL_LEGACY_FIELDS, + "web": ["internet_nl_web_overall_score"] + + WEB_IPV6_CATEGORY + + WEB_IPV6_FIELDS + + WEB_DNSSEC_CATEGORY + + WEB_DNSSEC_FIELDS + + WEB_TLS_CATEGORY + + WEB_TLS_HTTP_FIELDS + + WEB_TLS_TLS_FIELDS + + WEB_TLS_CERTIFICATE_FIELDS + + WEB_TLS_DANE_FIELDS + + WEB_APPSECPRIV_CATEGORY + + WEB_APPSECPRIV_FIELDS + + WEB_RPKI_CATEGORY + + WEB_RPKI_FIELDS + + WEB_LEGACY_FIELDS + + WEB_LEGACY_FIELDS + + WEB_LEGACY_CATEGORY, } @@ -67,7 +114,7 @@ def compose_task(**kwargs) -> Task: return group(tasks) -@app.task(queue='storage') +@app.task(queue="storage") def create_dashboard_report(scan_id: int): """ Simplified (and perhaps straightforward) version of rate_urllists_now, which returns a report id. @@ -79,7 +126,7 @@ def create_dashboard_report(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: - log.warning(f'Trying to creating_report with unknown scan: {scan_id}.') + log.warning(f"Trying to creating_report with unknown scan: {scan_id}.") return [] log.debug(f"Creating dashboard report for urllist {scan.urllist}.") @@ -99,7 +146,7 @@ def create_dashboard_report(scan_id: int): return rate_urllist_on_moment(scan.urllist, when=now, prevent_duplicates=False, scan_type=scan_type) -@app.task(queue='storage') +@app.task(queue="storage") def create_dashboard_report_at(urllist: UrlList, at_when: datetime, scan_type: str = "web"): """ Simplified (and perhaps straightforward) version of rate_urllists_now, which returns a report id. @@ -112,17 +159,17 @@ def create_dashboard_report_at(urllist: UrlList, at_when: datetime, scan_type: s return rate_urllist_on_moment(urllist, when=at_when, prevent_duplicates=False, scan_type=scan_type) -@app.task(queue='storage', ignore_result=True) +@app.task(queue="storage", ignore_result=True) def rate_urllists_now(urllists: List[UrlList], prevent_duplicates: bool = True, scan_type: str = "web"): for urllist in urllists: now = datetime.now(timezone.utc) rate_urllist_on_moment(urllist, now, prevent_duplicates, scan_type) -@app.task(queue='storage') +@app.task(queue="storage") def rate_urllist_on_moment( - urllist: UrlList, when: Optional[datetime] = None, prevent_duplicates: bool = True, - scan_type: str = "web") -> int: + urllist: UrlList, when: Optional[datetime] = None, prevent_duplicates: bool = True, scan_type: str = "web" +) -> int: """ :param urllist: :param when: A moment in time of which data should be aggregated @@ -142,16 +189,16 @@ def rate_urllist_on_moment( return int(existing_report.id) urls = relevant_urls_at_timepoint_urllist(urllist=urllist, when=when) - log.debug(f'Found {len(urls)} to be relevant at this moment.') + log.debug(f"Found {len(urls)} to be relevant at this moment.") calculation = create_calculation_on_urls(urls, when, scan_type=scan_type) try: - last = UrlListReport.objects.filter(urllist=urllist, at_when__lte=when).latest('at_when') + last = UrlListReport.objects.filter(urllist=urllist, at_when__lte=when).latest("at_when") except UrlListReport.DoesNotExist: last = UrlListReport() # create a dummy one for comparison - calculation['name'] = urllist.name + calculation["name"] = urllist.name if prevent_duplicates: if not DeepDiff(last.calculation, calculation, ignore_order=True, report_repetition=True): @@ -163,8 +210,8 @@ def rate_urllist_on_moment( # remove urls and name from scores object, so it can be used as initialization parameters (saves lines) # this is by reference, meaning that the calculation will be affected if we don't work on a clone. init_scores = deepcopy(calculation) - del init_scores['name'] - del init_scores['urls'] + del init_scores["name"] + del init_scores["urls"] external_scan_type = {"web": "web", "mail": "mail", "mail_dashboard": "mail"} report = UrlListReport(**init_scores) @@ -187,7 +234,8 @@ def create_calculation_on_urls(urls: List[Url], when: datetime, scan_type: str) # and only the endpoint types for urlrating in all_url_ratings: calculation: Dict[str, Any] = remove_issues_from_calculation( - urlrating.calculation, urllist_report_content[scan_type]) + urlrating.calculation, urllist_report_content[scan_type] + ) # Some endpoint types use the same ratings, such as dns_soa and dns_mx... This means that not # all endpoints will be removed for internet.nl. We need the following endpoints per scan: @@ -207,9 +255,10 @@ def create_calculation_on_urls(urls: List[Url], when: datetime, scan_type: str) def only_include_endpoint_protocols(calculation, only_include_endpoint_types: List[str]): - new_endpoints = [endpoint - for endpoint in calculation['endpoints'] if endpoint['protocol'] in only_include_endpoint_types] - calculation['endpoints'] = new_endpoints + new_endpoints = [ + endpoint for endpoint in calculation["endpoints"] if endpoint["protocol"] in only_include_endpoint_types + ] + calculation["endpoints"] = new_endpoints return calculation @@ -223,15 +272,15 @@ def sum_internet_nl_scores_over_rating(url_ratings: Dict[str, Any]) -> float: score = 0 number_of_scores = 0 - score_fields = ['internet_nl_mail_dashboard_overall_score', 'internet_nl_web_overall_score'] + score_fields = ["internet_nl_mail_dashboard_overall_score", "internet_nl_web_overall_score"] - for url in url_ratings.get('urls', []): - for endpoint in url.get('endpoints', []): - for rating in endpoint.get('ratings', []): - if rating.get('type', "") in score_fields: + for url in url_ratings.get("urls", []): + for endpoint in url.get("endpoints", []): + for rating in endpoint.get("ratings", []): + if rating.get("type", "") in score_fields: # explanation":"75 https://batch.internet.nl/mail/portaal.digimelding.nl/289480/", log.debug(f"Explanation: {rating['explanation']}") - value = rating['explanation'].split(" ") + value = rating["explanation"].split(" ") # in case the internet.nl api fails for a domain, all scanned values are set to error. # this value is ignored in the average, not influencing the average with a 0 or 100. diff --git a/dashboard/internet_nl_dashboard/logic/usage.py b/dashboard/internet_nl_dashboard/logic/usage.py index ebc0b6e9..ca48d468 100644 --- a/dashboard/internet_nl_dashboard/logic/usage.py +++ b/dashboard/internet_nl_dashboard/logic/usage.py @@ -53,53 +53,53 @@ def dashboard_months(year) -> range: def usage_metrics(): return { - 'users': { - 'total': User.objects.all().count(), - 'logged_in_the_past_1_days': user_logged_in_past_n_days(1), - 'logged_in_the_past_2_days': user_logged_in_past_n_days(2), - 'logged_in_the_past_3_days': user_logged_in_past_n_days(3), - 'logged_in_the_past_5_days': user_logged_in_past_n_days(5), - 'logged_in_the_past_7_days': user_logged_in_past_n_days(7), - 'logged_in_the_past_14_days': user_logged_in_past_n_days(14), - 'logged_in_the_past_21_days': user_logged_in_past_n_days(21), - 'logged_in_the_past_30_days': user_logged_in_past_n_days(30), - 'logged_in_the_past_60_days': user_logged_in_past_n_days(60), - 'logged_in_the_past_90_days': user_logged_in_past_n_days(90), - 'logged_in_the_past_120_days': user_logged_in_past_n_days(120), - 'logged_in_the_past_150_days': user_logged_in_past_n_days(150), - 'logged_in_the_past_180_days': user_logged_in_past_n_days(180), - 'logged_in_the_past_210_days': user_logged_in_past_n_days(210), - 'logged_in_the_past_240_days': user_logged_in_past_n_days(240), - 'logged_in_the_past_270_days': user_logged_in_past_n_days(270), - 'logged_in_the_past_300_days': user_logged_in_past_n_days(300), + "users": { + "total": User.objects.all().count(), + "logged_in_the_past_1_days": user_logged_in_past_n_days(1), + "logged_in_the_past_2_days": user_logged_in_past_n_days(2), + "logged_in_the_past_3_days": user_logged_in_past_n_days(3), + "logged_in_the_past_5_days": user_logged_in_past_n_days(5), + "logged_in_the_past_7_days": user_logged_in_past_n_days(7), + "logged_in_the_past_14_days": user_logged_in_past_n_days(14), + "logged_in_the_past_21_days": user_logged_in_past_n_days(21), + "logged_in_the_past_30_days": user_logged_in_past_n_days(30), + "logged_in_the_past_60_days": user_logged_in_past_n_days(60), + "logged_in_the_past_90_days": user_logged_in_past_n_days(90), + "logged_in_the_past_120_days": user_logged_in_past_n_days(120), + "logged_in_the_past_150_days": user_logged_in_past_n_days(150), + "logged_in_the_past_180_days": user_logged_in_past_n_days(180), + "logged_in_the_past_210_days": user_logged_in_past_n_days(210), + "logged_in_the_past_240_days": user_logged_in_past_n_days(240), + "logged_in_the_past_270_days": user_logged_in_past_n_days(270), + "logged_in_the_past_300_days": user_logged_in_past_n_days(300), }, - 'scans': { - 'total': AccountInternetNLScan.objects.all().count(), - 'per_year': abstract_per_year(AccountInternetNLScan.objects.all(), 'started_on'), - 'per_month': abstract_per_month(AccountInternetNLScan.objects.all(), 'started_on'), + "scans": { + "total": AccountInternetNLScan.objects.all().count(), + "per_year": abstract_per_year(AccountInternetNLScan.objects.all(), "started_on"), + "per_month": abstract_per_month(AccountInternetNLScan.objects.all(), "started_on"), }, - 'lists': { - 'total': UrlList.objects.all().count(), + "lists": { + "total": UrlList.objects.all().count(), }, - 'domains': { - 'total': Url.objects.all().count(), + "domains": { + "total": Url.objects.all().count(), # Expect this to lower over time, as these are only new unique domains. - 'per_year': abstract_per_year(Url.objects.all(), 'created_on'), - 'per_month': abstract_per_month(Url.objects.all(), 'created_on'), + "per_year": abstract_per_year(Url.objects.all(), "created_on"), + "per_month": abstract_per_month(Url.objects.all(), "created_on"), }, - 'metrics': { - 'total': EndpointGenericScan.objects.all().count(), + "metrics": { + "total": EndpointGenericScan.objects.all().count(), # Expect this to lower over time too, as the first time new metrics are measured, then only updates - 'last_scan_moment': { - 'per_year': abstract_per_year(EndpointGenericScan.objects.all(), 'last_scan_moment'), - 'per_month': abstract_per_month(EndpointGenericScan.objects.all(), 'last_scan_moment'), + "last_scan_moment": { + "per_year": abstract_per_year(EndpointGenericScan.objects.all(), "last_scan_moment"), + "per_month": abstract_per_month(EndpointGenericScan.objects.all(), "last_scan_moment"), + }, + "rating_determined_on": { + "per_year": abstract_per_year(EndpointGenericScan.objects.all(), "rating_determined_on"), + "per_month": abstract_per_month(EndpointGenericScan.objects.all(), "rating_determined_on"), }, - 'rating_determined_on': { - 'per_year': abstract_per_year(EndpointGenericScan.objects.all(), 'rating_determined_on'), - 'per_month': abstract_per_month(EndpointGenericScan.objects.all(), 'rating_determined_on'), - } }, - 'actions': abstract_action_total() + "actions": abstract_action_total(), } @@ -131,12 +131,12 @@ def abstract_per_month(query, datetime_field): def abstract_action_total(): - verbs = list(set(Action.objects.all().values_list('verb', flat=True).order_by('verb'))) + verbs = list(set(Action.objects.all().values_list("verb", flat=True).order_by("verb"))) stats: Dict[Any, Any] = defaultdict(dict) for verb in verbs: stats[verb] = { - 'total': Action.objects.all().filter(verb=verb).count(), - 'per_year': abstract_per_year(Action.objects.all().filter(verb=verb), 'timestamp'), - 'per_month': abstract_per_month(Action.objects.all().filter(verb=verb), 'timestamp') + "total": Action.objects.all().filter(verb=verb).count(), + "per_year": abstract_per_year(Action.objects.all().filter(verb=verb), "timestamp"), + "per_month": abstract_per_month(Action.objects.all().filter(verb=verb), "timestamp"), } return stats diff --git a/dashboard/internet_nl_dashboard/logic/user.py b/dashboard/internet_nl_dashboard/logic/user.py index 43b7fcb1..6a6f72a1 100644 --- a/dashboard/internet_nl_dashboard/logic/user.py +++ b/dashboard/internet_nl_dashboard/logic/user.py @@ -21,15 +21,15 @@ def get_user_settings(dashboarduser_id): return {} data = { - 'first_name': user.first_name, - 'last_name': user.last_name, - 'date_joined': None, - 'last_login': None, - 'account_id': user.dashboarduser.account.id, - 'account_name': user.dashboarduser.account.name, - 'mail_preferred_mail_address': user.dashboarduser.mail_preferred_mail_address, - 'mail_preferred_language': user.dashboarduser.mail_preferred_language.code.lower(), - 'mail_send_mail_after_scan_finished': user.dashboarduser.mail_send_mail_after_scan_finished + "first_name": user.first_name, + "last_name": user.last_name, + "date_joined": None, + "last_login": None, + "account_id": user.dashboarduser.account.id, + "account_name": user.dashboarduser.account.name, + "mail_preferred_mail_address": user.dashboarduser.mail_preferred_mail_address, + "mail_preferred_language": user.dashboarduser.mail_preferred_language.code.lower(), + "mail_send_mail_after_scan_finished": user.dashboarduser.mail_send_mail_after_scan_finished, } return data @@ -51,8 +51,13 @@ def save_user_settings(dashboarduser_id, data): - mail_send_mail_after_scan_finished """ - expected_keys = ['first_name', 'last_name', 'mail_preferred_mail_address', 'mail_preferred_language', - 'mail_send_mail_after_scan_finished'] + expected_keys = [ + "first_name", + "last_name", + "mail_preferred_mail_address", + "mail_preferred_language", + "mail_send_mail_after_scan_finished", + ] if not keys_are_present_in_object(expected_keys, data): return operation_response(error=True, message="save_user_settings_error_incomplete_data") @@ -60,24 +65,24 @@ def save_user_settings(dashboarduser_id, data): # this breaks the 'form-style' logic. Perhaps we'd move to django rest framework to optimize this. # Otoh there's no time for that now. Assuming the form is entered well this is no direct issue now. # I'm currently slowly developing a framework. But as long as it's just a few forms and pages it's fine. - if data['mail_preferred_language'] not in [language_code for language_code, name in LANGUAGES]: + if data["mail_preferred_language"] not in [language_code for language_code, name in LANGUAGES]: return operation_response(error=True, message="save_user_settings_error_form_unsupported_language") # email is allowed to be empty: - if data['mail_preferred_mail_address']: + if data["mail_preferred_mail_address"]: email_field = forms.EmailField() try: - email_field.clean(data['mail_preferred_mail_address']) + email_field.clean(data["mail_preferred_mail_address"]) except ValidationError: return operation_response(error=True, message="save_user_settings_error_form_incorrect_mail_address") - user.first_name = data['first_name'] - user.last_name = data['last_name'] + user.first_name = data["first_name"] + user.last_name = data["last_name"] user.save() - user.dashboarduser.mail_preferred_mail_address = data['mail_preferred_mail_address'] - user.dashboarduser.mail_preferred_language = data['mail_preferred_language'] - user.dashboarduser.mail_send_mail_after_scan_finished = data['mail_send_mail_after_scan_finished'] + user.dashboarduser.mail_preferred_mail_address = data["mail_preferred_mail_address"] + user.dashboarduser.mail_preferred_language = data["mail_preferred_language"] + user.dashboarduser.mail_send_mail_after_scan_finished = data["mail_send_mail_after_scan_finished"] user.dashboarduser.save() return operation_response(success=True, message="save_user_settings_success") diff --git a/dashboard/internet_nl_dashboard/management/commands/check_running_dashboard_scans.py b/dashboard/internet_nl_dashboard/management/commands/check_running_dashboard_scans.py index 8f1dfbd7..5e6936ae 100644 --- a/dashboard/internet_nl_dashboard/management/commands/check_running_dashboard_scans.py +++ b/dashboard/internet_nl_dashboard/management/commands/check_running_dashboard_scans.py @@ -12,7 +12,7 @@ class Command(TaskCommand): def add_arguments(self, parser): # https://stackoverflow.com/questions/8259001/python-argparse-command-line-flags-without-arguments - parser.add_argument('--reimport', action='store_true', help='Execute the task directly or on remote workers.') + parser.add_argument("--reimport", action="store_true", help="Execute the task directly or on remote workers.") return super().add_arguments(parser) @@ -27,7 +27,7 @@ def handle(self, *args, **options): def compose(self, *args, **options): model_filter = {} - if options['reimport']: - model_filter = {'accountinternetnlscan_filter': {'scan__finished': False}} + if options["reimport"]: + model_filter = {"accountinternetnlscan_filter": {"scan__finished": False}} return check_running_dashboard_scans(**model_filter) diff --git a/dashboard/internet_nl_dashboard/management/commands/dashboard_celery.py b/dashboard/internet_nl_dashboard/management/commands/dashboard_celery.py index 57080162..c924e6ea 100644 --- a/dashboard/internet_nl_dashboard/management/commands/dashboard_celery.py +++ b/dashboard/internet_nl_dashboard/management/commands/dashboard_celery.py @@ -12,8 +12,8 @@ def reusable_run_from_argv(argv): """Replace python with celery process with given arguments.""" - appname = __name__.split('.', 1)[0] + '.celery:app' - appname_arguments = ['-A', appname] + appname = __name__.split(".", 1)[0] + ".celery:app" + appname_arguments = ["-A", appname] log.info(argv[1]) log.info(argv[1:2] + appname_arguments + argv[2:]) @@ -26,6 +26,7 @@ def reusable_run_from_argv(argv): class Command(BaseCommand): # pylint: disable=abstract-method """Celery command wrapper.""" + help = __doc__ # disable (MySQL) check on startup diff --git a/dashboard/internet_nl_dashboard/management/commands/dashboard_prdserver.py b/dashboard/internet_nl_dashboard/management/commands/dashboard_prdserver.py index fc9a39fb..7a23aa36 100644 --- a/dashboard/internet_nl_dashboard/management/commands/dashboard_prdserver.py +++ b/dashboard/internet_nl_dashboard/management/commands/dashboard_prdserver.py @@ -5,6 +5,7 @@ import sys from django.core.management import call_command + # django_uwsgi is only in the deployement depencies as it has issues on M1 macs. from django_uwsgi.management.commands.runuwsgi import Command as RunserverCommand # pylint: disable=import-error @@ -23,13 +24,15 @@ class Command(RunserverCommand): """Run a Failmap production server.""" - command = 'runuwsgi' + command = "runuwsgi" def add_arguments(self, parser): - parser.add_argument('-m', '--migrate', action='store_true', - help='Before starting server run Django migrations.') - parser.add_argument('-l', '--loaddata', default=None, type=str, - help='Comma separated list of data fixtures to load.') + parser.add_argument( + "-m", "--migrate", action="store_true", help="Before starting server run Django migrations." + ) + parser.add_argument( + "-l", "--loaddata", default=None, type=str, help="Comma separated list of data fixtures to load." + ) super().add_arguments(parser) @@ -42,22 +45,22 @@ def handle(self, *args, **options): # todo: fix Starting a process with a partial executable path # todo: fix subprocess call with shell=True seems safe, but may be changed in the future, consider rewriting # without shell - subprocess.check_call('command -v uwsgi', shell=True, stdout=subprocess.DEVNULL) # nosec + subprocess.check_call("command -v uwsgi", shell=True, stdout=subprocess.DEVNULL) # nosec except subprocess.CalledProcessError: print(UWSGI_INFO) sys.exit(1) - if set(options).intersection(['migrate', 'loaddata']): + if set(options).intersection(["migrate", "loaddata"]): # detect if we run inside the autoreloader's second thread - inner_run = os.environ.get('RUN_MAIN', False) + inner_run = os.environ.get("RUN_MAIN", False) if inner_run: - log.info('Inner run: skipping --migrate/--loaddata.') + log.info("Inner run: skipping --migrate/--loaddata.") else: - if options['migrate']: - call_command('migrate') - if options['loaddata']: - call_command('load_dataset', *options['loaddata'].split(',')) + if options["migrate"]: + call_command("migrate") + if options["loaddata"]: + call_command("load_dataset", *options["loaddata"].split(",")) sys.stdout.flush() super().handle(*args, **options) diff --git a/dashboard/internet_nl_dashboard/management/commands/one_shot_2023_03_add_missing_report_share_codes.py b/dashboard/internet_nl_dashboard/management/commands/one_shot_2023_03_add_missing_report_share_codes.py index 4470f7fa..72177dff 100644 --- a/dashboard/internet_nl_dashboard/management/commands/one_shot_2023_03_add_missing_report_share_codes.py +++ b/dashboard/internet_nl_dashboard/management/commands/one_shot_2023_03_add_missing_report_share_codes.py @@ -10,7 +10,7 @@ class Command(BaseCommand): - help = 'Upgrades reports to early 2020 style, which is faster and more complete.' + help = "Upgrades reports to early 2020 style, which is faster and more complete." def handle(self, *args, **options): add_share_codes_to_reports() diff --git a/dashboard/internet_nl_dashboard/management/commands/one_shot_upgrade_reports.py b/dashboard/internet_nl_dashboard/management/commands/one_shot_upgrade_reports.py index 22f7e366..ec184100 100644 --- a/dashboard/internet_nl_dashboard/management/commands/one_shot_upgrade_reports.py +++ b/dashboard/internet_nl_dashboard/management/commands/one_shot_upgrade_reports.py @@ -4,14 +4,16 @@ from django.core.management.base import BaseCommand from dashboard.internet_nl_dashboard.models import AccountInternetNLScan, UrlListReport -from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import (upgrade_report_with_statistics, - upgrade_report_with_unscannable_urls) +from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import ( + upgrade_report_with_statistics, + upgrade_report_with_unscannable_urls, +) log = logging.getLogger(__package__) class Command(BaseCommand): - help = 'Upgrades reports to early 2020 style, which is faster and more complete.' + help = "Upgrades reports to early 2020 style, which is faster and more complete." def handle(self, *args, **options): diff --git a/dashboard/internet_nl_dashboard/management/commands/rewrite_report_for_scan.py b/dashboard/internet_nl_dashboard/management/commands/rewrite_report_for_scan.py index 0557f541..d3a6f1ce 100644 --- a/dashboard/internet_nl_dashboard/management/commands/rewrite_report_for_scan.py +++ b/dashboard/internet_nl_dashboard/management/commands/rewrite_report_for_scan.py @@ -6,26 +6,28 @@ from dashboard.internet_nl_dashboard.logic.urllist_dashboard_report import create_dashboard_report_at from dashboard.internet_nl_dashboard.models import AccountInternetNLScan from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import ( - connect_urllistreport_to_accountinternetnlscan, upgrade_report_with_statistics, - upgrade_report_with_unscannable_urls) + connect_urllistreport_to_accountinternetnlscan, + upgrade_report_with_statistics, + upgrade_report_with_unscannable_urls, +) log = logging.getLogger(__package__) class Command(BaseCommand): - help = 'Overwrite an existing report. This can be useful if the reporting logic has changed.' + help = "Overwrite an existing report. This can be useful if the reporting logic has changed." def add_arguments(self, parser): - parser.add_argument('--scan', type=int, required=True) + parser.add_argument("--scan", type=int, required=True) super().add_arguments(parser) def handle(self, *args, **options): # See if we can find the scan - if not options['scan']: + if not options["scan"]: raise ValueError("No scan id given") - scan = AccountInternetNLScan.objects.all().filter(scan=options['scan']).first() + scan = AccountInternetNLScan.objects.all().filter(scan=options["scan"]).first() if not scan: raise ValueError(f"Scan {options['scan']} does not exist") @@ -34,8 +36,10 @@ def handle(self, *args, **options): raise ValueError(f"Scan {options['scan']} does not have a report attached") if not scan.report.at_when: - raise ValueError(f"No report created for scan {options['scan']}, " - "will not create a new report when the process is running.") + raise ValueError( + f"No report created for scan {options['scan']}, " + "will not create a new report when the process is running." + ) old_report_moment = scan.report.at_when diff --git a/dashboard/internet_nl_dashboard/management/commands/send_testmail.py b/dashboard/internet_nl_dashboard/management/commands/send_testmail.py index e579ec39..3e2ed41d 100644 --- a/dashboard/internet_nl_dashboard/management/commands/send_testmail.py +++ b/dashboard/internet_nl_dashboard/management/commands/send_testmail.py @@ -9,9 +9,10 @@ def handle(self, *args, **options): mail.send( sender=config.EMAIL_NOTIFICATION_SENDER, recipients=config.EMAIL_TEST_RECIPIENT, - subject='Dashboard test e-mail', - message='This is a test email from the dashboard.', + subject="Dashboard test e-mail", + message="This is a test email from the dashboard.", priority=models.PRIORITY.now, - html_message='This is a test email from the dashboard hi!', + html_message="This is a test email from the dashboard hi!", ) + print("Mail sent!") diff --git a/dashboard/internet_nl_dashboard/management/commands/set_scan_state.py b/dashboard/internet_nl_dashboard/management/commands/set_scan_state.py index 5ba3cdac..aebc16d2 100644 --- a/dashboard/internet_nl_dashboard/management/commands/set_scan_state.py +++ b/dashboard/internet_nl_dashboard/management/commands/set_scan_state.py @@ -14,27 +14,27 @@ class Command(BaseCommand): help = 'Example: set_scan_state id=7 state="create report"' def add_arguments(self, parser): - parser.add_argument('--id', type=int, required=True) - parser.add_argument('--state', type=str, required=True) + parser.add_argument("--id", type=int, required=True) + parser.add_argument("--state", type=str, required=True) super().add_arguments(parser) def handle(self, *args, **options): if not settings.DEBUG: - log.info('Can only be used in development environment.') + log.info("Can only be used in development environment.") - if not options['id']: + if not options["id"]: log.error('Specify the scan id: set_scan_state --id=7 --state="create report"') - if not options['state']: + if not options["state"]: log.error('specify the state: set_scan_state --id=7 --state="create report"') - scan = AccountInternetNLScan.objects.all().filter(id=options['id']).first() + scan = AccountInternetNLScan.objects.all().filter(id=options["id"]).first() if not scan: log.error("Scan does not exist.") return - scan.state = options['state'] - update_state(options['state'], scan.id) + scan.state = options["state"] + update_state(options["state"], scan.id) log.info(f"Scan {scan} is set to {options['state']}.") diff --git a/dashboard/internet_nl_dashboard/management/commands/solidify_report_types.py b/dashboard/internet_nl_dashboard/management/commands/solidify_report_types.py index 5ca04540..68f4abec 100644 --- a/dashboard/internet_nl_dashboard/management/commands/solidify_report_types.py +++ b/dashboard/internet_nl_dashboard/management/commands/solidify_report_types.py @@ -7,7 +7,7 @@ class Command(BaseCommand): def handle(self, *args, **options): - for report in UrlListReport.objects.all().defer('calculation'): + for report in UrlListReport.objects.all().defer("calculation"): print(report) report.report_type = report.urllist.scan_type report.save() diff --git a/dashboard/internet_nl_dashboard/migrations/0001_initial.py b/dashboard/internet_nl_dashboard/migrations/0001_initial.py index d5e617e5..9ded8bc7 100644 --- a/dashboard/internet_nl_dashboard/migrations/0001_initial.py +++ b/dashboard/internet_nl_dashboard/migrations/0001_initial.py @@ -15,25 +15,39 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='Account', + name="Account", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(blank=True, max_length=120, null=True)), - ('enable_logins', models.BooleanField(blank=True, default=False, - help_text='Inactive accounts cannot be logged-in to.', null=True)), - ('internet_nl_api_username', models.CharField(blank=True, - help_text='Internet.nl API Username', max_length=255, null=True)), - ('internet_nl_api_password', models.CharField(blank=True, - help_text='Will be encrypted.', max_length=255, null=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(blank=True, max_length=120, null=True)), + ( + "enable_logins", + models.BooleanField( + blank=True, default=False, help_text="Inactive accounts cannot be logged-in to.", null=True + ), + ), + ( + "internet_nl_api_username", + models.CharField(blank=True, help_text="Internet.nl API Username", max_length=255, null=True), + ), + ( + "internet_nl_api_password", + models.CharField(blank=True, help_text="Will be encrypted.", max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='DashboardUser', + name="DashboardUser", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('notes', models.TextField(blank=True, max_length=800, null=True)), - ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.Account')), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("notes", models.TextField(blank=True, max_length=800, null=True)), + ( + "account", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.Account"), + ), + ( + "user", + models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), ], ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0001_squashed_0045_auto_20201027_1039.py b/dashboard/internet_nl_dashboard/migrations/0001_squashed_0045_auto_20201027_1039.py index 9b530391..6fb5c239 100644 --- a/dashboard/internet_nl_dashboard/migrations/0001_squashed_0045_auto_20201027_1039.py +++ b/dashboard/internet_nl_dashboard/migrations/0001_squashed_0045_auto_20201027_1039.py @@ -11,209 +11,507 @@ class Migration(migrations.Migration): - replaces = [('internet_nl_dashboard', '0001_initial'), ('internet_nl_dashboard', '0002_auto_20190318_1656'), ('internet_nl_dashboard', '0003_uploadlog'), ('internet_nl_dashboard', '0004_uploadlog_user'), ('internet_nl_dashboard', '0005_auto_20190320_1454'), ('internet_nl_dashboard', '0006_uploadlog_status'), ('internet_nl_dashboard', '0007_remove_account_enable_logins'), ('internet_nl_dashboard', '0008_auto_20190325_1721'), ('internet_nl_dashboard', '0009_accountinternetnlscan'), ('internet_nl_dashboard', '0010_auto_20190326_0957'), ('internet_nl_dashboard', '0011_auto_20190326_1013'), ('internet_nl_dashboard', '0012_account_can_connect_to_internet_nl_api'), ('internet_nl_dashboard', '0013_auto_20190326_1224'), ('internet_nl_dashboard', '0014_accountinternetnlscan_urllist'), ('internet_nl_dashboard', '0015_auto_20190329_1325'), ('internet_nl_dashboard', '0016_auto_20190401_1626'), ('internet_nl_dashboard', '0017_urllist_scan_type'), ('internet_nl_dashboard', '0018_auto_20190402_1554'), ('internet_nl_dashboard', '0019_urllistreport_created_on'), ('internet_nl_dashboard', '0020_auto_20190412_1157'), ('internet_nl_dashboard', '0021_auto_20190425_0910'), ('internet_nl_dashboard', '0022_auto_20190425_1438'), ('internet_nl_dashboard', '0023_auto_20190429_0910'), - ('internet_nl_dashboard', '0024_auto_20190429_0923'), ('internet_nl_dashboard', '0025_auto_20190430_1021'), ('internet_nl_dashboard', '0026_auto_20190430_1452'), ('internet_nl_dashboard', '0027_auto_20190507_1233'), ('internet_nl_dashboard', '0028_auto_20190507_1509'), ('internet_nl_dashboard', '0029_auto_20190515_1206'), ('internet_nl_dashboard', '0030_auto_20190515_1209'), ('internet_nl_dashboard', '0031_account_report_settings'), ('internet_nl_dashboard', '0032_auto_20190528_1352'), ('internet_nl_dashboard', '0033_auto_20190604_1242'), ('internet_nl_dashboard', '0034_auto_20190613_0946'), ('internet_nl_dashboard', '0035_auto_20190624_0712'), ('internet_nl_dashboard', '0036_urllistreport_average_internet_nl_score'), ('internet_nl_dashboard', '0037_auto_20191121_1408'), ('internet_nl_dashboard', '0038_accountinternetnlscan_state_changed_on'), ('internet_nl_dashboard', '0039_accountinternetnlscan_report'), ('internet_nl_dashboard', '0040_auto_20200508_1013'), ('internet_nl_dashboard', '0041_auto_20200513_1351'), ('internet_nl_dashboard', '0042_auto_20200530_1735'), ('internet_nl_dashboard', '0043_auto_20201006_1309'), ('internet_nl_dashboard', '0044_dashboarduser_mail_after_mail_unsubscribe_code'), ('internet_nl_dashboard', '0045_auto_20201027_1039')] + replaces = [ + ("internet_nl_dashboard", "0001_initial"), + ("internet_nl_dashboard", "0002_auto_20190318_1656"), + ("internet_nl_dashboard", "0003_uploadlog"), + ("internet_nl_dashboard", "0004_uploadlog_user"), + ("internet_nl_dashboard", "0005_auto_20190320_1454"), + ("internet_nl_dashboard", "0006_uploadlog_status"), + ("internet_nl_dashboard", "0007_remove_account_enable_logins"), + ("internet_nl_dashboard", "0008_auto_20190325_1721"), + ("internet_nl_dashboard", "0009_accountinternetnlscan"), + ("internet_nl_dashboard", "0010_auto_20190326_0957"), + ("internet_nl_dashboard", "0011_auto_20190326_1013"), + ("internet_nl_dashboard", "0012_account_can_connect_to_internet_nl_api"), + ("internet_nl_dashboard", "0013_auto_20190326_1224"), + ("internet_nl_dashboard", "0014_accountinternetnlscan_urllist"), + ("internet_nl_dashboard", "0015_auto_20190329_1325"), + ("internet_nl_dashboard", "0016_auto_20190401_1626"), + ("internet_nl_dashboard", "0017_urllist_scan_type"), + ("internet_nl_dashboard", "0018_auto_20190402_1554"), + ("internet_nl_dashboard", "0019_urllistreport_created_on"), + ("internet_nl_dashboard", "0020_auto_20190412_1157"), + ("internet_nl_dashboard", "0021_auto_20190425_0910"), + ("internet_nl_dashboard", "0022_auto_20190425_1438"), + ("internet_nl_dashboard", "0023_auto_20190429_0910"), + ("internet_nl_dashboard", "0024_auto_20190429_0923"), + ("internet_nl_dashboard", "0025_auto_20190430_1021"), + ("internet_nl_dashboard", "0026_auto_20190430_1452"), + ("internet_nl_dashboard", "0027_auto_20190507_1233"), + ("internet_nl_dashboard", "0028_auto_20190507_1509"), + ("internet_nl_dashboard", "0029_auto_20190515_1206"), + ("internet_nl_dashboard", "0030_auto_20190515_1209"), + ("internet_nl_dashboard", "0031_account_report_settings"), + ("internet_nl_dashboard", "0032_auto_20190528_1352"), + ("internet_nl_dashboard", "0033_auto_20190604_1242"), + ("internet_nl_dashboard", "0034_auto_20190613_0946"), + ("internet_nl_dashboard", "0035_auto_20190624_0712"), + ("internet_nl_dashboard", "0036_urllistreport_average_internet_nl_score"), + ("internet_nl_dashboard", "0037_auto_20191121_1408"), + ("internet_nl_dashboard", "0038_accountinternetnlscan_state_changed_on"), + ("internet_nl_dashboard", "0039_accountinternetnlscan_report"), + ("internet_nl_dashboard", "0040_auto_20200508_1013"), + ("internet_nl_dashboard", "0041_auto_20200513_1351"), + ("internet_nl_dashboard", "0042_auto_20200530_1735"), + ("internet_nl_dashboard", "0043_auto_20201006_1309"), + ("internet_nl_dashboard", "0044_dashboarduser_mail_after_mail_unsubscribe_code"), + ("internet_nl_dashboard", "0045_auto_20201027_1039"), + ] initial = True dependencies = [ - ('scanners', '0060_auto_20190116_0937'), + ("scanners", "0060_auto_20190116_0937"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('scanners', '0072_auto_20200506_1313'), - ('organizations', '0053_url_do_not_find_subdomains'), + ("scanners", "0072_auto_20200506_1313"), + ("organizations", "0053_url_do_not_find_subdomains"), ] operations = [ migrations.CreateModel( - name='Account', + name="Account", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(blank=True, max_length=120, null=True)), - ('internet_nl_api_username', models.CharField(blank=True, - help_text='Internet.nl API Username', max_length=255, null=True)), - ('internet_nl_api_password', models.TextField(blank=True, - help_text='New values will automatically be encrypted.', null=True)), - ('enable_scans', models.BooleanField(default=True)), - ('can_connect_to_internet_nl_api', models.BooleanField(default=False)), - ('report_settings', jsonfield.fields.JSONField( - blank=True, help_text='This stores reporting preferences: what fields are shown in the UI and so on (if any other).This field can be edited on the report page.', null=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(blank=True, max_length=120, null=True)), + ( + "internet_nl_api_username", + models.CharField(blank=True, help_text="Internet.nl API Username", max_length=255, null=True), + ), + ( + "internet_nl_api_password", + models.TextField(blank=True, help_text="New values will automatically be encrypted.", null=True), + ), + ("enable_scans", models.BooleanField(default=True)), + ("can_connect_to_internet_nl_api", models.BooleanField(default=False)), + ( + "report_settings", + jsonfield.fields.JSONField( + blank=True, + help_text="This stores reporting preferences: what fields are shown in the UI and so on (if any other).This field can be edited on the report page.", + null=True, + ), + ), ], ), migrations.CreateModel( - name='DashboardUser', + name="DashboardUser", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('notes', models.TextField(blank=True, max_length=800, null=True)), - ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.account')), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ('mail_preferred_language', django_countries.fields.CountryField(default='EN', max_length=2)), - ('mail_preferred_mail_address', models.EmailField( - blank=True, help_text='This address can deviate from the account mail address for password resets and other account features.', max_length=254, null=True)), - ('mail_send_mail_after_scan_finished', models.BooleanField(default=False, - help_text='After a scan is finished, an e-mail is sent informing the user that a report is ready.')), - ('mail_after_mail_unsubscribe_code', models.CharField(blank=True, default='', - help_text='This is autofilled when sending an e-mail. The user can use this code to set mail_send_mail_after_scan_finished to false without logging in.', max_length=255)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("notes", models.TextField(blank=True, max_length=800, null=True)), + ( + "account", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.account"), + ), + ( + "user", + models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + ("mail_preferred_language", django_countries.fields.CountryField(default="EN", max_length=2)), + ( + "mail_preferred_mail_address", + models.EmailField( + blank=True, + help_text="This address can deviate from the account mail address for password resets and other account features.", + max_length=254, + null=True, + ), + ), + ( + "mail_send_mail_after_scan_finished", + models.BooleanField( + default=False, + help_text="After a scan is finished, an e-mail is sent informing the user that a report is ready.", + ), + ), + ( + "mail_after_mail_unsubscribe_code", + models.CharField( + blank=True, + default="", + help_text="This is autofilled when sending an e-mail. The user can use this code to set mail_send_mail_after_scan_finished to false without logging in.", + max_length=255, + ), + ), ], ), migrations.CreateModel( - name='UploadLog', + name="UploadLog", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('original_filename', models.CharField(blank=True, help_text='The original filename of the file that has been uploaded. Django appends a random string if the file already exists. This is a reconstruction of the original filename and may not be 100% accurate.', max_length=255, null=True)), - ('internal_filename', models.CharField( - blank=True, help_text='Generated filename by Django. This can be used to find specific files for debugging purposes.', max_length=255, null=True)), - ('message', models.CharField(blank=True, help_text='This message gives more specific information about what happened. For example, it might be the case that a file has been rejected because it had the wrong filetype etc.', max_length=255, null=True)), - ('upload_date', models.DateTimeField(blank=True, null=True)), - ('filesize', models.PositiveIntegerField( - default=0, help_text='Gives an indication if your local file has changed (different size). The size is in bytes.')), - ('user', models.ForeignKey(blank=True, help_text='What user performed this upload.', null=True, - on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.dashboarduser')), - ('status', models.CharField( - blank=True, help_text="If the upload was successful or not. Might contain 'success' or 'error'.", max_length=255, null=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "original_filename", + models.CharField( + blank=True, + help_text="The original filename of the file that has been uploaded. Django appends a random string if the file already exists. This is a reconstruction of the original filename and may not be 100% accurate.", + max_length=255, + null=True, + ), + ), + ( + "internal_filename", + models.CharField( + blank=True, + help_text="Generated filename by Django. This can be used to find specific files for debugging purposes.", + max_length=255, + null=True, + ), + ), + ( + "message", + models.CharField( + blank=True, + help_text="This message gives more specific information about what happened. For example, it might be the case that a file has been rejected because it had the wrong filetype etc.", + max_length=255, + null=True, + ), + ), + ("upload_date", models.DateTimeField(blank=True, null=True)), + ( + "filesize", + models.PositiveIntegerField( + default=0, + help_text="Gives an indication if your local file has changed (different size). The size is in bytes.", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + help_text="What user performed this upload.", + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="internet_nl_dashboard.dashboarduser", + ), + ), + ( + "status", + models.CharField( + blank=True, + help_text="If the upload was successful or not. Might contain 'success' or 'error'.", + max_length=255, + null=True, + ), + ), ], ), migrations.CreateModel( - name='UrlList', + name="UrlList", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(help_text='Name of the UrlList, for example name of the organization in it.', max_length=120)), - ('account', models.ForeignKey(help_text='Who owns and manages this urllist.', - on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.account')), - ('urls', models.ManyToManyField(blank=True, related_name='urls_in_dashboard_list', to='organizations.Url')), - ('enable_scans', models.BooleanField(default=True)), - ('scan_type', models.CharField(choices=[('web', 'web'), - ('mail', 'mail')], default='web', max_length=4)), - ('automated_scan_frequency', models.CharField(choices=[('disabled', 'disabled'), ('every half year', 'every half year'), ('at the start of every quarter', 'at the start of every quarter'), ( - 'every 1st day of the month', 'every 1st day of the month'), ('twice per month', 'twice per month')], default='disabled', help_text='At what moment should the scan start?', max_length=30)), - ('scheduled_next_scan', models.DateTimeField(default=datetime.datetime(2030, 1, 1, 1, 1, 1, 601526, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.')), - ('deleted_on', models.DateTimeField(blank=True, null=True)), - ('is_deleted', models.BooleanField(default=False)), - ('last_manual_scan', models.DateTimeField(blank=True, null=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "name", + models.CharField( + help_text="Name of the UrlList, for example name of the organization in it.", max_length=120 + ), + ), + ( + "account", + models.ForeignKey( + help_text="Who owns and manages this urllist.", + on_delete=django.db.models.deletion.CASCADE, + to="internet_nl_dashboard.account", + ), + ), + ( + "urls", + models.ManyToManyField(blank=True, related_name="urls_in_dashboard_list", to="organizations.Url"), + ), + ("enable_scans", models.BooleanField(default=True)), + ( + "scan_type", + models.CharField(choices=[("web", "web"), ("mail", "mail")], default="web", max_length=4), + ), + ( + "automated_scan_frequency", + models.CharField( + choices=[ + ("disabled", "disabled"), + ("every half year", "every half year"), + ("at the start of every quarter", "at the start of every quarter"), + ("every 1st day of the month", "every 1st day of the month"), + ("twice per month", "twice per month"), + ], + default="disabled", + help_text="At what moment should the scan start?", + max_length=30, + ), + ), + ( + "scheduled_next_scan", + models.DateTimeField( + default=datetime.datetime(2030, 1, 1, 1, 1, 1, 601526, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + ), + ), + ("deleted_on", models.DateTimeField(blank=True, null=True)), + ("is_deleted", models.BooleanField(default=False)), + ("last_manual_scan", models.DateTimeField(blank=True, null=True)), ], ), migrations.CreateModel( - name='UrlListReport', + name="UrlListReport", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('total_issues', models.IntegerField(default=0)), - ('high', models.IntegerField(default=0)), - ('medium', models.IntegerField(default=0)), - ('low', models.IntegerField(default=0)), - ('ok', models.IntegerField(default=0)), - ('total_urls', models.IntegerField(default=0, help_text='Amount of urls for this organization.')), - ('high_urls', models.IntegerField(default=0, help_text='Amount of urls with (1 or more) high risk issues.')), - ('medium_urls', models.IntegerField(default=0, help_text='Amount of urls with (1 or more) medium risk issues.')), - ('low_urls', models.IntegerField(default=0, help_text='Amount of urls with (1 or more) low risk issues.')), - ('ok_urls', models.IntegerField(default=0, help_text='Amount of urls with zero issues.')), - ('total_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints for this url.')), - ('high_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) high risk issues.')), - ('medium_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) medium risk issues.')), - ('low_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints with (1 or more) low risk issues.')), - ('ok_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints with zero issues.')), - ('total_url_issues', models.IntegerField(default=0, help_text='Total amount of issues on url level.')), - ('url_issues_high', models.IntegerField(default=0, help_text='Number of high issues on url level.')), - ('url_issues_medium', models.IntegerField(default=0, help_text='Number of medium issues on url level.')), - ('url_issues_low', models.IntegerField(default=0, help_text='Number of low issues on url level.')), - ('url_ok', models.IntegerField(default=0, help_text='Zero issues on these urls.')), - ('total_endpoint_issues', models.IntegerField(default=0, - help_text='A sum of all endpoint issues for this endpoint, it includes all high, medium and lows.')), - ('endpoint_issues_high', models.IntegerField(default=0, - help_text='Total amount of high risk issues on this endpoint.')), - ('endpoint_issues_medium', models.IntegerField(default=0, - help_text='Total amount of medium risk issues on this endpoint.')), - ('endpoint_issues_low', models.IntegerField(default=0, - help_text='Total amount of low risk issues on this endpoint')), - ('endpoint_ok', models.IntegerField(default=0, - help_text='Amount of measurements that resulted in an OK score on this endpoint.')), - ('explained_total_issues', models.IntegerField(default=0, - help_text='The summed number of all vulnerabilities and failures.')), - ('explained_high', models.IntegerField(default=0, help_text='The number of high risk vulnerabilities and failures.')), - ('explained_medium', models.IntegerField(default=0, - help_text='The number of medium risk vulnerabilities and failures.')), - ('explained_low', models.IntegerField(default=0, help_text='The number of low risk vulnerabilities and failures.')), - ('explained_total_urls', models.IntegerField(default=0, help_text='Amount of urls for this organization.')), - ('explained_high_urls', models.IntegerField(default=0, - help_text='Amount of urls with (1 or more) high risk issues.')), - ('explained_medium_urls', models.IntegerField(default=0, - help_text='Amount of urls with (1 or more) medium risk issues.')), - ('explained_low_urls', models.IntegerField(default=0, - help_text='Amount of urls with (1 or more) low risk issues.')), - ('explained_total_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints for this url.')), - ('explained_high_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) high risk issues.')), - ('explained_medium_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) medium risk issues.')), - ('explained_low_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) low risk issues.')), - ('explained_total_url_issues', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_url_issues_high', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_url_issues_medium', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_url_issues_low', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_total_endpoint_issues', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_endpoint_issues_high', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_endpoint_issues_medium', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_endpoint_issues_low', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('at_when', models.DateTimeField(db_index=True)), - ('calculation', jsonfield.fields.JSONField( - help_text='Contains JSON with a calculation of all scanners at this moment, for all urls of this organization. This can be a lot.')), - ('urllist', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.urllist')), - ('endpoint_not_applicable', models.IntegerField(default=0, - help_text='Amount of things that are not applicable on this endpoint.')), - ('endpoint_not_testable', models.IntegerField(default=0, - help_text='Amount of things that could not be tested on this endpoint.')), - ('not_applicable', models.IntegerField(default=0)), - ('not_testable', models.IntegerField(default=0)), - ('url_not_applicable', models.IntegerField(default=0, - help_text='Amount of things that are not applicable on this url.')), - ('url_not_testable', models.IntegerField(default=0, - help_text='Amount of things that could not be tested on this url.')), - ('average_internet_nl_score', models.FloatField( - default=0, help_text='Internet.nl scores are retrieved in point. The calculation done for that is complex and subject to change over time. Therefore it is impossible to re-calculate that score here.Instead the score is stored as a given.')), - ('endpoint_error_in_test', models.IntegerField(default=0, - help_text='Amount of errors in tests performed on this endpoint.')), - ('error_in_test', models.IntegerField(default=0)), - ('url_error_in_test', models.IntegerField(default=0, help_text='Amount of errors in tests on this url.')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("total_issues", models.IntegerField(default=0)), + ("high", models.IntegerField(default=0)), + ("medium", models.IntegerField(default=0)), + ("low", models.IntegerField(default=0)), + ("ok", models.IntegerField(default=0)), + ("total_urls", models.IntegerField(default=0, help_text="Amount of urls for this organization.")), + ( + "high_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) high risk issues."), + ), + ( + "medium_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) medium risk issues."), + ), + ( + "low_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) low risk issues."), + ), + ("ok_urls", models.IntegerField(default=0, help_text="Amount of urls with zero issues.")), + ("total_endpoints", models.IntegerField(default=0, help_text="Amount of endpoints for this url.")), + ( + "high_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) high risk issues."), + ), + ( + "medium_endpoints", + models.IntegerField( + default=0, help_text="Amount of endpoints with (1 or more) medium risk issues." + ), + ), + ( + "low_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) low risk issues."), + ), + ("ok_endpoints", models.IntegerField(default=0, help_text="Amount of endpoints with zero issues.")), + ("total_url_issues", models.IntegerField(default=0, help_text="Total amount of issues on url level.")), + ("url_issues_high", models.IntegerField(default=0, help_text="Number of high issues on url level.")), + ( + "url_issues_medium", + models.IntegerField(default=0, help_text="Number of medium issues on url level."), + ), + ("url_issues_low", models.IntegerField(default=0, help_text="Number of low issues on url level.")), + ("url_ok", models.IntegerField(default=0, help_text="Zero issues on these urls.")), + ( + "total_endpoint_issues", + models.IntegerField( + default=0, + help_text="A sum of all endpoint issues for this endpoint, it includes all high, medium and lows.", + ), + ), + ( + "endpoint_issues_high", + models.IntegerField(default=0, help_text="Total amount of high risk issues on this endpoint."), + ), + ( + "endpoint_issues_medium", + models.IntegerField(default=0, help_text="Total amount of medium risk issues on this endpoint."), + ), + ( + "endpoint_issues_low", + models.IntegerField(default=0, help_text="Total amount of low risk issues on this endpoint"), + ), + ( + "endpoint_ok", + models.IntegerField( + default=0, help_text="Amount of measurements that resulted in an OK score on this endpoint." + ), + ), + ( + "explained_total_issues", + models.IntegerField(default=0, help_text="The summed number of all vulnerabilities and failures."), + ), + ( + "explained_high", + models.IntegerField(default=0, help_text="The number of high risk vulnerabilities and failures."), + ), + ( + "explained_medium", + models.IntegerField(default=0, help_text="The number of medium risk vulnerabilities and failures."), + ), + ( + "explained_low", + models.IntegerField(default=0, help_text="The number of low risk vulnerabilities and failures."), + ), + ( + "explained_total_urls", + models.IntegerField(default=0, help_text="Amount of urls for this organization."), + ), + ( + "explained_high_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) high risk issues."), + ), + ( + "explained_medium_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) medium risk issues."), + ), + ( + "explained_low_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) low risk issues."), + ), + ( + "explained_total_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints for this url."), + ), + ( + "explained_high_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) high risk issues."), + ), + ( + "explained_medium_endpoints", + models.IntegerField( + default=0, help_text="Amount of endpoints with (1 or more) medium risk issues." + ), + ), + ( + "explained_low_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) low risk issues."), + ), + ( + "explained_total_url_issues", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_url_issues_high", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_url_issues_medium", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_url_issues_low", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_total_endpoint_issues", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_endpoint_issues_high", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_endpoint_issues_medium", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_endpoint_issues_low", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ("at_when", models.DateTimeField(db_index=True)), + ( + "calculation", + jsonfield.fields.JSONField( + help_text="Contains JSON with a calculation of all scanners at this moment, for all urls of this organization. This can be a lot." + ), + ), + ( + "urllist", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.urllist"), + ), + ( + "endpoint_not_applicable", + models.IntegerField( + default=0, help_text="Amount of things that are not applicable on this endpoint." + ), + ), + ( + "endpoint_not_testable", + models.IntegerField( + default=0, help_text="Amount of things that could not be tested on this endpoint." + ), + ), + ("not_applicable", models.IntegerField(default=0)), + ("not_testable", models.IntegerField(default=0)), + ( + "url_not_applicable", + models.IntegerField(default=0, help_text="Amount of things that are not applicable on this url."), + ), + ( + "url_not_testable", + models.IntegerField(default=0, help_text="Amount of things that could not be tested on this url."), + ), + ( + "average_internet_nl_score", + models.FloatField( + default=0, + help_text="Internet.nl scores are retrieved in point. The calculation done for that is complex and subject to change over time. Therefore it is impossible to re-calculate that score here.Instead the score is stored as a given.", + ), + ), + ( + "endpoint_error_in_test", + models.IntegerField(default=0, help_text="Amount of errors in tests performed on this endpoint."), + ), + ("error_in_test", models.IntegerField(default=0)), + ( + "url_error_in_test", + models.IntegerField(default=0, help_text="Amount of errors in tests on this url."), + ), ], options={ - 'get_latest_by': 'at_when', - 'index_together': {('at_when', 'id')}, + "get_latest_by": "at_when", + "index_together": {("at_when", "id")}, }, ), migrations.CreateModel( - name='AccountInternetNLScan', + name="AccountInternetNLScan", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.account')), - ('scan', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='scanners.internetnlv2scan')), - ('urllist', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.urllist')), - ('state', models.CharField(blank=True, default='', help_text='The current state', max_length=255)), - ('state_changed_on', models.DateTimeField(blank=True, null=True)), - ('report', models.ForeignKey(blank=True, help_text='After a scan has finished, a report is created. This points to that report so no guessing is needed to figure out what report belongs to what scan.', - null=True, on_delete=django.db.models.deletion.SET_NULL, to='internet_nl_dashboard.urllistreport')), - ('finished_on', models.DateTimeField(blank=True, null=True)), - ('started_on', models.DateTimeField(blank=True, null=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "account", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.account"), + ), + ( + "scan", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to="scanners.internetnlv2scan" + ), + ), + ( + "urllist", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.urllist"), + ), + ("state", models.CharField(blank=True, default="", help_text="The current state", max_length=255)), + ("state_changed_on", models.DateTimeField(blank=True, null=True)), + ( + "report", + models.ForeignKey( + blank=True, + help_text="After a scan has finished, a report is created. This points to that report so no guessing is needed to figure out what report belongs to what scan.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="internet_nl_dashboard.urllistreport", + ), + ), + ("finished_on", models.DateTimeField(blank=True, null=True)), + ("started_on", models.DateTimeField(blank=True, null=True)), ], ), migrations.CreateModel( - name='AccountInternetNLScanLog', + name="AccountInternetNLScanLog", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('state', models.CharField(blank=True, default='', - help_text='The state that was registered at a certain moment in time.', max_length=255)), - ('at_when', models.DateTimeField(blank=True, null=True)), - ('scan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, - to='internet_nl_dashboard.accountinternetnlscan')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "state", + models.CharField( + blank=True, + default="", + help_text="The state that was registered at a certain moment in time.", + max_length=255, + ), + ), + ("at_when", models.DateTimeField(blank=True, null=True)), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.accountinternetnlscan" + ), + ), ], ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0002_auto_20190318_1656.py b/dashboard/internet_nl_dashboard/migrations/0002_auto_20190318_1656.py index ff807b02..f98bd5bb 100644 --- a/dashboard/internet_nl_dashboard/migrations/0002_auto_20190318_1656.py +++ b/dashboard/internet_nl_dashboard/migrations/0002_auto_20190318_1656.py @@ -7,34 +7,45 @@ class Migration(migrations.Migration): dependencies = [ - ('organizations', '0053_url_do_not_find_subdomains'), - ('internet_nl_dashboard', '0001_initial'), + ("organizations", "0053_url_do_not_find_subdomains"), + ("internet_nl_dashboard", "0001_initial"), ] operations = [ migrations.CreateModel( - name='UrlList', + name="UrlList", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField( - blank=True, help_text='Name of the UrlList, for example name of the organization in it.', max_length=120, null=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "name", + models.CharField( + blank=True, + help_text="Name of the UrlList, for example name of the organization in it.", + max_length=120, + null=True, + ), + ), ], ), migrations.AlterField( - model_name='account', - name='internet_nl_api_password', + model_name="account", + name="internet_nl_api_password", field=models.CharField( - blank=True, help_text='New values will automatically be encrypted.', max_length=255, null=True), + blank=True, help_text="New values will automatically be encrypted.", max_length=255, null=True + ), ), migrations.AddField( - model_name='urllist', - name='account', - field=models.ForeignKey(help_text='Who owns and manages this urllist.', - on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.Account'), + model_name="urllist", + name="account", + field=models.ForeignKey( + help_text="Who owns and manages this urllist.", + on_delete=django.db.models.deletion.CASCADE, + to="internet_nl_dashboard.Account", + ), ), migrations.AddField( - model_name='urllist', - name='urls', - field=models.ManyToManyField(blank=True, related_name='urls_in_dashboard_list', to='organizations.Url'), + model_name="urllist", + name="urls", + field=models.ManyToManyField(blank=True, related_name="urls_in_dashboard_list", to="organizations.Url"), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0002_auto_20210729_1549.py b/dashboard/internet_nl_dashboard/migrations/0002_auto_20210729_1549.py index c7703117..6f6bfede 100644 --- a/dashboard/internet_nl_dashboard/migrations/0002_auto_20210729_1549.py +++ b/dashboard/internet_nl_dashboard/migrations/0002_auto_20210729_1549.py @@ -6,26 +6,33 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0001_squashed_0045_auto_20201027_1039'), + ("internet_nl_dashboard", "0001_squashed_0045_auto_20201027_1039"), ] operations = [ migrations.AddField( - model_name='urllistreport', - name='is_publicly_shared', + model_name="urllistreport", + name="is_publicly_shared", field=models.BooleanField( - default=False, help_text='Sharing can be disabled and re-enabled where the report code and the share code (password) stay the same.'), + default=False, + help_text="Sharing can be disabled and re-enabled where the report code and the share code (password) stay the same.", + ), ), migrations.AddField( - model_name='urllistreport', - name='public_report_code', - field=models.CharField(blank=True, default='', - help_text='a unique code that used to identify this report', max_length=64), + model_name="urllistreport", + name="public_report_code", + field=models.CharField( + blank=True, default="", help_text="a unique code that used to identify this report", max_length=64 + ), ), migrations.AddField( - model_name='urllistreport', - name='public_share_code', + model_name="urllistreport", + name="public_share_code", field=models.CharField( - blank=True, default='', help_text='An unencrypted share code that can be seen by all users in an account. Can be modified by all.', max_length=64), + blank=True, + default="", + help_text="An unencrypted share code that can be seen by all users in an account. Can be modified by all.", + max_length=64, + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0003_uploadlog.py b/dashboard/internet_nl_dashboard/migrations/0003_uploadlog.py index c7b76196..b0d08d78 100644 --- a/dashboard/internet_nl_dashboard/migrations/0003_uploadlog.py +++ b/dashboard/internet_nl_dashboard/migrations/0003_uploadlog.py @@ -6,21 +6,49 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0002_auto_20190318_1656'), + ("internet_nl_dashboard", "0002_auto_20190318_1656"), ] operations = [ migrations.CreateModel( - name='UploadLog', + name="UploadLog", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('orginal_filename', models.CharField(blank=True, help_text='The original filename of the file that has been uploaded. Django appends a random string if the file already exists. This is a reconstruction of the original filename and may not be 100% accurate.', max_length=255, null=True)), - ('internal_filename', models.CharField( - blank=True, help_text='Generated filename by Django. This can be used to find specific files for debugging purposes.', max_length=255, null=True)), - ('message', models.CharField(blank=True, help_text='This message gives more specific information about what happened. For example, it might be the case that a file has been rejected because it had the wrong filetype etc.', max_length=255, null=True)), - ('upload_date', models.DateTimeField(blank=True, null=True)), - ('filesize', models.PositiveIntegerField( - default=0, help_text='Gives an indication if your local file has changed (different size). The size is in bytes.')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "orginal_filename", + models.CharField( + blank=True, + help_text="The original filename of the file that has been uploaded. Django appends a random string if the file already exists. This is a reconstruction of the original filename and may not be 100% accurate.", + max_length=255, + null=True, + ), + ), + ( + "internal_filename", + models.CharField( + blank=True, + help_text="Generated filename by Django. This can be used to find specific files for debugging purposes.", + max_length=255, + null=True, + ), + ), + ( + "message", + models.CharField( + blank=True, + help_text="This message gives more specific information about what happened. For example, it might be the case that a file has been rejected because it had the wrong filetype etc.", + max_length=255, + null=True, + ), + ), + ("upload_date", models.DateTimeField(blank=True, null=True)), + ( + "filesize", + models.PositiveIntegerField( + default=0, + help_text="Gives an indication if your local file has changed (different size). The size is in bytes.", + ), + ), ], ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0003_urllistreport_report_type.py b/dashboard/internet_nl_dashboard/migrations/0003_urllistreport_report_type.py index 5f643c9b..8c80d344 100644 --- a/dashboard/internet_nl_dashboard/migrations/0003_urllistreport_report_type.py +++ b/dashboard/internet_nl_dashboard/migrations/0003_urllistreport_report_type.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0002_auto_20210729_1549'), + ("internet_nl_dashboard", "0002_auto_20210729_1549"), ] operations = [ migrations.AddField( - model_name='urllistreport', - name='report_type', - field=models.CharField(default='web', max_length=10), + model_name="urllistreport", + name="report_type", + field=models.CharField(default="web", max_length=10), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0004_taggedurlinurllist.py b/dashboard/internet_nl_dashboard/migrations/0004_taggedurlinurllist.py index 35bfe7de..1dd0f41c 100644 --- a/dashboard/internet_nl_dashboard/migrations/0004_taggedurlinurllist.py +++ b/dashboard/internet_nl_dashboard/migrations/0004_taggedurlinurllist.py @@ -8,24 +8,34 @@ class Migration(migrations.Migration): dependencies = [ - ('taggit', '0003_taggeditem_add_unique_index'), - ('organizations', '0060_auto_20200908_1055'), - ('internet_nl_dashboard', '0003_urllistreport_report_type'), + ("taggit", "0003_taggeditem_add_unique_index"), + ("organizations", "0060_auto_20200908_1055"), + ("internet_nl_dashboard", "0003_urllistreport_report_type"), ] operations = [ migrations.CreateModel( - name='TaggedUrlInUrllist', + name="TaggedUrlInUrllist", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', - through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), - ('url', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='organizations.url')), - ('urllist', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.urllist')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "tags", + taggit.managers.TaggableManager( + help_text="A comma-separated list of tags.", + through="taggit.TaggedItem", + to="taggit.Tag", + verbose_name="Tags", + ), + ), + ("url", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="organizations.url")), + ( + "urllist", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.urllist"), + ), ], options={ - 'db_table': 'internet_nl_dashboard_urllist_x_tagged_url', - 'unique_together': {('urllist', 'url')}, + "db_table": "internet_nl_dashboard_urllist_x_tagged_url", + "unique_together": {("urllist", "url")}, }, ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0004_uploadlog_user.py b/dashboard/internet_nl_dashboard/migrations/0004_uploadlog_user.py index f2eeb501..2f23c71c 100644 --- a/dashboard/internet_nl_dashboard/migrations/0004_uploadlog_user.py +++ b/dashboard/internet_nl_dashboard/migrations/0004_uploadlog_user.py @@ -7,14 +7,19 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0003_uploadlog'), + ("internet_nl_dashboard", "0003_uploadlog"), ] operations = [ migrations.AddField( - model_name='uploadlog', - name='user', - field=models.ForeignKey(blank=True, help_text='What user performed this upload.', null=True, - on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.DashboardUser'), + model_name="uploadlog", + name="user", + field=models.ForeignKey( + blank=True, + help_text="What user performed this upload.", + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="internet_nl_dashboard.DashboardUser", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0005_auto_20190320_1454.py b/dashboard/internet_nl_dashboard/migrations/0005_auto_20190320_1454.py index 8340636f..781d7fe5 100644 --- a/dashboard/internet_nl_dashboard/migrations/0005_auto_20190320_1454.py +++ b/dashboard/internet_nl_dashboard/migrations/0005_auto_20190320_1454.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0004_uploadlog_user'), + ("internet_nl_dashboard", "0004_uploadlog_user"), ] operations = [ migrations.RenameField( - model_name='uploadlog', - old_name='orginal_filename', - new_name='original_filename', + model_name="uploadlog", + old_name="orginal_filename", + new_name="original_filename", ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0005_migrate_urlist_to_tagged_urllist.py b/dashboard/internet_nl_dashboard/migrations/0005_migrate_urlist_to_tagged_urllist.py index 15267413..10e7af92 100644 --- a/dashboard/internet_nl_dashboard/migrations/0005_migrate_urlist_to_tagged_urllist.py +++ b/dashboard/internet_nl_dashboard/migrations/0005_migrate_urlist_to_tagged_urllist.py @@ -12,9 +12,7 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0004_taggedurlinurllist'), + ("internet_nl_dashboard", "0004_taggedurlinurllist"), ] - operations = [ - migrations.RunSQL(migrate_join_table_data) - ] + operations = [migrations.RunSQL(migrate_join_table_data)] diff --git a/dashboard/internet_nl_dashboard/migrations/0006_remove_urllist_urls.py b/dashboard/internet_nl_dashboard/migrations/0006_remove_urllist_urls.py index 1cb7afd0..4e78da8d 100644 --- a/dashboard/internet_nl_dashboard/migrations/0006_remove_urllist_urls.py +++ b/dashboard/internet_nl_dashboard/migrations/0006_remove_urllist_urls.py @@ -6,12 +6,12 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0005_migrate_urlist_to_tagged_urllist'), + ("internet_nl_dashboard", "0005_migrate_urlist_to_tagged_urllist"), ] operations = [ migrations.RemoveField( - model_name='urllist', - name='urls', + model_name="urllist", + name="urls", ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0006_uploadlog_status.py b/dashboard/internet_nl_dashboard/migrations/0006_uploadlog_status.py index 9690ef7e..b72b9c5f 100644 --- a/dashboard/internet_nl_dashboard/migrations/0006_uploadlog_status.py +++ b/dashboard/internet_nl_dashboard/migrations/0006_uploadlog_status.py @@ -6,14 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0005_auto_20190320_1454'), + ("internet_nl_dashboard", "0005_auto_20190320_1454"), ] operations = [ migrations.AddField( - model_name='uploadlog', - name='status', + model_name="uploadlog", + name="status", field=models.CharField( - blank=True, help_text="If the upload was successful or not. Might contain 'success' or 'error'.", max_length=255, null=True), + blank=True, + help_text="If the upload was successful or not. Might contain 'success' or 'error'.", + max_length=255, + null=True, + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0007_remove_account_enable_logins.py b/dashboard/internet_nl_dashboard/migrations/0007_remove_account_enable_logins.py index 95dca9b1..59b3e988 100644 --- a/dashboard/internet_nl_dashboard/migrations/0007_remove_account_enable_logins.py +++ b/dashboard/internet_nl_dashboard/migrations/0007_remove_account_enable_logins.py @@ -6,12 +6,12 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0006_uploadlog_status'), + ("internet_nl_dashboard", "0006_uploadlog_status"), ] operations = [ migrations.RemoveField( - model_name='account', - name='enable_logins', + model_name="account", + name="enable_logins", ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0007_urllist_urls.py b/dashboard/internet_nl_dashboard/migrations/0007_urllist_urls.py index a0dcd089..0b2da9f0 100644 --- a/dashboard/internet_nl_dashboard/migrations/0007_urllist_urls.py +++ b/dashboard/internet_nl_dashboard/migrations/0007_urllist_urls.py @@ -6,14 +6,14 @@ class Migration(migrations.Migration): dependencies = [ - ('organizations', '0060_auto_20200908_1055'), - ('internet_nl_dashboard', '0006_remove_urllist_urls'), + ("organizations", "0060_auto_20200908_1055"), + ("internet_nl_dashboard", "0006_remove_urllist_urls"), ] operations = [ migrations.AddField( - model_name='urllist', - name='urls', - field=models.ManyToManyField(through='internet_nl_dashboard.TaggedUrlInUrllist', to='organizations.Url'), + model_name="urllist", + name="urls", + field=models.ManyToManyField(through="internet_nl_dashboard.TaggedUrlInUrllist", to="organizations.Url"), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0008_auto_20190325_1721.py b/dashboard/internet_nl_dashboard/migrations/0008_auto_20190325_1721.py index f3d849c3..e94b8229 100644 --- a/dashboard/internet_nl_dashboard/migrations/0008_auto_20190325_1721.py +++ b/dashboard/internet_nl_dashboard/migrations/0008_auto_20190325_1721.py @@ -6,18 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0007_remove_account_enable_logins'), + ("internet_nl_dashboard", "0007_remove_account_enable_logins"), ] operations = [ migrations.AddField( - model_name='account', - name='enable_scans', + model_name="account", + name="enable_scans", field=models.BooleanField(default=True), ), migrations.AddField( - model_name='urllist', - name='enable_scans', + model_name="urllist", + name="enable_scans", field=models.BooleanField(default=True), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0008_auto_20210908_1307.py b/dashboard/internet_nl_dashboard/migrations/0008_auto_20210908_1307.py index 2fa49363..8930875d 100644 --- a/dashboard/internet_nl_dashboard/migrations/0008_auto_20210908_1307.py +++ b/dashboard/internet_nl_dashboard/migrations/0008_auto_20210908_1307.py @@ -6,15 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('organizations', '0060_auto_20200908_1055'), - ('internet_nl_dashboard', '0007_urllist_urls'), + ("organizations", "0060_auto_20200908_1055"), + ("internet_nl_dashboard", "0007_urllist_urls"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='urls', - field=models.ManyToManyField(related_name='urls_in_dashboard_list_2', - through='internet_nl_dashboard.TaggedUrlInUrllist', to='organizations.Url'), + model_name="urllist", + name="urls", + field=models.ManyToManyField( + related_name="urls_in_dashboard_list_2", + through="internet_nl_dashboard.TaggedUrlInUrllist", + to="organizations.Url", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0009_accountinternetnlscan.py b/dashboard/internet_nl_dashboard/migrations/0009_accountinternetnlscan.py index 93cbccff..278d77e5 100644 --- a/dashboard/internet_nl_dashboard/migrations/0009_accountinternetnlscan.py +++ b/dashboard/internet_nl_dashboard/migrations/0009_accountinternetnlscan.py @@ -7,17 +7,20 @@ class Migration(migrations.Migration): dependencies = [ - ('scanners', '0060_auto_20190116_0937'), - ('internet_nl_dashboard', '0008_auto_20190325_1721'), + ("scanners", "0060_auto_20190116_0937"), + ("internet_nl_dashboard", "0008_auto_20190325_1721"), ] operations = [ migrations.CreateModel( - name='AccountInternetNLScan', + name="AccountInternetNLScan", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.Account')), - ('scan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scanners.InternetNLScan')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "account", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.Account"), + ), + ("scan", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="scanners.InternetNLScan")), ], ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0009_auto_20210927_0933.py b/dashboard/internet_nl_dashboard/migrations/0009_auto_20210927_0933.py index 2f7ea920..aa771078 100644 --- a/dashboard/internet_nl_dashboard/migrations/0009_auto_20210927_0933.py +++ b/dashboard/internet_nl_dashboard/migrations/0009_auto_20210927_0933.py @@ -6,14 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0008_auto_20210908_1307'), + ("internet_nl_dashboard", "0008_auto_20210908_1307"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='scan_type', - field=models.CharField(choices=[('web', 'web'), ('mail', 'mail'), - ('all', 'all')], default='web', max_length=4), + model_name="urllist", + name="scan_type", + field=models.CharField( + choices=[("web", "web"), ("mail", "mail"), ("all", "all")], default="web", max_length=4 + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0010_auto_20190326_0957.py b/dashboard/internet_nl_dashboard/migrations/0010_auto_20190326_0957.py index 2a8b7b86..9a47457a 100644 --- a/dashboard/internet_nl_dashboard/migrations/0010_auto_20190326_0957.py +++ b/dashboard/internet_nl_dashboard/migrations/0010_auto_20190326_0957.py @@ -6,14 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0009_accountinternetnlscan'), + ("internet_nl_dashboard", "0009_accountinternetnlscan"), ] operations = [ migrations.AlterField( - model_name='account', - name='internet_nl_api_password', + model_name="account", + name="internet_nl_api_password", field=models.BinaryField( - blank=True, help_text='New values will automatically be encrypted.', max_length=255, null=True), + blank=True, help_text="New values will automatically be encrypted.", max_length=255, null=True + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0010_subdomaindiscoveryscan.py b/dashboard/internet_nl_dashboard/migrations/0010_subdomaindiscoveryscan.py index e07252d7..59ce9c90 100644 --- a/dashboard/internet_nl_dashboard/migrations/0010_subdomaindiscoveryscan.py +++ b/dashboard/internet_nl_dashboard/migrations/0010_subdomaindiscoveryscan.py @@ -7,20 +7,29 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0009_auto_20210927_0933'), + ("internet_nl_dashboard", "0009_auto_20210927_0933"), ] operations = [ migrations.CreateModel( - name='SubdomainDiscoveryScan', + name="SubdomainDiscoveryScan", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('state', models.CharField(default='requested', - help_text='Name of the UrlList, for example name of the organization in it.', max_length=20)), - ('state_changed_on', models.DateTimeField(blank=True, null=True)), - ('state_message', models.CharField(max_length=200)), - ('domains_discovered', models.TextField()), - ('urllist', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.urllist')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "state", + models.CharField( + default="requested", + help_text="Name of the UrlList, for example name of the organization in it.", + max_length=20, + ), + ), + ("state_changed_on", models.DateTimeField(blank=True, null=True)), + ("state_message", models.CharField(max_length=200)), + ("domains_discovered", models.TextField()), + ( + "urllist", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.urllist"), + ), ], ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0011_auto_20190326_1013.py b/dashboard/internet_nl_dashboard/migrations/0011_auto_20190326_1013.py index b55da68c..d02a5c11 100644 --- a/dashboard/internet_nl_dashboard/migrations/0011_auto_20190326_1013.py +++ b/dashboard/internet_nl_dashboard/migrations/0011_auto_20190326_1013.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0010_auto_20190326_0957'), + ("internet_nl_dashboard", "0010_auto_20190326_0957"), ] operations = [ migrations.AlterField( - model_name='account', - name='internet_nl_api_password', - field=models.BinaryField(blank=True, help_text='New values will automatically be encrypted.', null=True), + model_name="account", + name="internet_nl_api_password", + field=models.BinaryField(blank=True, help_text="New values will automatically be encrypted.", null=True), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0011_auto_20211004_1419.py b/dashboard/internet_nl_dashboard/migrations/0011_auto_20211004_1419.py index 8995270c..0b1ac960 100644 --- a/dashboard/internet_nl_dashboard/migrations/0011_auto_20211004_1419.py +++ b/dashboard/internet_nl_dashboard/migrations/0011_auto_20211004_1419.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0010_subdomaindiscoveryscan'), + ("internet_nl_dashboard", "0010_subdomaindiscoveryscan"), ] operations = [ migrations.AlterField( - model_name='subdomaindiscoveryscan', - name='state_message', + model_name="subdomaindiscoveryscan", + name="state_message", field=models.CharField(blank=True, max_length=200), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0012_account_can_connect_to_internet_nl_api.py b/dashboard/internet_nl_dashboard/migrations/0012_account_can_connect_to_internet_nl_api.py index be77264e..a72ea4d8 100644 --- a/dashboard/internet_nl_dashboard/migrations/0012_account_can_connect_to_internet_nl_api.py +++ b/dashboard/internet_nl_dashboard/migrations/0012_account_can_connect_to_internet_nl_api.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0011_auto_20190326_1013'), + ("internet_nl_dashboard", "0011_auto_20190326_1013"), ] operations = [ migrations.AddField( - model_name='account', - name='can_connect_to_internet_nl_api', + model_name="account", + name="can_connect_to_internet_nl_api", field=models.BooleanField(default=False), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0012_urllistreport_is_shared_on_homepage.py b/dashboard/internet_nl_dashboard/migrations/0012_urllistreport_is_shared_on_homepage.py index 5bea57f8..82907169 100644 --- a/dashboard/internet_nl_dashboard/migrations/0012_urllistreport_is_shared_on_homepage.py +++ b/dashboard/internet_nl_dashboard/migrations/0012_urllistreport_is_shared_on_homepage.py @@ -6,14 +6,16 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0011_auto_20211004_1419'), + ("internet_nl_dashboard", "0011_auto_20211004_1419"), ] operations = [ migrations.AddField( - model_name='urllistreport', - name='is_shared_on_homepage', + model_name="urllistreport", + name="is_shared_on_homepage", field=models.BooleanField( - default=False, help_text='A public report can also be shared on the homepage with a link. Can only be shared on the homepage if the report is publicly shared. This is currently admin only.'), + default=False, + help_text="A public report can also be shared on the homepage with a link. Can only be shared on the homepage if the report is publicly shared. This is currently admin only.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0013_auto_20190326_1224.py b/dashboard/internet_nl_dashboard/migrations/0013_auto_20190326_1224.py index 90fb67b8..5db7d390 100644 --- a/dashboard/internet_nl_dashboard/migrations/0013_auto_20190326_1224.py +++ b/dashboard/internet_nl_dashboard/migrations/0013_auto_20190326_1224.py @@ -6,14 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0012_account_can_connect_to_internet_nl_api'), + ("internet_nl_dashboard", "0012_account_can_connect_to_internet_nl_api"), ] operations = [ migrations.AlterField( - model_name='account', - name='internet_nl_api_password', - field=models.BinaryField(blank=True, editable=True, - help_text='New values will automatically be encrypted.', null=True), + model_name="account", + name="internet_nl_api_password", + field=models.BinaryField( + blank=True, editable=True, help_text="New values will automatically be encrypted.", null=True + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0013_auto_20230302_1418.py b/dashboard/internet_nl_dashboard/migrations/0013_auto_20230302_1418.py index 72bfd6d6..30e18fe5 100644 --- a/dashboard/internet_nl_dashboard/migrations/0013_auto_20230302_1418.py +++ b/dashboard/internet_nl_dashboard/migrations/0013_auto_20230302_1418.py @@ -6,25 +6,34 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0012_urllistreport_is_shared_on_homepage'), + ("internet_nl_dashboard", "0012_urllistreport_is_shared_on_homepage"), ] operations = [ migrations.AddField( - model_name='urllist', - name='automatically_share_new_reports', + model_name="urllist", + name="automatically_share_new_reports", field=models.BooleanField( - default=False, help_text='Sharing can be disabled and re-enabled where the report code and the share code (password) stay the same. Sharing means that all new reports will be made public under a set of standard urls.'), + default=False, + help_text="Sharing can be disabled and re-enabled where the report code and the share code (password) stay the same. Sharing means that all new reports will be made public under a set of standard urls.", + ), ), migrations.AddField( - model_name='urllist', - name='default_public_share_code_for_new_reports', - field=models.CharField(blank=True, default='', help_text='An unencrypted share code that can be seen by all users in an account. Can be modified by all. New reports get this code set automatically. You can change this per report. An empty field means no share code and the report is accessible publicly.', max_length=64), + model_name="urllist", + name="default_public_share_code_for_new_reports", + field=models.CharField( + blank=True, + default="", + help_text="An unencrypted share code that can be seen by all users in an account. Can be modified by all. New reports get this code set automatically. You can change this per report. An empty field means no share code and the report is accessible publicly.", + max_length=64, + ), ), migrations.AddField( - model_name='urllist', - name='enable_report_sharing_page', + model_name="urllist", + name="enable_report_sharing_page", field=models.BooleanField( - default=False, help_text='When true there will be page under the list-id that shows all reports that are shared publicly.'), + default=False, + help_text="When true there will be page under the list-id that shows all reports that are shared publicly.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0014_accountinternetnlscan_urllist.py b/dashboard/internet_nl_dashboard/migrations/0014_accountinternetnlscan_urllist.py index cd1aaed7..cc65821e 100644 --- a/dashboard/internet_nl_dashboard/migrations/0014_accountinternetnlscan_urllist.py +++ b/dashboard/internet_nl_dashboard/migrations/0014_accountinternetnlscan_urllist.py @@ -7,14 +7,15 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0013_auto_20190326_1224'), + ("internet_nl_dashboard", "0013_auto_20190326_1224"), ] operations = [ migrations.AddField( - model_name='accountinternetnlscan', - name='urllist', - field=models.ForeignKey(blank=True, null=True, - on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.UrlList'), + model_name="accountinternetnlscan", + name="urllist", + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.UrlList" + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0015_auto_20190329_1325.py b/dashboard/internet_nl_dashboard/migrations/0015_auto_20190329_1325.py index 709c1e35..ab2be1ec 100644 --- a/dashboard/internet_nl_dashboard/migrations/0015_auto_20190329_1325.py +++ b/dashboard/internet_nl_dashboard/migrations/0015_auto_20190329_1325.py @@ -7,19 +7,20 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0014_accountinternetnlscan_urllist'), + ("internet_nl_dashboard", "0014_accountinternetnlscan_urllist"), ] operations = [ migrations.AlterField( - model_name='accountinternetnlscan', - name='urllist', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.UrlList'), + model_name="accountinternetnlscan", + name="urllist", + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.UrlList"), ), migrations.AlterField( - model_name='urllist', - name='name', + model_name="urllist", + name="name", field=models.CharField( - help_text='Name of the UrlList, for example name of the organization in it.', max_length=120), + help_text="Name of the UrlList, for example name of the organization in it.", max_length=120 + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0016_auto_20190401_1626.py b/dashboard/internet_nl_dashboard/migrations/0016_auto_20190401_1626.py index 4e3a6ac9..8c6b3755 100644 --- a/dashboard/internet_nl_dashboard/migrations/0016_auto_20190401_1626.py +++ b/dashboard/internet_nl_dashboard/migrations/0016_auto_20190401_1626.py @@ -8,87 +8,186 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0015_auto_20190329_1325'), + ("internet_nl_dashboard", "0015_auto_20190329_1325"), ] operations = [ migrations.CreateModel( - name='UrlListReport', + name="UrlListReport", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('total_issues', models.IntegerField(default=0, help_text='The summed number of all vulnerabilities and failures.')), - ('high', models.IntegerField(default=0, help_text='The number of high risk vulnerabilities and failures.')), - ('medium', models.IntegerField(default=0, help_text='The number of medium risk vulnerabilities and failures.')), - ('low', models.IntegerField(default=0, help_text='The number of low risk vulnerabilities and failures.')), - ('ok', models.IntegerField(default=0, help_text='No issues found at all.')), - ('total_urls', models.IntegerField(default=0, help_text='Amount of urls for this organization.')), - ('high_urls', models.IntegerField(default=0, help_text='Amount of urls with (1 or more) high risk issues.')), - ('medium_urls', models.IntegerField(default=0, help_text='Amount of urls with (1 or more) medium risk issues.')), - ('low_urls', models.IntegerField(default=0, help_text='Amount of urls with (1 or more) low risk issues.')), - ('ok_urls', models.IntegerField(default=0, help_text='Amount of urls with zero issues.')), - ('total_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints for this url.')), - ('high_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) high risk issues.')), - ('medium_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) medium risk issues.')), - ('low_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints with (1 or more) low risk issues.')), - ('ok_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints with zero issues.')), - ('total_url_issues', models.IntegerField(default=0, help_text='Total amount of issues on url level.')), - ('url_issues_high', models.IntegerField(default=0, help_text='Number of high issues on url level.')), - ('url_issues_medium', models.IntegerField(default=0, help_text='Number of medium issues on url level.')), - ('url_issues_low', models.IntegerField(default=0, help_text='Number of low issues on url level.')), - ('url_ok', models.IntegerField(default=0, help_text='Zero issues on these urls.')), - ('total_endpoint_issues', models.IntegerField(default=0, help_text='Total amount of issues on endpoint level.')), - ('endpoint_issues_high', models.IntegerField(default=0, help_text='Total amount of issues on endpoint level.')), - ('endpoint_issues_medium', models.IntegerField(default=0, - help_text='Total amount of issues on endpoint level.')), - ('endpoint_issues_low', models.IntegerField(default=0, help_text='Total amount of issues on endpoint level.')), - ('endpoint_ok', models.IntegerField(default=0, help_text='Zero issues on these endpoints.')), - ('explained_total_issues', models.IntegerField(default=0, - help_text='The summed number of all vulnerabilities and failures.')), - ('explained_high', models.IntegerField(default=0, help_text='The number of high risk vulnerabilities and failures.')), - ('explained_medium', models.IntegerField(default=0, - help_text='The number of medium risk vulnerabilities and failures.')), - ('explained_low', models.IntegerField(default=0, help_text='The number of low risk vulnerabilities and failures.')), - ('explained_total_urls', models.IntegerField(default=0, help_text='Amount of urls for this organization.')), - ('explained_high_urls', models.IntegerField(default=0, - help_text='Amount of urls with (1 or more) high risk issues.')), - ('explained_medium_urls', models.IntegerField(default=0, - help_text='Amount of urls with (1 or more) medium risk issues.')), - ('explained_low_urls', models.IntegerField(default=0, - help_text='Amount of urls with (1 or more) low risk issues.')), - ('explained_total_endpoints', models.IntegerField(default=0, help_text='Amount of endpoints for this url.')), - ('explained_high_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) high risk issues.')), - ('explained_medium_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) medium risk issues.')), - ('explained_low_endpoints', models.IntegerField(default=0, - help_text='Amount of endpoints with (1 or more) low risk issues.')), - ('explained_total_url_issues', models.IntegerField( - default=0, help_text='Total amount of issues on url level.')), - ('explained_url_issues_high', models.IntegerField(default=0, help_text='Number of high issues on url level.')), - ('explained_url_issues_medium', models.IntegerField( - default=0, help_text='Number of medium issues on url level.')), - ('explained_url_issues_low', models.IntegerField(default=0, help_text='Number of low issues on url level.')), - ('explained_total_endpoint_issues', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_endpoint_issues_high', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_endpoint_issues_medium', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('explained_endpoint_issues_low', models.IntegerField( - default=0, help_text='Total amount of issues on endpoint level.')), - ('when', models.DateTimeField(db_index=True)), - ('calculation', jsonfield.fields.JSONField( - help_text='Contains JSON with a calculation of all scanners at this moment, for all urls of this organization. This can be a lot.')), - ('urllist', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.UrlList')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "total_issues", + models.IntegerField(default=0, help_text="The summed number of all vulnerabilities and failures."), + ), + ( + "high", + models.IntegerField(default=0, help_text="The number of high risk vulnerabilities and failures."), + ), + ( + "medium", + models.IntegerField(default=0, help_text="The number of medium risk vulnerabilities and failures."), + ), + ( + "low", + models.IntegerField(default=0, help_text="The number of low risk vulnerabilities and failures."), + ), + ("ok", models.IntegerField(default=0, help_text="No issues found at all.")), + ("total_urls", models.IntegerField(default=0, help_text="Amount of urls for this organization.")), + ( + "high_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) high risk issues."), + ), + ( + "medium_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) medium risk issues."), + ), + ( + "low_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) low risk issues."), + ), + ("ok_urls", models.IntegerField(default=0, help_text="Amount of urls with zero issues.")), + ("total_endpoints", models.IntegerField(default=0, help_text="Amount of endpoints for this url.")), + ( + "high_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) high risk issues."), + ), + ( + "medium_endpoints", + models.IntegerField( + default=0, help_text="Amount of endpoints with (1 or more) medium risk issues." + ), + ), + ( + "low_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) low risk issues."), + ), + ("ok_endpoints", models.IntegerField(default=0, help_text="Amount of endpoints with zero issues.")), + ("total_url_issues", models.IntegerField(default=0, help_text="Total amount of issues on url level.")), + ("url_issues_high", models.IntegerField(default=0, help_text="Number of high issues on url level.")), + ( + "url_issues_medium", + models.IntegerField(default=0, help_text="Number of medium issues on url level."), + ), + ("url_issues_low", models.IntegerField(default=0, help_text="Number of low issues on url level.")), + ("url_ok", models.IntegerField(default=0, help_text="Zero issues on these urls.")), + ( + "total_endpoint_issues", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "endpoint_issues_high", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "endpoint_issues_medium", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "endpoint_issues_low", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ("endpoint_ok", models.IntegerField(default=0, help_text="Zero issues on these endpoints.")), + ( + "explained_total_issues", + models.IntegerField(default=0, help_text="The summed number of all vulnerabilities and failures."), + ), + ( + "explained_high", + models.IntegerField(default=0, help_text="The number of high risk vulnerabilities and failures."), + ), + ( + "explained_medium", + models.IntegerField(default=0, help_text="The number of medium risk vulnerabilities and failures."), + ), + ( + "explained_low", + models.IntegerField(default=0, help_text="The number of low risk vulnerabilities and failures."), + ), + ( + "explained_total_urls", + models.IntegerField(default=0, help_text="Amount of urls for this organization."), + ), + ( + "explained_high_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) high risk issues."), + ), + ( + "explained_medium_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) medium risk issues."), + ), + ( + "explained_low_urls", + models.IntegerField(default=0, help_text="Amount of urls with (1 or more) low risk issues."), + ), + ( + "explained_total_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints for this url."), + ), + ( + "explained_high_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) high risk issues."), + ), + ( + "explained_medium_endpoints", + models.IntegerField( + default=0, help_text="Amount of endpoints with (1 or more) medium risk issues." + ), + ), + ( + "explained_low_endpoints", + models.IntegerField(default=0, help_text="Amount of endpoints with (1 or more) low risk issues."), + ), + ( + "explained_total_url_issues", + models.IntegerField(default=0, help_text="Total amount of issues on url level."), + ), + ( + "explained_url_issues_high", + models.IntegerField(default=0, help_text="Number of high issues on url level."), + ), + ( + "explained_url_issues_medium", + models.IntegerField(default=0, help_text="Number of medium issues on url level."), + ), + ( + "explained_url_issues_low", + models.IntegerField(default=0, help_text="Number of low issues on url level."), + ), + ( + "explained_total_endpoint_issues", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_endpoint_issues_high", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_endpoint_issues_medium", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ( + "explained_endpoint_issues_low", + models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), + ), + ("when", models.DateTimeField(db_index=True)), + ( + "calculation", + jsonfield.fields.JSONField( + help_text="Contains JSON with a calculation of all scanners at this moment, for all urls of this organization. This can be a lot." + ), + ), + ( + "urllist", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.UrlList"), + ), ], options={ - 'get_latest_by': 'when', + "get_latest_by": "when", }, ), migrations.AlterIndexTogether( - name='urllistreport', - index_together={('when', 'id')}, + name="urllistreport", + index_together={("when", "id")}, ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0017_urllist_scan_type.py b/dashboard/internet_nl_dashboard/migrations/0017_urllist_scan_type.py index 090b0a0a..df2ba90e 100644 --- a/dashboard/internet_nl_dashboard/migrations/0017_urllist_scan_type.py +++ b/dashboard/internet_nl_dashboard/migrations/0017_urllist_scan_type.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0016_auto_20190401_1626'), + ("internet_nl_dashboard", "0016_auto_20190401_1626"), ] operations = [ migrations.AddField( - model_name='urllist', - name='scan_type', - field=models.CharField(choices=[('web', 'web'), ('mail', 'mail')], default='web', max_length=4), + model_name="urllist", + name="scan_type", + field=models.CharField(choices=[("web", "web"), ("mail", "mail")], default="web", max_length=4), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0018_auto_20190402_1554.py b/dashboard/internet_nl_dashboard/migrations/0018_auto_20190402_1554.py index 81dadd24..a7991ca1 100644 --- a/dashboard/internet_nl_dashboard/migrations/0018_auto_20190402_1554.py +++ b/dashboard/internet_nl_dashboard/migrations/0018_auto_20190402_1554.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0017_urllist_scan_type'), + ("internet_nl_dashboard", "0017_urllist_scan_type"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='scan_type', - field=models.CharField(choices=[('web', 'web'), ('mail', 'mail')], default='web', max_length=4), + model_name="urllist", + name="scan_type", + field=models.CharField(choices=[("web", "web"), ("mail", "mail")], default="web", max_length=4), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0019_urllistreport_created_on.py b/dashboard/internet_nl_dashboard/migrations/0019_urllistreport_created_on.py index aa0871ce..adbfb5ac 100644 --- a/dashboard/internet_nl_dashboard/migrations/0019_urllistreport_created_on.py +++ b/dashboard/internet_nl_dashboard/migrations/0019_urllistreport_created_on.py @@ -7,13 +7,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0018_auto_20190402_1554'), + ("internet_nl_dashboard", "0018_auto_20190402_1554"), ] operations = [ migrations.AddField( - model_name='urllistreport', - name='created_on', + model_name="urllistreport", + name="created_on", field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), preserve_default=False, ), diff --git a/dashboard/internet_nl_dashboard/migrations/0020_auto_20190412_1157.py b/dashboard/internet_nl_dashboard/migrations/0020_auto_20190412_1157.py index 6b6fb99c..7b443ed9 100644 --- a/dashboard/internet_nl_dashboard/migrations/0020_auto_20190412_1157.py +++ b/dashboard/internet_nl_dashboard/migrations/0020_auto_20190412_1157.py @@ -6,21 +6,21 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0019_urllistreport_created_on'), + ("internet_nl_dashboard", "0019_urllistreport_created_on"), ] operations = [ migrations.AlterModelOptions( - name='urllistreport', - options={'get_latest_by': 'at_when'}, + name="urllistreport", + options={"get_latest_by": "at_when"}, ), migrations.RenameField( - model_name='urllistreport', - old_name='when', - new_name='at_when', + model_name="urllistreport", + old_name="when", + new_name="at_when", ), migrations.AlterIndexTogether( - name='urllistreport', - index_together={('at_when', 'id')}, + name="urllistreport", + index_together={("at_when", "id")}, ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0021_auto_20190425_0910.py b/dashboard/internet_nl_dashboard/migrations/0021_auto_20190425_0910.py index 3f0ddccc..46408efa 100644 --- a/dashboard/internet_nl_dashboard/migrations/0021_auto_20190425_0910.py +++ b/dashboard/internet_nl_dashboard/migrations/0021_auto_20190425_0910.py @@ -7,21 +7,33 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0020_auto_20190412_1157'), + ("internet_nl_dashboard", "0020_auto_20190412_1157"), ] operations = [ migrations.AddField( - model_name='urllist', - name='automated_scan_frequency', - field=models.CharField(choices=[('disabled', 'disabled'), ('every half year', 'every half year'), ('at the start of every quarter', 'at the start of every quarter'), ( - 'every 1st day of the month', 'every 1st day of the month'), ('twice per month', 'twice per month')], default='disabled', help_text='At what moment should the scan start?', max_length=30), + model_name="urllist", + name="automated_scan_frequency", + field=models.CharField( + choices=[ + ("disabled", "disabled"), + ("every half year", "every half year"), + ("at the start of every quarter", "at the start of every quarter"), + ("every 1st day of the month", "every 1st day of the month"), + ("twice per month", "twice per month"), + ], + default="disabled", + help_text="At what moment should the scan start?", + max_length=30, + ), ), migrations.AddField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=django.utils.timezone.now, - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=django.utils.timezone.now, + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished.", + ), preserve_default=False, ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0022_auto_20190425_1438.py b/dashboard/internet_nl_dashboard/migrations/0022_auto_20190425_1438.py index 1c700af3..c12b083c 100644 --- a/dashboard/internet_nl_dashboard/migrations/0022_auto_20190425_1438.py +++ b/dashboard/internet_nl_dashboard/migrations/0022_auto_20190425_1438.py @@ -6,18 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0021_auto_20190425_0910'), + ("internet_nl_dashboard", "0021_auto_20190425_0910"), ] operations = [ migrations.AddField( - model_name='urllist', - name='deleted_on', + model_name="urllist", + name="deleted_on", field=models.DateTimeField(null=True), ), migrations.AddField( - model_name='urllist', - name='is_deleted', + model_name="urllist", + name="is_deleted", field=models.BooleanField(default=False), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0023_auto_20190429_0910.py b/dashboard/internet_nl_dashboard/migrations/0023_auto_20190429_0910.py index 73f7043b..46e8696e 100644 --- a/dashboard/internet_nl_dashboard/migrations/0023_auto_20190429_0910.py +++ b/dashboard/internet_nl_dashboard/migrations/0023_auto_20190429_0910.py @@ -8,14 +8,16 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0022_auto_20190425_1438'), + ("internet_nl_dashboard", "0022_auto_20190425_1438"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2019, 4, 28, 9, 10, 42, 684534, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2019, 4, 28, 9, 10, 42, 684534, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0024_auto_20190429_0923.py b/dashboard/internet_nl_dashboard/migrations/0024_auto_20190429_0923.py index 1cecd794..3a637f86 100644 --- a/dashboard/internet_nl_dashboard/migrations/0024_auto_20190429_0923.py +++ b/dashboard/internet_nl_dashboard/migrations/0024_auto_20190429_0923.py @@ -8,14 +8,16 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0023_auto_20190429_0910'), + ("internet_nl_dashboard", "0023_auto_20190429_0910"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2019, 4, 28, 9, 23, 16, 512927, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2019, 4, 28, 9, 23, 16, 512927, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0025_auto_20190430_1021.py b/dashboard/internet_nl_dashboard/migrations/0025_auto_20190430_1021.py index 42a9f038..e4e0350c 100644 --- a/dashboard/internet_nl_dashboard/migrations/0025_auto_20190430_1021.py +++ b/dashboard/internet_nl_dashboard/migrations/0025_auto_20190430_1021.py @@ -8,14 +8,16 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0024_auto_20190429_0923'), + ("internet_nl_dashboard", "0024_auto_20190429_0923"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2019, 4, 29, 10, 21, 57, 14534, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2019, 4, 29, 10, 21, 57, 14534, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0026_auto_20190430_1452.py b/dashboard/internet_nl_dashboard/migrations/0026_auto_20190430_1452.py index 26f22522..1425f158 100644 --- a/dashboard/internet_nl_dashboard/migrations/0026_auto_20190430_1452.py +++ b/dashboard/internet_nl_dashboard/migrations/0026_auto_20190430_1452.py @@ -8,19 +8,21 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0025_auto_20190430_1021'), + ("internet_nl_dashboard", "0025_auto_20190430_1021"), ] operations = [ migrations.AddField( - model_name='urllist', - name='last_manual_scan', + model_name="urllist", + name="last_manual_scan", field=models.DateTimeField(null=True), ), migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2019, 4, 29, 14, 52, 12, 15679, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2019, 4, 29, 14, 52, 12, 15679, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0027_auto_20190507_1233.py b/dashboard/internet_nl_dashboard/migrations/0027_auto_20190507_1233.py index 6b21e270..3049a0f6 100644 --- a/dashboard/internet_nl_dashboard/migrations/0027_auto_20190507_1233.py +++ b/dashboard/internet_nl_dashboard/migrations/0027_auto_20190507_1233.py @@ -8,18 +8,20 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0026_auto_20190430_1452'), + ("internet_nl_dashboard", "0026_auto_20190430_1452"), ] operations = [ migrations.RemoveField( - model_name='urllistreport', - name='created_on', + model_name="urllistreport", + name="created_on", ), migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2019, 5, 6, 12, 33, 46, 444527, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2019, 5, 6, 12, 33, 46, 444527, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0028_auto_20190507_1509.py b/dashboard/internet_nl_dashboard/migrations/0028_auto_20190507_1509.py index c3896b17..b4fea33e 100644 --- a/dashboard/internet_nl_dashboard/migrations/0028_auto_20190507_1509.py +++ b/dashboard/internet_nl_dashboard/migrations/0028_auto_20190507_1509.py @@ -8,19 +8,21 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0027_auto_20190507_1233'), + ("internet_nl_dashboard", "0027_auto_20190507_1233"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='deleted_on', + model_name="urllist", + name="deleted_on", field=models.DateTimeField(blank=True, null=True), ), migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2019, 5, 6, 15, 9, 11, 211754, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2019, 5, 6, 15, 9, 11, 211754, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0029_auto_20190515_1206.py b/dashboard/internet_nl_dashboard/migrations/0029_auto_20190515_1206.py index fa305206..2805c266 100644 --- a/dashboard/internet_nl_dashboard/migrations/0029_auto_20190515_1206.py +++ b/dashboard/internet_nl_dashboard/migrations/0029_auto_20190515_1206.py @@ -8,14 +8,16 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0028_auto_20190507_1509'), + ("internet_nl_dashboard", "0028_auto_20190507_1509"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2019, 5, 14, 12, 6, 37, 601526, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2019, 5, 14, 12, 6, 37, 601526, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0030_auto_20190515_1209.py b/dashboard/internet_nl_dashboard/migrations/0030_auto_20190515_1209.py index 508c2d30..9c91ac3a 100644 --- a/dashboard/internet_nl_dashboard/migrations/0030_auto_20190515_1209.py +++ b/dashboard/internet_nl_dashboard/migrations/0030_auto_20190515_1209.py @@ -8,14 +8,16 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0029_auto_20190515_1206'), + ("internet_nl_dashboard", "0029_auto_20190515_1206"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='scheduled_next_scan', - field=models.DateTimeField(default=datetime.datetime(2030, 1, 1, 1, 1, 1, 601526, tzinfo=datetime.timezone.utc), - help_text='An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.'), + model_name="urllist", + name="scheduled_next_scan", + field=models.DateTimeField( + default=datetime.datetime(2030, 1, 1, 1, 1, 1, 601526, tzinfo=datetime.timezone.utc), + help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0031_account_report_settings.py b/dashboard/internet_nl_dashboard/migrations/0031_account_report_settings.py index 34b5ae00..1d132547 100644 --- a/dashboard/internet_nl_dashboard/migrations/0031_account_report_settings.py +++ b/dashboard/internet_nl_dashboard/migrations/0031_account_report_settings.py @@ -7,15 +7,17 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0030_auto_20190515_1209'), + ("internet_nl_dashboard", "0030_auto_20190515_1209"), ] operations = [ migrations.AddField( - model_name='account', - name='report_settings', + model_name="account", + name="report_settings", field=jsonfield.fields.JSONField( - default={}, help_text='This stores reporting preferences: what fields are shown in the UI and so on (if any other).This field can be edited on the report page.'), + default={}, + help_text="This stores reporting preferences: what fields are shown in the UI and so on (if any other).This field can be edited on the report page.", + ), preserve_default=False, ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0032_auto_20190528_1352.py b/dashboard/internet_nl_dashboard/migrations/0032_auto_20190528_1352.py index 76f178ed..b62bd929 100644 --- a/dashboard/internet_nl_dashboard/migrations/0032_auto_20190528_1352.py +++ b/dashboard/internet_nl_dashboard/migrations/0032_auto_20190528_1352.py @@ -7,14 +7,17 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0031_account_report_settings'), + ("internet_nl_dashboard", "0031_account_report_settings"), ] operations = [ migrations.AlterField( - model_name='account', - name='report_settings', + model_name="account", + name="report_settings", field=jsonfield.fields.JSONField( - blank=True, help_text='This stores reporting preferences: what fields are shown in the UI and so on (if any other).This field can be edited on the report page.', null=True), + blank=True, + help_text="This stores reporting preferences: what fields are shown in the UI and so on (if any other).This field can be edited on the report page.", + null=True, + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0033_auto_20190604_1242.py b/dashboard/internet_nl_dashboard/migrations/0033_auto_20190604_1242.py index 8d9f6617..f45c2bb3 100644 --- a/dashboard/internet_nl_dashboard/migrations/0033_auto_20190604_1242.py +++ b/dashboard/internet_nl_dashboard/migrations/0033_auto_20190604_1242.py @@ -6,112 +6,117 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0032_auto_20190528_1352'), + ("internet_nl_dashboard", "0032_auto_20190528_1352"), ] operations = [ migrations.AddField( - model_name='urllistreport', - name='endpoint_not_applicable', + model_name="urllistreport", + name="endpoint_not_applicable", field=models.IntegerField( - default=0, help_text='Amount of things that are not applicable on this endpoint.'), + default=0, help_text="Amount of things that are not applicable on this endpoint." + ), ), migrations.AddField( - model_name='urllistreport', - name='endpoint_not_testable', + model_name="urllistreport", + name="endpoint_not_testable", field=models.IntegerField( - default=0, help_text='Amount of things that could not be tested on this endpoint.'), + default=0, help_text="Amount of things that could not be tested on this endpoint." + ), ), migrations.AddField( - model_name='urllistreport', - name='not_applicable', + model_name="urllistreport", + name="not_applicable", field=models.IntegerField(default=0), ), migrations.AddField( - model_name='urllistreport', - name='not_testable', + model_name="urllistreport", + name="not_testable", field=models.IntegerField(default=0), ), migrations.AddField( - model_name='urllistreport', - name='url_not_applicable', - field=models.IntegerField(default=0, help_text='Amount of things that are not applicable on this url.'), + model_name="urllistreport", + name="url_not_applicable", + field=models.IntegerField(default=0, help_text="Amount of things that are not applicable on this url."), ), migrations.AddField( - model_name='urllistreport', - name='url_not_testable', - field=models.IntegerField(default=0, help_text='Amount of things that could not be tested on this url.'), + model_name="urllistreport", + name="url_not_testable", + field=models.IntegerField(default=0, help_text="Amount of things that could not be tested on this url."), ), migrations.AlterField( - model_name='urllistreport', - name='endpoint_issues_high', - field=models.IntegerField(default=0, help_text='Total amount of high risk issues on this endpoint.'), + model_name="urllistreport", + name="endpoint_issues_high", + field=models.IntegerField(default=0, help_text="Total amount of high risk issues on this endpoint."), ), migrations.AlterField( - model_name='urllistreport', - name='endpoint_issues_low', - field=models.IntegerField(default=0, help_text='Total amount of low risk issues on this endpoint'), + model_name="urllistreport", + name="endpoint_issues_low", + field=models.IntegerField(default=0, help_text="Total amount of low risk issues on this endpoint"), ), migrations.AlterField( - model_name='urllistreport', - name='endpoint_issues_medium', - field=models.IntegerField(default=0, help_text='Total amount of medium risk issues on this endpoint.'), + model_name="urllistreport", + name="endpoint_issues_medium", + field=models.IntegerField(default=0, help_text="Total amount of medium risk issues on this endpoint."), ), migrations.AlterField( - model_name='urllistreport', - name='endpoint_ok', + model_name="urllistreport", + name="endpoint_ok", field=models.IntegerField( - default=0, help_text='Amount of measurements that resulted in an OK score on this endpoint.'), + default=0, help_text="Amount of measurements that resulted in an OK score on this endpoint." + ), ), migrations.AlterField( - model_name='urllistreport', - name='explained_total_url_issues', - field=models.IntegerField(default=0, help_text='Total amount of issues on endpoint level.'), + model_name="urllistreport", + name="explained_total_url_issues", + field=models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), ), migrations.AlterField( - model_name='urllistreport', - name='explained_url_issues_high', - field=models.IntegerField(default=0, help_text='Total amount of issues on endpoint level.'), + model_name="urllistreport", + name="explained_url_issues_high", + field=models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), ), migrations.AlterField( - model_name='urllistreport', - name='explained_url_issues_low', - field=models.IntegerField(default=0, help_text='Total amount of issues on endpoint level.'), + model_name="urllistreport", + name="explained_url_issues_low", + field=models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), ), migrations.AlterField( - model_name='urllistreport', - name='explained_url_issues_medium', - field=models.IntegerField(default=0, help_text='Total amount of issues on endpoint level.'), + model_name="urllistreport", + name="explained_url_issues_medium", + field=models.IntegerField(default=0, help_text="Total amount of issues on endpoint level."), ), migrations.AlterField( - model_name='urllistreport', - name='high', + model_name="urllistreport", + name="high", field=models.IntegerField(default=0), ), migrations.AlterField( - model_name='urllistreport', - name='low', + model_name="urllistreport", + name="low", field=models.IntegerField(default=0), ), migrations.AlterField( - model_name='urllistreport', - name='medium', + model_name="urllistreport", + name="medium", field=models.IntegerField(default=0), ), migrations.AlterField( - model_name='urllistreport', - name='ok', + model_name="urllistreport", + name="ok", field=models.IntegerField(default=0), ), migrations.AlterField( - model_name='urllistreport', - name='total_endpoint_issues', + model_name="urllistreport", + name="total_endpoint_issues", field=models.IntegerField( - default=0, help_text='A sum of all endpoint issues for this endpoint, it includes all high, medium and lows.'), + default=0, + help_text="A sum of all endpoint issues for this endpoint, it includes all high, medium and lows.", + ), ), migrations.AlterField( - model_name='urllistreport', - name='total_issues', + model_name="urllistreport", + name="total_issues", field=models.IntegerField(default=0), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0034_auto_20190613_0946.py b/dashboard/internet_nl_dashboard/migrations/0034_auto_20190613_0946.py index b4153f6b..a23e25f6 100644 --- a/dashboard/internet_nl_dashboard/migrations/0034_auto_20190613_0946.py +++ b/dashboard/internet_nl_dashboard/migrations/0034_auto_20190613_0946.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0033_auto_20190604_1242'), + ("internet_nl_dashboard", "0033_auto_20190604_1242"), ] operations = [ migrations.AlterField( - model_name='account', - name='internet_nl_api_password', - field=models.TextField(blank=True, help_text='New values will automatically be encrypted.', null=True), + model_name="account", + name="internet_nl_api_password", + field=models.TextField(blank=True, help_text="New values will automatically be encrypted.", null=True), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0035_auto_20190624_0712.py b/dashboard/internet_nl_dashboard/migrations/0035_auto_20190624_0712.py index 6e2c8819..0b156ecf 100644 --- a/dashboard/internet_nl_dashboard/migrations/0035_auto_20190624_0712.py +++ b/dashboard/internet_nl_dashboard/migrations/0035_auto_20190624_0712.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0034_auto_20190613_0946'), + ("internet_nl_dashboard", "0034_auto_20190613_0946"), ] operations = [ migrations.AlterField( - model_name='urllist', - name='last_manual_scan', + model_name="urllist", + name="last_manual_scan", field=models.DateTimeField(blank=True, null=True), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0036_urllistreport_average_internet_nl_score.py b/dashboard/internet_nl_dashboard/migrations/0036_urllistreport_average_internet_nl_score.py index de64746a..a272adae 100644 --- a/dashboard/internet_nl_dashboard/migrations/0036_urllistreport_average_internet_nl_score.py +++ b/dashboard/internet_nl_dashboard/migrations/0036_urllistreport_average_internet_nl_score.py @@ -6,14 +6,16 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0035_auto_20190624_0712'), + ("internet_nl_dashboard", "0035_auto_20190624_0712"), ] operations = [ migrations.AddField( - model_name='urllistreport', - name='average_internet_nl_score', + model_name="urllistreport", + name="average_internet_nl_score", field=models.FloatField( - default=0, help_text='Internet.nl scores are retrieved in point. The calculation done for that is complex and subject to change over time. Therefore it is impossible to re-calculate that score here.Instead the score is stored as a given.'), + default=0, + help_text="Internet.nl scores are retrieved in point. The calculation done for that is complex and subject to change over time. Therefore it is impossible to re-calculate that score here.Instead the score is stored as a given.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0037_auto_20191121_1408.py b/dashboard/internet_nl_dashboard/migrations/0037_auto_20191121_1408.py index 52159b79..136d9d58 100644 --- a/dashboard/internet_nl_dashboard/migrations/0037_auto_20191121_1408.py +++ b/dashboard/internet_nl_dashboard/migrations/0037_auto_20191121_1408.py @@ -7,30 +7,42 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0036_urllistreport_average_internet_nl_score'), + ("internet_nl_dashboard", "0036_urllistreport_average_internet_nl_score"), ] operations = [ migrations.AddField( - model_name='accountinternetnlscan', - name='state', - field=models.CharField(blank=True, default='', help_text='The current state', max_length=255), + model_name="accountinternetnlscan", + name="state", + field=models.CharField(blank=True, default="", help_text="The current state", max_length=255), ), migrations.AlterField( - model_name='accountinternetnlscan', - name='scan', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, - to='scanners.InternetNLScan'), + model_name="accountinternetnlscan", + name="scan", + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to="scanners.InternetNLScan" + ), ), migrations.CreateModel( - name='AccountInternetNLScanLog', + name="AccountInternetNLScanLog", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('state', models.CharField(blank=True, default='', - help_text='The state that was registered at a certain moment in time.', max_length=255)), - ('at_when', models.DateTimeField(blank=True, null=True)), - ('scan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, - to='internet_nl_dashboard.AccountInternetNLScan')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "state", + models.CharField( + blank=True, + default="", + help_text="The state that was registered at a certain moment in time.", + max_length=255, + ), + ), + ("at_when", models.DateTimeField(blank=True, null=True)), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="internet_nl_dashboard.AccountInternetNLScan" + ), + ), ], ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0038_accountinternetnlscan_state_changed_on.py b/dashboard/internet_nl_dashboard/migrations/0038_accountinternetnlscan_state_changed_on.py index 6949c763..45258405 100644 --- a/dashboard/internet_nl_dashboard/migrations/0038_accountinternetnlscan_state_changed_on.py +++ b/dashboard/internet_nl_dashboard/migrations/0038_accountinternetnlscan_state_changed_on.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0037_auto_20191121_1408'), + ("internet_nl_dashboard", "0037_auto_20191121_1408"), ] operations = [ migrations.AddField( - model_name='accountinternetnlscan', - name='state_changed_on', + model_name="accountinternetnlscan", + name="state_changed_on", field=models.DateTimeField(blank=True, null=True), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0039_accountinternetnlscan_report.py b/dashboard/internet_nl_dashboard/migrations/0039_accountinternetnlscan_report.py index 89cc7c11..addd1cf3 100644 --- a/dashboard/internet_nl_dashboard/migrations/0039_accountinternetnlscan_report.py +++ b/dashboard/internet_nl_dashboard/migrations/0039_accountinternetnlscan_report.py @@ -7,14 +7,19 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0038_accountinternetnlscan_state_changed_on'), + ("internet_nl_dashboard", "0038_accountinternetnlscan_state_changed_on"), ] operations = [ migrations.AddField( - model_name='accountinternetnlscan', - name='report', - field=models.ForeignKey(blank=True, help_text='After a scan has finished, a report is created. This points to that report so no guessing is needed to figure out what report belongs to what scan.', - null=True, on_delete=django.db.models.deletion.CASCADE, to='internet_nl_dashboard.UrlListReport'), + model_name="accountinternetnlscan", + name="report", + field=models.ForeignKey( + blank=True, + help_text="After a scan has finished, a report is created. This points to that report so no guessing is needed to figure out what report belongs to what scan.", + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="internet_nl_dashboard.UrlListReport", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0040_auto_20200508_1013.py b/dashboard/internet_nl_dashboard/migrations/0040_auto_20200508_1013.py index 8878a8bd..4815b3b5 100644 --- a/dashboard/internet_nl_dashboard/migrations/0040_auto_20200508_1013.py +++ b/dashboard/internet_nl_dashboard/migrations/0040_auto_20200508_1013.py @@ -7,25 +7,26 @@ class Migration(migrations.Migration): dependencies = [ - ('scanners', '0072_auto_20200506_1313'), - ('internet_nl_dashboard', '0039_accountinternetnlscan_report'), + ("scanners", "0072_auto_20200506_1313"), + ("internet_nl_dashboard", "0039_accountinternetnlscan_report"), ] operations = [ migrations.AddField( - model_name='accountinternetnlscan', - name='finished_on', + model_name="accountinternetnlscan", + name="finished_on", field=models.DateTimeField(blank=True, null=True), ), migrations.AddField( - model_name='accountinternetnlscan', - name='started_on', + model_name="accountinternetnlscan", + name="started_on", field=models.DateTimeField(blank=True, null=True), ), migrations.AlterField( - model_name='accountinternetnlscan', - name='scan', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, - to='scanners.InternetNLV2Scan'), + model_name="accountinternetnlscan", + name="scan", + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to="scanners.InternetNLV2Scan" + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0041_auto_20200513_1351.py b/dashboard/internet_nl_dashboard/migrations/0041_auto_20200513_1351.py index 9771b16c..e13fc6d6 100644 --- a/dashboard/internet_nl_dashboard/migrations/0041_auto_20200513_1351.py +++ b/dashboard/internet_nl_dashboard/migrations/0041_auto_20200513_1351.py @@ -7,14 +7,19 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0040_auto_20200508_1013'), + ("internet_nl_dashboard", "0040_auto_20200508_1013"), ] operations = [ migrations.AlterField( - model_name='accountinternetnlscan', - name='report', - field=models.ForeignKey(blank=True, help_text='After a scan has finished, a report is created. This points to that report so no guessing is needed to figure out what report belongs to what scan.', - null=True, on_delete=django.db.models.deletion.SET_NULL, to='internet_nl_dashboard.UrlListReport'), + model_name="accountinternetnlscan", + name="report", + field=models.ForeignKey( + blank=True, + help_text="After a scan has finished, a report is created. This points to that report so no guessing is needed to figure out what report belongs to what scan.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="internet_nl_dashboard.UrlListReport", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0042_auto_20200530_1735.py b/dashboard/internet_nl_dashboard/migrations/0042_auto_20200530_1735.py index 8c4e546b..5a10fe3a 100644 --- a/dashboard/internet_nl_dashboard/migrations/0042_auto_20200530_1735.py +++ b/dashboard/internet_nl_dashboard/migrations/0042_auto_20200530_1735.py @@ -6,23 +6,23 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0041_auto_20200513_1351'), + ("internet_nl_dashboard", "0041_auto_20200513_1351"), ] operations = [ migrations.AddField( - model_name='urllistreport', - name='endpoint_error_in_test', - field=models.IntegerField(default=0, help_text='Amount of errors in tests performed on this endpoint.'), + model_name="urllistreport", + name="endpoint_error_in_test", + field=models.IntegerField(default=0, help_text="Amount of errors in tests performed on this endpoint."), ), migrations.AddField( - model_name='urllistreport', - name='error_in_test', + model_name="urllistreport", + name="error_in_test", field=models.IntegerField(default=0), ), migrations.AddField( - model_name='urllistreport', - name='url_error_in_test', - field=models.IntegerField(default=0, help_text='Amount of errors in tests on this url.'), + model_name="urllistreport", + name="url_error_in_test", + field=models.IntegerField(default=0, help_text="Amount of errors in tests on this url."), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0043_auto_20201006_1309.py b/dashboard/internet_nl_dashboard/migrations/0043_auto_20201006_1309.py index 802ef0a5..2f0fefb8 100644 --- a/dashboard/internet_nl_dashboard/migrations/0043_auto_20201006_1309.py +++ b/dashboard/internet_nl_dashboard/migrations/0043_auto_20201006_1309.py @@ -7,25 +7,31 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0042_auto_20200530_1735'), + ("internet_nl_dashboard", "0042_auto_20200530_1735"), ] operations = [ migrations.AddField( - model_name='dashboarduser', - name='mail_preferred_language', - field=django_countries.fields.CountryField(default='EN', max_length=2), + model_name="dashboarduser", + name="mail_preferred_language", + field=django_countries.fields.CountryField(default="EN", max_length=2), ), migrations.AddField( - model_name='dashboarduser', - name='mail_preferred_mail_address', + model_name="dashboarduser", + name="mail_preferred_mail_address", field=models.EmailField( - blank=True, help_text='This address can deviate from the account mail address for password resets and other account features.', max_length=254, null=True), + blank=True, + help_text="This address can deviate from the account mail address for password resets and other account features.", + max_length=254, + null=True, + ), ), migrations.AddField( - model_name='dashboarduser', - name='mail_send_mail_after_scan_finished', + model_name="dashboarduser", + name="mail_send_mail_after_scan_finished", field=models.BooleanField( - default=False, help_text='After a scan is finished, an e-mail is sent informing the user that a report is ready.'), + default=False, + help_text="After a scan is finished, an e-mail is sent informing the user that a report is ready.", + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0044_dashboarduser_mail_after_mail_unsubscribe_code.py b/dashboard/internet_nl_dashboard/migrations/0044_dashboarduser_mail_after_mail_unsubscribe_code.py index 78b03bfe..6fb5ec59 100644 --- a/dashboard/internet_nl_dashboard/migrations/0044_dashboarduser_mail_after_mail_unsubscribe_code.py +++ b/dashboard/internet_nl_dashboard/migrations/0044_dashboarduser_mail_after_mail_unsubscribe_code.py @@ -6,14 +6,17 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0043_auto_20201006_1309'), + ("internet_nl_dashboard", "0043_auto_20201006_1309"), ] operations = [ migrations.AddField( - model_name='dashboarduser', - name='mail_after_mail_unsubscribe_code', + model_name="dashboarduser", + name="mail_after_mail_unsubscribe_code", field=models.CharField( - default='', help_text='This is autofilled when sending an e-mail. The user can use this code to set mail_send_mail_after_scan_finished to false without logging in.', max_length=255), + default="", + help_text="This is autofilled when sending an e-mail. The user can use this code to set mail_send_mail_after_scan_finished to false without logging in.", + max_length=255, + ), ), ] diff --git a/dashboard/internet_nl_dashboard/migrations/0045_auto_20201027_1039.py b/dashboard/internet_nl_dashboard/migrations/0045_auto_20201027_1039.py index 5ebbf0d2..2be39018 100644 --- a/dashboard/internet_nl_dashboard/migrations/0045_auto_20201027_1039.py +++ b/dashboard/internet_nl_dashboard/migrations/0045_auto_20201027_1039.py @@ -6,14 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('internet_nl_dashboard', '0044_dashboarduser_mail_after_mail_unsubscribe_code'), + ("internet_nl_dashboard", "0044_dashboarduser_mail_after_mail_unsubscribe_code"), ] operations = [ migrations.AlterField( - model_name='dashboarduser', - name='mail_after_mail_unsubscribe_code', + model_name="dashboarduser", + name="mail_after_mail_unsubscribe_code", field=models.CharField( - blank=True, default='', help_text='This is autofilled when sending an e-mail. The user can use this code to set mail_send_mail_after_scan_finished to false without logging in.', max_length=255), + blank=True, + default="", + help_text="This is autofilled when sending an e-mail. The user can use this code to set mail_send_mail_after_scan_finished to false without logging in.", + max_length=255, + ), ), ] diff --git a/dashboard/internet_nl_dashboard/models.py b/dashboard/internet_nl_dashboard/models.py index 5167d62d..fdf6540f 100644 --- a/dashboard/internet_nl_dashboard/models.py +++ b/dashboard/internet_nl_dashboard/models.py @@ -25,22 +25,14 @@ class Account(models.Model): An account is the entity that start scans. Multiple people can manage the account. """ - name = models.CharField( - max_length=120, - blank=True, - null=True, - help_text="" - ) + name = models.CharField(max_length=120, blank=True, null=True, help_text="") enable_scans = models.BooleanField( default=True, ) internet_nl_api_username = models.CharField( - max_length=255, - blank=True, - null=True, - help_text="Internet.nl API Username" + max_length=255, blank=True, null=True, help_text="Internet.nl API Username" ) # BinaryFields become MemoryView objects in postgres, which handle differently than normal strings. @@ -48,21 +40,16 @@ class Account(models.Model): # bytes are stored as a string, while ugly, it might just function better and more consistent. # https://code.djangoproject.com/ticket/27813 internet_nl_api_password = models.TextField( - blank=True, - null=True, - help_text="New values will automatically be encrypted.", - editable=True + blank=True, null=True, help_text="New values will automatically be encrypted.", editable=True ) - can_connect_to_internet_nl_api = models.BooleanField( - default=False - ) + can_connect_to_internet_nl_api = models.BooleanField(default=False) report_settings = JSONField( help_text="This stores reporting preferences: what fields are shown in the UI and so on (if any other)." - "This field can be edited on the report page.", + "This field can be edited on the report page.", null=True, - blank=True + blank=True, ) """ @@ -83,7 +70,7 @@ def connect_to_internet_nl_api(username: str, password: str): config.CREDENTIAL_CHECK_URL, auth=HTTPBasicAuth(username, password), # a massive timeout for a large file. - timeout=(5, 5) + timeout=(5, 5), ) # Any status code means the account is not valid. @@ -97,12 +84,12 @@ def connect_to_internet_nl_api(username: str, password: str): def decrypt_password(self): if not self.internet_nl_api_password: - raise ValueError('Password was not set.') + raise ValueError("Password was not set.") fernet = Fernet(settings.FIELD_ENCRYPTION_KEY) # Convert the string back to bytes again is not beautiful. But it's a bit more reliable than # storing the encrypted password in 'bytes', which somewhere goes wrong. - return fernet.decrypt(bytes(self.internet_nl_api_password[2:-1], encoding='UTF-8')).decode('utf-8') + return fernet.decrypt(bytes(self.internet_nl_api_password[2:-1], encoding="UTF-8")).decode("utf-8") def __str__(self): return f"{self.name}" @@ -114,10 +101,8 @@ class DashboardUser(models.Model): one to one relation with an extended model. An additional benefit/feature is that we can easily switch what user is connected to what account. """ - user = models.OneToOneField( - User, - on_delete=models.CASCADE - ) + + user = models.OneToOneField(User, on_delete=models.CASCADE) account = models.ForeignKey( Account, @@ -126,15 +111,15 @@ class DashboardUser(models.Model): mail_preferred_mail_address = models.EmailField( help_text="This address can deviate from the account mail address for password resets and other account" - " features.", + " features.", null=True, - blank=True + blank=True, ) - mail_preferred_language = CountryField(default='EN') + mail_preferred_language = CountryField(default="EN") mail_send_mail_after_scan_finished = models.BooleanField( default=False, - help_text="After a scan is finished, an e-mail is sent informing the user that a report is ready." + help_text="After a scan is finished, an e-mail is sent informing the user that a report is ready.", ) mail_after_mail_unsubscribe_code = models.CharField( @@ -142,7 +127,7 @@ class DashboardUser(models.Model): default="", blank=True, help_text="This is autofilled when sending an e-mail. The user can use this code to set " - "mail_send_mail_after_scan_finished to false without logging in." + "mail_send_mail_after_scan_finished to false without logging in.", ) notes = models.TextField( @@ -162,21 +147,12 @@ class UrlList(models.Model): """ name = models.CharField( - max_length=120, - help_text="Name of the UrlList, for example name of the organization in it." + max_length=120, help_text="Name of the UrlList, for example name of the organization in it." ) - account = models.ForeignKey( - Account, - on_delete=models.CASCADE, - help_text="Who owns and manages this urllist." - ) + account = models.ForeignKey(Account, on_delete=models.CASCADE, help_text="Who owns and manages this urllist.") - urls = models.ManyToManyField( - through="TaggedUrlInUrllist", - to=Url, - related_name='urls_in_dashboard_list_2' - ) + urls = models.ManyToManyField(through="TaggedUrlInUrllist", to=Url, related_name="urls_in_dashboard_list_2") enable_scans = models.BooleanField( default=True, @@ -185,49 +161,43 @@ class UrlList(models.Model): scan_type = models.CharField( max_length=4, choices=( - ('web', 'web'), - ('mail', 'mail'), - ('all', 'all'), + ("web", "web"), + ("mail", "mail"), + ("all", "all"), ), - default='web', + default="web", ) automated_scan_frequency = models.CharField( max_length=30, choices=( - ('disabled', 'disabled'), - ('every half year', 'every half year'), - ('at the start of every quarter', 'at the start of every quarter'), - ('every 1st day of the month', 'every 1st day of the month'), - ('twice per month', 'twice per month'), + ("disabled", "disabled"), + ("every half year", "every half year"), + ("at the start of every quarter", "at the start of every quarter"), + ("every 1st day of the month", "every 1st day of the month"), + ("twice per month", "twice per month"), ), - default='disabled', - help_text="At what moment should the scan start?" + default="disabled", + help_text="At what moment should the scan start?", ) scheduled_next_scan = models.DateTimeField( help_text="An indication at what moment the scan will be started. The scan can take a while, thus this does " - "not tell you when a scan will be finished. All dates in the past will be scanned and updated.", - default=datetime(2030, 1, 1, 1, 1, 1, 601526, tzinfo=timezone.utc) + "not tell you when a scan will be finished. All dates in the past will be scanned and updated.", + default=datetime(2030, 1, 1, 1, 1, 1, 601526, tzinfo=timezone.utc), ) is_deleted = models.BooleanField( default=False, ) - deleted_on = models.DateTimeField( - null=True, - blank=True - ) + deleted_on = models.DateTimeField(null=True, blank=True) - last_manual_scan = models.DateTimeField( - null=True, - blank=True - ) + last_manual_scan = models.DateTimeField(null=True, blank=True) enable_report_sharing_page = models.BooleanField( default=False, - help_text="When true there will be page under the list-id that shows all reports that are shared publicly." + help_text="When true there will be page under the list-id that shows all reports that are shared publicly.", ) # will be available under: /public/account-id/list-id/latest @@ -237,17 +207,17 @@ class UrlList(models.Model): # and /public/account-id/list-name-slug/report-id automatically_share_new_reports = models.BooleanField( help_text="Sharing can be disabled and re-enabled where the report code and the share code (password) " - "stay the same. Sharing means that all new reports will be made public under a set of standard urls.", - default=False + "stay the same. Sharing means that all new reports will be made public under a set of standard urls.", + default=False, ) default_public_share_code_for_new_reports = models.CharField( max_length=64, help_text="An unencrypted share code that can be seen by all users in an account. Can be modified by all. " - "New reports get this code set automatically. You can change this per report. An empty field " - "means no share code and the report is accessible publicly.", + "New reports get this code set automatically. You can change this per report. An empty field " + "means no share code and the report is accessible publicly.", blank=True, - default="" + default="", ) def __str__(self): @@ -255,14 +225,14 @@ def __str__(self): def is_due_for_scanning(self) -> bool: # when disabled, will not be scanned automatically anymore. - if self.automated_scan_frequency == 'disabled': + if self.automated_scan_frequency == "disabled": return False return datetime.now(timezone.utc) > self.scheduled_next_scan def renew_scan_moment(self) -> None: self.scheduled_next_scan = determine_next_scan_moment(self.automated_scan_frequency) - self.save(update_fields=['scheduled_next_scan']) + self.save(update_fields=["scheduled_next_scan"]) # @pysnooper.snoop() def is_scan_now_available(self) -> bool: @@ -300,7 +270,7 @@ def is_scan_now_available(self) -> bool: # finished_on = last_scan.scan.finished_on \ # if last_scan.scan.finished_on else datetime.now(timezone.utc) - timedelta(days=30) # and finished_on < yesterday - if last_scan.state in ['finished', 'cancelled']: + if last_scan.state in ["finished", "cancelled"]: # log.debug("Sc an now available: last scan finished over 24 hours ago on list %s" % self) return True @@ -315,9 +285,7 @@ class TaggedUrlInUrllist(models.Model): class Meta: db_table = "internet_nl_dashboard_urllist_x_tagged_url" - unique_together = [ - ['urllist', 'url'] - ] + unique_together = [["urllist", "url"]] class SubdomainDiscoveryScan(models.Model): @@ -330,16 +298,12 @@ class SubdomainDiscoveryScan(models.Model): Flow: requested -> scanning -> finished|error|cancelled """ + urllist = models.ForeignKey(UrlList, on_delete=models.CASCADE) state = models.CharField( - max_length=20, - default="requested", - help_text="Name of the UrlList, for example name of the organization in it." - ) - state_changed_on = models.DateTimeField( - blank=True, - null=True + max_length=20, default="requested", help_text="Name of the UrlList, for example name of the organization in it." ) + state_changed_on = models.DateTimeField(blank=True, null=True) state_message = models.CharField(max_length=200, blank=True) # Archive what subdomains have been discovered and have been added to the list. This can be an enormous # list. It's handy for inspection purposes. @@ -356,38 +320,38 @@ def determine_next_scan_moment(preference: str) -> datetime: now = datetime.now(timezone.utc) returned = datetime(year=now.year, month=1, day=1, tzinfo=timezone.utc) - if preference == 'disabled': + if preference == "disabled": # far, far in the future, so it will not be scanned and probably will be re-calculated. 24 years... return now + timedelta(days=9000) # months are base 1: january = 1 etc. - if preference == 'every half year': + if preference == "every half year": # every half year: first upcoming 1 july or 1 january return returned.replace(month=7) if now.month in [1, 2, 3, 4, 5, 6] else returned.replace(year=now.year + 1) - if preference == 'at the start of every quarter': + if preference == "at the start of every quarter": # at the start of every quarter: 1 january, 1 april, 1 juli, 1 october pick = { - 1: {'year': now.year, 'month': 4}, - 2: {'year': now.year, 'month': 4}, - 3: {'year': now.year, 'month': 4}, - 4: {'year': now.year, 'month': 7}, - 5: {'year': now.year, 'month': 7}, - 6: {'year': now.year, 'month': 7}, - 7: {'year': now.year, 'month': 10}, - 8: {'year': now.year, 'month': 10}, - 9: {'year': now.year, 'month': 10}, - 10: {'year': now.year + 1, 'month': 1}, - 11: {'year': now.year + 1, 'month': 1}, - 12: {'year': now.year + 1, 'month': 1}, + 1: {"year": now.year, "month": 4}, + 2: {"year": now.year, "month": 4}, + 3: {"year": now.year, "month": 4}, + 4: {"year": now.year, "month": 7}, + 5: {"year": now.year, "month": 7}, + 6: {"year": now.year, "month": 7}, + 7: {"year": now.year, "month": 10}, + 8: {"year": now.year, "month": 10}, + 9: {"year": now.year, "month": 10}, + 10: {"year": now.year + 1, "month": 1}, + 11: {"year": now.year + 1, "month": 1}, + 12: {"year": now.year + 1, "month": 1}, } - return returned.replace(year=pick[now.month]['year'], month=pick[now.month]['month']) + return returned.replace(year=pick[now.month]["year"], month=pick[now.month]["month"]) - if preference == 'every 1st day of the month': + if preference == "every 1st day of the month": # every 1st day of the month: 1 january, 1 february, etc. return returned.replace(year=now.year + 1) if now.month == 12 else returned.replace(month=now.month + 1) - if preference == 'twice per month': + if preference == "twice per month": # twice per month: 1 january, 1 january + 2 weeks, 1 february, 1 february + 2 weeks, etc # since the 14'th day never causes a month or year rollover, we can simply schedule for the 15th day. # note: range is not used because range is _to_ a certain moment. @@ -397,7 +361,7 @@ def determine_next_scan_moment(preference: str) -> datetime: # otherwise exactly the same as the 1st day of every month return returned.replace(year=now.year + 1) if now.month == 12 else returned.replace(month=now.month + 1) - raise ValueError(f'String {preference} could not be translated to a scan moment.') + raise ValueError(f"String {preference} could not be translated to a scan moment.") class UploadLog(models.Model): @@ -406,21 +370,21 @@ class UploadLog(models.Model): blank=True, null=True, help_text="The original filename of the file that has been uploaded. Django appends a random string if the " - "file already exists. This is a reconstruction of the original filename and may not be 100% accurate." + "file already exists. This is a reconstruction of the original filename and may not be 100% accurate.", ) internal_filename = models.CharField( max_length=255, blank=True, null=True, - help_text="Generated filename by Django. This can be used to find specific files for debugging purposes." + help_text="Generated filename by Django. This can be used to find specific files for debugging purposes.", ) status = models.CharField( max_length=255, blank=True, null=True, - help_text="If the upload was successful or not. Might contain 'success' or 'error'." + help_text="If the upload was successful or not. Might contain 'success' or 'error'.", ) message = models.CharField( @@ -428,34 +392,24 @@ class UploadLog(models.Model): blank=True, null=True, help_text="This message gives more specific information about what happened. For example, it might be the " - "case that a file has been rejected because it had the wrong filetype etc." + "case that a file has been rejected because it had the wrong filetype etc.", ) - upload_date = models.DateTimeField( - blank=True, - null=True - ) + upload_date = models.DateTimeField(blank=True, null=True) filesize = models.PositiveIntegerField( default=0, blank=False, null=False, - help_text="Gives an indication if your local file has changed (different size). The size is in bytes." + help_text="Gives an indication if your local file has changed (different size). The size is in bytes.", ) user = models.ForeignKey( - DashboardUser, - on_delete=models.CASCADE, - help_text="What user performed this upload.", - blank=True, - null=True + DashboardUser, on_delete=models.CASCADE, help_text="What user performed this upload.", blank=True, null=True ) percentage = models.PositiveIntegerField( - default=0, - blank=True, - null=True, - help_text="The percentage of domains added in the upload." + default=0, blank=True, null=True, help_text="The percentage of domains added in the upload." ) @@ -481,23 +435,24 @@ class UrlListReport(SeriesOfUrlsReportMixin): # pylint: disable=too-many-ancest Also this should not know too much about different scanners. In OO fashion, it should ask a scanner to explain why something is the way it is (over time). """ + urllist = models.ForeignKey(UrlList, on_delete=models.CASCADE) average_internet_nl_score = models.FloatField( help_text="Internet.nl scores are retrieved in point. The calculation done for that is complex and " - "subject to change over time. Therefore it is impossible to re-calculate that score here." - "Instead the score is stored as a given.", + "subject to change over time. Therefore it is impossible to re-calculate that score here." + "Instead the score is stored as a given.", default=0, ) # the urllist might change type of scan, or perform both web and mail scan. So store the type of report here. # This is web or mail. todo: persist through the application, change responses. - report_type = models.CharField(default='web', max_length=10) + report_type = models.CharField(default="web", max_length=10) is_publicly_shared = models.BooleanField( help_text="Sharing can be disabled and re-enabled where the report code and the share code (password) " - "stay the same.", - default=False + "stay the same.", + default=False, ) public_report_code = models.CharField( max_length=64, @@ -505,19 +460,19 @@ class UrlListReport(SeriesOfUrlsReportMixin): # pylint: disable=too-many-ancest # not unique in database, but enforced in software. Codes of deleted reports might be reused. unique=False, blank=True, - default="" + default="", ) public_share_code = models.CharField( max_length=64, help_text="An unencrypted share code that can be seen by all users in an account. Can be modified by all.", blank=True, - default="" + default="", ) is_shared_on_homepage = models.BooleanField( help_text="A public report can also be shared on the homepage with a link. Can only be shared on the homepage " - "if the report is publicly shared. This is currently admin only.", - default=False + "if the report is publicly shared. This is currently admin only.", + default=False, ) class Meta: @@ -539,11 +494,13 @@ def get_previous_report_from_this_list(self): :return: """ try: - return UrlListReport.objects.all().filter( - urllist=self.urllist, - report_type=self.report_type, - at_when__lt=self.at_when - ).exclude(id=self.id).defer('calculation').latest() + return ( + UrlListReport.objects.all() + .filter(urllist=self.urllist, report_type=self.report_type, at_when__lt=self.at_when) + .exclude(id=self.id) + .defer("calculation") + .latest() + ) except UrlListReport.DoesNotExist: return None @@ -562,9 +519,8 @@ class AccountInternetNLScan(models.Model): scan = models.ForeignKey( InternetNLV2Scan, on_delete=models.CASCADE, - # When there is no scan registered at internet.nl, but the scan has to show up as requested - null=True + null=True, ) urllist = models.ForeignKey( @@ -573,27 +529,13 @@ class AccountInternetNLScan(models.Model): ) # The current state of the scan. - state = models.CharField( - max_length=255, - blank=True, - default="", - help_text="The current state" - ) + state = models.CharField(max_length=255, blank=True, default="", help_text="The current state") - started_on = models.DateTimeField( - blank=True, - null=True - ) + started_on = models.DateTimeField(blank=True, null=True) - finished_on = models.DateTimeField( - blank=True, - null=True - ) + finished_on = models.DateTimeField(blank=True, null=True) - state_changed_on = models.DateTimeField( - blank=True, - null=True - ) + state_changed_on = models.DateTimeField(blank=True, null=True) report = models.ForeignKey( UrlListReport, @@ -601,7 +543,7 @@ class AccountInternetNLScan(models.Model): blank=True, on_delete=models.SET_NULL, help_text="After a scan has finished, a report is created. This points to that report so no guessing " - "is needed to figure out what report belongs to what scan." + "is needed to figure out what report belongs to what scan.", ) @property @@ -619,13 +561,7 @@ class AccountInternetNLScanLog(models.Model): ) state = models.CharField( - max_length=255, - blank=True, - default="", - help_text="The state that was registered at a certain moment in time." + max_length=255, blank=True, default="", help_text="The state that was registered at a certain moment in time." ) - at_when = models.DateTimeField( - blank=True, - null=True - ) + at_when = models.DateTimeField(blank=True, null=True) diff --git a/dashboard/internet_nl_dashboard/scanners/scan_internet_nl_per_account.py b/dashboard/internet_nl_dashboard/scanners/scan_internet_nl_per_account.py index 8d6a08a8..3476dbf7 100644 --- a/dashboard/internet_nl_dashboard/scanners/scan_internet_nl_per_account.py +++ b/dashboard/internet_nl_dashboard/scanners/scan_internet_nl_per_account.py @@ -22,8 +22,13 @@ from dashboard.internet_nl_dashboard.logic.mail import email_configration_is_correct, send_scan_finished_mails from dashboard.internet_nl_dashboard.logic.report import optimize_calculation_and_add_statistics from dashboard.internet_nl_dashboard.logic.urllist_dashboard_report import create_dashboard_report -from dashboard.internet_nl_dashboard.models import (AccountInternetNLScan, AccountInternetNLScanLog, UrlList, - UrlListReport) +from dashboard.internet_nl_dashboard.models import ( + AccountInternetNLScan, + AccountInternetNLScanLog, + UrlList, + UrlListReport, +) + # done: create more flexible filters # done: map mail scans to an endpoint (changed the scanner for it) # done: make nice tracking name for internet nl that is echoed in the scan results. @@ -48,13 +53,13 @@ def create_api_settings(v2_scan_id: InternetNLV2Scan) -> Dict[str, Union[str, int]]: scan = InternetNLV2Scan.objects.all().filter(id=v2_scan_id).first() if not scan: - log.error(f'Did not find an internetnLV2scan with id {v2_scan_id}') + log.error(f"Did not find an internetnLV2scan with id {v2_scan_id}") return InternetNLApiSettings().__dict__ # type: ignore # figure out which AccountInternetNLScan object uses this scan. Retrieve the credentials from that account. account_scan = AccountInternetNLScan.objects.all().filter(scan=scan).first() if not account_scan: - log.error(f'Could not find accountscan from scan {scan}') + log.error(f"Could not find accountscan from scan {scan}") return InternetNLApiSettings().__dict__ # type: ignore apisettings = InternetNLApiSettings() @@ -76,7 +81,7 @@ def create_api_settings(v2_scan_id: InternetNLV2Scan) -> Dict[str, Union[str, in internet_nl_websecmap.create_api_settings = create_api_settings -@app.task(queue='storage') +@app.task(queue="storage") def initialize_scan(urllist_id: int, manual_or_scheduled: str = "scheduled") -> int: urllist = UrlList.objects.all().filter(id=urllist_id).first() if not urllist: @@ -97,8 +102,9 @@ def create_scan(internal_scan_type: str, urllist: UrlList, manual_or_scheduled: new_scan = InternetNLV2Scan() new_scan.type = internal_scan_type new_scan.save() - internet_nl_websecmap.update_state(new_scan.id, "requested and empty", - "requested a scan to be performed on internet.nl api") + internet_nl_websecmap.update_state( + new_scan.id, "requested and empty", "requested a scan to be performed on internet.nl api" + ) # We need to store the scan type in the InternetNLV2Scan at creation, because the type in the list might change: accountinternetnlscan = AccountInternetNLScan() @@ -113,12 +119,12 @@ def create_scan(internal_scan_type: str, urllist: UrlList, manual_or_scheduled: update_state("requested", accountinternetnlscan.id) # Sprinkling an activity stream action. - action.send(urllist.account, verb=f'started {manual_or_scheduled} scan', target=accountinternetnlscan, public=False) + action.send(urllist.account, verb=f"started {manual_or_scheduled} scan", target=accountinternetnlscan, public=False) return accountinternetnlscan.id -@app.task(queue='storage') +@app.task(queue="storage") def check_running_dashboard_scans(**kwargs) -> Task: """ Gets status on all running scans from internet, per account. @@ -132,7 +138,7 @@ def check_running_dashboard_scans(**kwargs) -> Task: """ lock_timeout = 300 - lock_name = 'check_running_dashboard_scans' + lock_name = "check_running_dashboard_scans" # You now have five minutes to perform database operations, which should have happened in 1 second. if temporary_file_lock(lock_name, timeout_in_seconds=lock_timeout): @@ -140,10 +146,11 @@ def check_running_dashboard_scans(**kwargs) -> Task: scans = AccountInternetNLScan.objects.all() scans = add_model_filter(scans, **kwargs) else: - scans = AccountInternetNLScan.objects.all().exclude( - Q(state="finished") - | Q(state__startswith="error") - | Q(state__startswith="cancelled")).only('id') + scans = ( + AccountInternetNLScan.objects.all() + .exclude(Q(state="finished") | Q(state__startswith="error") | Q(state__startswith="cancelled")) + .only("id") + ) log.debug(f"Checking the state of scan {scans}.") tasks = [progress_running_scan(scan.id) for scan in scans] @@ -217,12 +224,10 @@ def progress_running_scan(scan_id: int) -> Task: "skipped sending mail: no e-mail addresses associated with account": finishing_scan, "skipped sending mail: no mail server configured": finishing_scan, # "finished" - # handle error situations of the scan in websecmap: "network_error": continue_running_scan, "configuration_error": continue_running_scan, "timeout": continue_running_scan, - # monitors on active states: "discovering endpoints": monitor_timeout, "retrieving scannable urls": monitor_timeout, @@ -247,13 +252,23 @@ def recover_and_retry(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: - log.warning(f'Trying to recover_and_retry with unknown scan: {scan_id}.') + log.warning(f"Trying to recover_and_retry with unknown scan: {scan_id}.") return group([]) - valid_states = ['requested', 'discovered endpoints', 'retrieved scannable urls', 'registered scan at internet.nl', - 'registered', "running scan", "scan results ready", "scan results stored", "created report", - "sent mail", "skipped sending mail: no e-mail addresses associated with account", - "skipped sending mail: no mail server configured"] + valid_states = [ + "requested", + "discovered endpoints", + "retrieved scannable urls", + "registered scan at internet.nl", + "registered", + "running scan", + "scan results ready", + "scan results stored", + "created report", + "sent mail", + "skipped sending mail: no e-mail addresses associated with account", + "skipped sending mail: no mail server configured", + ] error_states = ["network_error", "configuration_error", "timeout"] if scan.state in valid_states: @@ -261,16 +276,19 @@ def recover_and_retry(scan_id: int): return group([]) # get the latest valid state from the scan log: - latest_valid = AccountInternetNLScanLog.objects.all().filter( - scan=scan, state__in=valid_states).order_by('-id').first() + latest_valid = ( + AccountInternetNLScanLog.objects.all().filter(scan=scan, state__in=valid_states).order_by("-id").first() + ) if not latest_valid: - log.error('Trying to recover from a scan that has no log history.') + log.error("Trying to recover from a scan that has no log history.") return group([]) log.warning(f"No valid rollback state for scan {scan_id}.") - log.debug(f"AccountInternetNLScan scan #{scan.id} is rolled back to retry from " - f"'{scan.state}' to '{latest_valid.state}'.") + log.debug( + f"AccountInternetNLScan scan #{scan.id} is rolled back to retry from " + f"'{scan.state}' to '{latest_valid.state}'." + ) if scan.state in error_states: update_state(latest_valid.state, scan.id) @@ -286,7 +304,7 @@ def recover_and_retry(scan_id: int): def handle_unknown_state(scan_id): # probably nothing to be done... - log.warning(f'Scan {scan_id} is in unknown state. It will not progress.') + log.warning(f"Scan {scan_id} is in unknown state. It will not progress.") return group([]) @@ -297,21 +315,18 @@ def discovering_endpoints(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: - log.warning(f'Trying to discovering_endpoints with unknown scan: {scan_id}.') + log.warning(f"Trying to discovering_endpoints with unknown scan: {scan_id}.") return group([]) - return ( - dns_endpoints.compose_discover_task(**{ - 'urls_filter': {'urls_in_dashboard_list_2__id': scan.urllist.id, 'is_dead': False, - 'not_resolvable': False}}) - | update_state.si("discovered endpoints", scan.id) - ) + return dns_endpoints.compose_discover_task( + **{"urls_filter": {"urls_in_dashboard_list_2__id": scan.urllist.id, "is_dead": False, "not_resolvable": False}} + ) | update_state.si("discovered endpoints", scan.id) def retrieving_scannable_urls(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan or not scan.scan: - log.warning(f'Trying to retrieving_scannable_urls with unknown scan: {scan_id}.') + log.warning(f"Trying to retrieving_scannable_urls with unknown scan: {scan_id}.") return group([]) # This step tries to prevent API calls with an empty list of urls. @@ -332,7 +347,7 @@ def registering_scan_at_internet_nl(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan or not scan.scan: - log.warning(f'Trying to registering_scan_at_internet_nl with unknown scan: {scan_id}.') + log.warning(f"Trying to registering_scan_at_internet_nl with unknown scan: {scan_id}.") return group([]) # mail = websecmap, mail_dashboard = internet.nl dashboard, web is the same on both. Mail here is a fallback @@ -342,29 +357,30 @@ def registering_scan_at_internet_nl(scan_id: int): # auto saved. scan.scan.subject_urls.set(get_relevant_urls(scan.urllist.id, relevant_endpoint_types[scan.scan.type])) - internet_nl_websecmap.update_state( - scan.scan.id, "requested", "requested a scan to be performed on internet.nl api") + internet_nl_websecmap.update_state(scan.scan.id, "requested", "requested a scan to be performed on internet.nl api") # use our own tracking information for this scan, based on #451 # max length = 255. tracking_info = { # add the configured tracking name: - 'origin': constance_cached_value("INTERNET_NL_SCAN_TRACKING_NAME")[:40], # + 6 + "origin": constance_cached_value("INTERNET_NL_SCAN_TRACKING_NAME")[:40], # + 6 # lists: - 'list': { # + 4 - 'id': scan.urllist.id, # 6 chars + 2 - 'name': scan.urllist.name[:80], # 60 + 4 - 'interval': scan.urllist.automated_scan_frequency, # 30 chars + 8 - 'account': { # + 6 - 'id': scan.urllist.account.id, # 6 chars + 2 - 'name': scan.urllist.account.name[:50], # 40 + 4 - } + "list": { # + 4 + "id": scan.urllist.id, # 6 chars + 2 + "name": scan.urllist.name[:80], # 60 + 4 + "interval": scan.urllist.automated_scan_frequency, # 30 chars + 8 + "account": { # + 6 + "id": scan.urllist.account.id, # 6 chars + 2 + "name": scan.urllist.account.name[:50], # 40 + 4 + }, }, } # total json overhead is about 40 characters. - return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id, tracking_info=json.dumps(tracking_info)) - | copy_state_from_websecmap_scan.si(scan.id)) + return chain( + internet_nl_websecmap.progress_running_scan(scan.scan.id, tracking_info=json.dumps(tracking_info)) + | copy_state_from_websecmap_scan.si(scan.id) + ) def running_scan(scan_id: int): @@ -372,22 +388,20 @@ def running_scan(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan or not scan.scan: - log.warning(f'Trying to running_scan with unknown scan: {scan_id}.') + log.warning(f"Trying to running_scan with unknown scan: {scan_id}.") return group([]) - return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) - | copy_state_from_websecmap_scan.si(scan.id)) + return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) | copy_state_from_websecmap_scan.si(scan.id)) def continue_running_scan(scan_id: int): # Used to progress in error situations. scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan or not scan.scan: - log.warning(f'Trying to continue_running_scan with unknown scan: {scan_id}.') + log.warning(f"Trying to continue_running_scan with unknown scan: {scan_id}.") return group([]) - return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) - | copy_state_from_websecmap_scan.si(scan.id)) + return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) | copy_state_from_websecmap_scan.si(scan.id)) def storing_scan_results(scan_id: int): @@ -395,11 +409,10 @@ def storing_scan_results(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan or not scan.scan: - log.warning(f'Trying to storing_scan_results with unknown scan: {scan_id}.') + log.warning(f"Trying to storing_scan_results with unknown scan: {scan_id}.") return group([]) - return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) - | copy_state_from_websecmap_scan.si(scan.id)) + return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) | copy_state_from_websecmap_scan.si(scan.id)) def processing_scan_results(scan_id: int): @@ -407,11 +420,10 @@ def processing_scan_results(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan or not scan.scan: - log.warning(f'Trying to processing_scan_results with unknown scan: {scan_id}.') + log.warning(f"Trying to processing_scan_results with unknown scan: {scan_id}.") return group([]) - return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) - | copy_state_from_websecmap_scan.si(scan.id)) + return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id) | copy_state_from_websecmap_scan.si(scan.id)) @app.task(queue="storage", ignore_result=True) @@ -449,18 +461,20 @@ def creating_report(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: - log.warning(f'Trying to creating_report with unknown scan: {scan_id}.') + log.warning(f"Trying to creating_report with unknown scan: {scan_id}.") return group([]) # Note that calling 'timezone.now()' at canvas creation time, means that you'll have a date in the past # at the moment the function is actually called. If you need accurate time in the function, make sure the # function calls 'timezone.now()' when the function is run. - return (group(recreate_url_reports(list(scan.urllist.urls.all().values_list('id', flat=True)))) - | create_dashboard_report.si(scan.id) - | connect_urllistreport_to_accountinternetnlscan.s(scan.id) - | upgrade_report_with_statistics.s() - | upgrade_report_with_unscannable_urls.s(scan.id) - | update_state.si("created report", scan.id)) + return ( + group(recreate_url_reports(list(scan.urllist.urls.all().values_list("id", flat=True)))) + | create_dashboard_report.si(scan.id) + | connect_urllistreport_to_accountinternetnlscan.s(scan.id) + | upgrade_report_with_statistics.s() + | upgrade_report_with_unscannable_urls.s(scan.id) + | update_state.si("created report", scan.id) + ) def sending_mail(scan_id: int): @@ -468,17 +482,16 @@ def sending_mail(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: - log.warning(f'Trying to sending_mail with unknown scan: {scan_id}.') + log.warning(f"Trying to sending_mail with unknown scan: {scan_id}.") return group([]) - return (send_after_scan_mail.si(scan.id) - | update_state.s(scan.id)) + return send_after_scan_mail.si(scan.id) | update_state.s(scan.id) def finishing_scan(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: - log.warning(f'Trying to finishing_scan with unknown scan: {scan_id}.') + log.warning(f"Trying to finishing_scan with unknown scan: {scan_id}.") return group([]) # No further actions, so not setting "finishing scan" as a state, but set it to "scan finished" directly. @@ -501,7 +514,7 @@ def monitor_timeout(scan_id: int): scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: - log.warning(f'Trying to monitor_timeout with unknown scan: {scan_id}.') + log.warning(f"Trying to monitor_timeout with unknown scan: {scan_id}.") return group([]) # Warning: timeouts are only useful when crashes happened, otherwise its just a capacity issue which timeouts @@ -510,33 +523,33 @@ def monitor_timeout(scan_id: int): # todo: recover from websecmap errors, by trying to recover there and writing the status to the dashboard. recovering_strategies = { "discovering endpoints": { - "timeout in minutes": constance_cached_value('SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS'), # 6 * 60, - "state after timeout": "requested" + "timeout in minutes": constance_cached_value("SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS"), # 6 * 60, + "state after timeout": "requested", }, "retrieving scannable urls": { - "timeout in minutes": constance_cached_value('SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS'), # 6 - "state after timeout": "discovered endpoints" + "timeout in minutes": constance_cached_value("SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS"), # 6 + "state after timeout": "discovered endpoints", }, "registering scan at internet.nl": { - "timeout in minutes": constance_cached_value('SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL'), # 6 - "state after timeout": "retrieved scannable urls" + "timeout in minutes": constance_cached_value("SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL"), # 6 + "state after timeout": "retrieved scannable urls", }, "importing scan results": { - "timeout in minutes": constance_cached_value('SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS'), # 24 - "state after timeout": "scan results stored" + "timeout in minutes": constance_cached_value("SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS"), # 24 + "state after timeout": "scan results stored", }, "creating report": { - "timeout in minutes": constance_cached_value('SCAN_TIMEOUT_MINUTES_CREATING_REPORT'), # 24 - "state after timeout": "imported scan results" + "timeout in minutes": constance_cached_value("SCAN_TIMEOUT_MINUTES_CREATING_REPORT"), # 24 + "state after timeout": "imported scan results", }, "sending mail": { - "timeout in minutes": constance_cached_value('SCAN_TIMEOUT_MINUTES_SENDING_MAIL'), # 6 - "state after timeout": "created report" + "timeout in minutes": constance_cached_value("SCAN_TIMEOUT_MINUTES_SENDING_MAIL"), # 6 + "state after timeout": "created report", }, # It's unclear where in the process we are... Just try again. "server_error": { - "timeout in minutes": constance_cached_value('SCAN_TIMEOUT_MINUTES_SERVER_ERROR'), # 1 - "state after timeout": "requested" + "timeout in minutes": constance_cached_value("SCAN_TIMEOUT_MINUTES_SERVER_ERROR"), # 1 + "state after timeout": "requested", }, } @@ -547,17 +560,19 @@ def monitor_timeout(scan_id: int): # determine if there is an actual timeout. if scan.state_changed_on: - scan_will_timeout_on = scan.state_changed_on + timedelta(minutes=strategy['timeout in minutes']) + scan_will_timeout_on = scan.state_changed_on + timedelta(minutes=strategy["timeout in minutes"]) if datetime.now(timezone.utc) > scan_will_timeout_on: - update_state(f"timeout reached for: '{scan.state}', " - f"performing recovery to '{strategy['state after timeout']}'", scan.id) - update_state(strategy['state after timeout'], scan.id) + update_state( + f"timeout reached for: '{scan.state}', " f"performing recovery to '{strategy['state after timeout']}'", + scan.id, + ) + update_state(strategy["state after timeout"], scan.id) # No further work to do... return group([]) -@app.task(queue='storage') +@app.task(queue="storage") def connect_urllistreport_to_accountinternetnlscan(urllistreport_id: int, scan_id: int) -> int: scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: @@ -581,7 +596,7 @@ def connect_urllistreport_to_accountinternetnlscan(urllistreport_id: int, scan_i return int(urllistreport.id) -@app.task(queue='storage') +@app.task(queue="storage") def upgrade_report_with_statistics(urllistreport_id: int) -> int: urllistreport = UrlListReport.objects.all().filter(id=urllistreport_id).first() if not urllistreport: @@ -594,7 +609,7 @@ def upgrade_report_with_statistics(urllistreport_id: int) -> int: return int(urllistreport.pk) -@app.task(queue='storage', ignore_result=True) +@app.task(queue="storage", ignore_result=True) def upgrade_report_with_unscannable_urls(urllistreport_id: int, scan_id: int): """ Urls that cannot be scanned using the internet.nl website are not allowed to be scanned. This is where endpoint @@ -637,7 +652,7 @@ def upgrade_report_with_unscannable_urls(urllistreport_id: int, scan_id: int): # the list of domains to scan. calculation = retrieve_report(urllistreport_id, "UrlListReport") - urls_in_report: List[str] = [url['url'] for url in calculation['urls']] + urls_in_report: List[str] = [url["url"] for url in calculation["urls"]] urls_in_list: List[Url] = list(scan.urllist.urls.all()) urls_not_in_report = [url.url for url in urls_in_list if url.url not in urls_in_report] @@ -671,19 +686,19 @@ def upgrade_report_with_unscannable_urls(urllistreport_id: int, scan_id: int): # Copy the template, otherwise all instances will point to the same text (the last domain in the list of # missing domains). tmp_empty_url_template = copy(empty_url_template) - tmp_empty_url_template['url'] = url_not_in_report - calculation['urls'].append(tmp_empty_url_template) + tmp_empty_url_template["url"] = url_not_in_report + calculation["urls"].append(tmp_empty_url_template) # also update the total urls, as that can be influenced: - calculation['total_urls'] = len(calculation['urls']) - urllistreport.total_urls = len(calculation['urls']) + calculation["total_urls"] = len(calculation["urls"]) + urllistreport.total_urls = len(calculation["urls"]) urllistreport.save() store_report(urllistreport_id, "UrlListReport", calculation) return -@app.task(queue='storage') +@app.task(queue="storage") def send_after_scan_mail(scan_id: int) -> str: scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: @@ -700,30 +715,31 @@ def send_after_scan_mail(scan_id: int) -> str: return "sent mail" -@app.task(queue='storage') +@app.task(queue="storage") def check_retrieved_scannable_urls(urls: List[int]): - """ Influences the process, see if we can continue. """ + """Influences the process, see if we can continue.""" if not urls: - return "error retrieving scannable urls: " \ - "no urls to scan found. Will not continue as the report will be empty." + return ( + "error retrieving scannable urls: " "no urls to scan found. Will not continue as the report will be empty." + ) return "retrieved scannable urls" -@app.task(queue='storage', ignore_result=True) +@app.task(queue="storage", ignore_result=True) def update_state(state: str, scan_id: int) -> None: """Update the current scan state. Also write it to the scan log. From this log we should also be able to see retries... when celery retries on exceptions etc...""" - scan = AccountInternetNLScan.objects.all().filter(id=scan_id).only('id', 'state').first() + scan = AccountInternetNLScan.objects.all().filter(id=scan_id).only("id", "state").first() if not scan: return # if the state is still the same, just update the last_check, don't append the log. # Don't get it from the scan object, that info might be obsolete. - last_state_for_scan = AccountInternetNLScanLog.objects.all().filter( - scan=scan - ).order_by("-at_when").only('state').first() + last_state_for_scan = ( + AccountInternetNLScanLog.objects.all().filter(scan=scan).order_by("-at_when").only("state").first() + ) if last_state_for_scan: # see: test_update_state @@ -748,12 +764,17 @@ def update_state(state: str, scan_id: int) -> None: scanlog.save() -@app.task(queue='storage') +@app.task(queue="storage") def get_relevant_urls(urllist_id: int, protocol: str) -> List[int]: urllist = UrlList.objects.all().filter(id=urllist_id).first() if not urllist: return [] - urls = Url.objects.all().filter(urls_in_dashboard_list_2=urllist, is_dead=False, not_resolvable=False, - endpoint__protocol__in=[protocol]).values_list('id', flat=True) + urls = ( + Url.objects.all() + .filter( + urls_in_dashboard_list_2=urllist, is_dead=False, not_resolvable=False, endpoint__protocol__in=[protocol] + ) + .values_list("id", flat=True) + ) return list(set(urls)) diff --git a/dashboard/internet_nl_dashboard/scanners/subdomains.py b/dashboard/internet_nl_dashboard/scanners/subdomains.py index 7c470b43..145d9c31 100644 --- a/dashboard/internet_nl_dashboard/scanners/subdomains.py +++ b/dashboard/internet_nl_dashboard/scanners/subdomains.py @@ -28,7 +28,7 @@ def request_scan(account: Account, urllist_id: int): update_state(scan.id, "requested") return scan_status(account, urllist_id) - if last_scan.state in ['requested', 'scanning']: + if last_scan.state in ["requested", "scanning"]: return scan_status(account, urllist_id) scan = SubdomainDiscoveryScan() @@ -48,9 +48,14 @@ def scan_status(account: Account, urllist_id: int): if not scan: return operation_response(error=True, message="not_scanned_at_all") - return {'success': True, 'error': False, 'state': scan.state, 'state_message': scan.state_message, - 'state_changed_on': scan.state_changed_on, - 'domains_discovered': json.loads(scan.domains_discovered) if scan.domains_discovered else {}} + return { + "success": True, + "error": False, + "state": scan.state, + "state_message": scan.state_message, + "state_changed_on": scan.state_changed_on, + "domains_discovered": json.loads(scan.domains_discovered) if scan.domains_discovered else {}, + } # Process diff --git a/dashboard/internet_nl_dashboard/signals.py b/dashboard/internet_nl_dashboard/signals.py index 583b8a11..5734000b 100644 --- a/dashboard/internet_nl_dashboard/signals.py +++ b/dashboard/internet_nl_dashboard/signals.py @@ -7,12 +7,12 @@ @receiver(user_logged_in) def stream_login(**kwargs): # sender = user - action.send(kwargs['user'], verb='logged in', public=False) + action.send(kwargs["user"], verb="logged in", public=False) @receiver(user_logged_out) def stream_logout(**kwargs): # sender = user # logging out via json requests went wrong somehow. - if kwargs.get('user', None): - action.send(kwargs['user'], verb='logged out', public=False) + if kwargs.get("user", None): + action.send(kwargs["user"], verb="logged out", public=False) diff --git a/dashboard/internet_nl_dashboard/tasks.py b/dashboard/internet_nl_dashboard/tasks.py index 85fa9294..01013764 100644 --- a/dashboard/internet_nl_dashboard/tasks.py +++ b/dashboard/internet_nl_dashboard/tasks.py @@ -19,7 +19,7 @@ # from websecmap.celery, not from dashboard.celery. Which still doesn't solve it after changing. -@app.task(queue='storage') +@app.task(queue="storage") def start_scans_for_lists_who_are_up_for_scanning() -> Task: """ This can be run every minute, only the ones that are up for scanning will be scanned. It will update all @@ -51,20 +51,19 @@ def start_scans_for_lists_who_are_up_for_scanning() -> Task: __all__: List[Module] = [scan_internet_nl_per_account, subdomains] # type: ignore -@app.task(queue='storage', ignore_result=True) +@app.task(queue="storage", ignore_result=True) def autoshare_report_to_front_page(): ids = config.DASHBOARD_FRONT_PAGE_URL_LISTS if not ids: return - ids = ids.split(',') + ids = ids.split(",") ints = [int(id_) for id_ in ids] # Do not publish historic reports, things can actually stay offline if there was an error with a report UrlListReport.objects.filter( - urllist__id__in=ints, - at_when__gte=datetime.now(timezone.utc) - timedelta(hours=24) + urllist__id__in=ints, at_when__gte=datetime.now(timezone.utc) - timedelta(hours=24) ).update( is_publicly_shared=True, is_shared_on_homepage=True, diff --git a/dashboard/internet_nl_dashboard/tests/__init__.py b/dashboard/internet_nl_dashboard/tests/__init__.py index 2f456782..d33a6661 100644 --- a/dashboard/internet_nl_dashboard/tests/__init__.py +++ b/dashboard/internet_nl_dashboard/tests/__init__.py @@ -15,14 +15,21 @@ def make_url_with_endpoint_and_scan(): account, _ = Account.objects.all().get_or_create(name="test") - url, _ = Url.objects.all().get_or_create(url='test.nl', created_on=day_0, not_resolvable=False) + url, _ = Url.objects.all().get_or_create(url="test.nl", created_on=day_0, not_resolvable=False) endpoint, _ = Endpoint.objects.all().get_or_create( - url=url, protocol='https', port='443', ip_version=4, discovered_on=day_1, is_dead=False) + url=url, protocol="https", port="443", ip_version=4, discovered_on=day_1, is_dead=False + ) scan, _ = EndpointGenericScan.objects.all().get_or_create( - endpoint=endpoint, type='tls_qualys_encryption_quality', rating='A+', rating_determined_on=day_1, - last_scan_moment=day_1, comply_or_explain_is_explained=False, is_the_latest_scan=True) + endpoint=endpoint, + type="tls_qualys_encryption_quality", + rating="A+", + rating_determined_on=day_1, + last_scan_moment=day_1, + comply_or_explain_is_explained=False, + is_the_latest_scan=True, + ) return account, url, endpoint, scan @@ -31,8 +38,7 @@ def create_scan_report(account: Account, urllist: UrlList): rate_urllists_now([urllist]) # create a scan for this list, - scan, _ = InternetNLV2Scan.objects.all().get_or_create( - pk=1, type='web', state="finished") + scan, _ = InternetNLV2Scan.objects.all().get_or_create(pk=1, type="web", state="finished") ainls = AccountInternetNLScan() ainls.account = account diff --git a/dashboard/internet_nl_dashboard/tests/test_account.py b/dashboard/internet_nl_dashboard/tests/test_account.py index fae8b901..13251976 100644 --- a/dashboard/internet_nl_dashboard/tests/test_account.py +++ b/dashboard/internet_nl_dashboard/tests/test_account.py @@ -6,14 +6,15 @@ def test_urllists(db) -> None: account, created = Account.objects.all().get_or_create(name="test") - settings = {"filters": { - "web": {"visible": True}, - "web_legacy": {"visible": True}, - "mail": {"visible": True}, - "mail_legacy": {"visible": True}, - } + settings = { + "filters": { + "web": {"visible": True}, + "web_legacy": {"visible": True}, + "mail": {"visible": True}, + "mail_legacy": {"visible": True}, + } } save_report_settings(account, settings) retrieved_settings = get_report_settings(account) - assert retrieved_settings['data'] == settings['filters'] + assert retrieved_settings["data"] == settings["filters"] diff --git a/dashboard/internet_nl_dashboard/tests/test_clean_urls.py b/dashboard/internet_nl_dashboard/tests/test_clean_urls.py index 1c693b80..4ff3a30a 100644 --- a/dashboard/internet_nl_dashboard/tests/test_clean_urls.py +++ b/dashboard/internet_nl_dashboard/tests/test_clean_urls.py @@ -10,30 +10,30 @@ def test_clean_urls() -> None: # domains can't contain spaces... - result = clean_urls(['kalsndlkas.asdakj .com']) - assert len(result['incorrect']) == 1 and len(result['correct']) == 0 + result = clean_urls(["kalsndlkas.asdakj .com"]) + assert len(result["incorrect"]) == 1 and len(result["correct"]) == 0 - result = clean_urls(['apple.com']) - assert len(result['incorrect']) == 0 and len(result['correct']) == 1 + result = clean_urls(["apple.com"]) + assert len(result["incorrect"]) == 0 and len(result["correct"]) == 1 - result = clean_urls(['vintage.museum']) - assert len(result['incorrect']) == 0 and len(result['correct']) == 1 + result = clean_urls(["vintage.museum"]) + assert len(result["incorrect"]) == 0 and len(result["correct"]) == 1 - result = clean_urls(['subsub.sub.sub.com']) - assert len(result['incorrect']) == 0 and len(result['correct']) == 1 + result = clean_urls(["subsub.sub.sub.com"]) + assert len(result["incorrect"]) == 0 and len(result["correct"]) == 1 # warning: i don't know what the below words mean or what connotations they have. # probably rent.org in russian / .xn--c1avg # both urlparse and validators say 'no' to this. # should we convert to punycode? # todo: it's not really clear in python how to convert unicode to punycode. Searching for it does not yield results. - result = clean_urls(['аренда.орг']) - assert len(result['incorrect']) == 0 and len(result['correct']) == 1 + result = clean_urls(["аренда.орг"]) + assert len(result["incorrect"]) == 0 and len(result["correct"]) == 1 # = xn--o1b5efa8g5c.xn--i1b6b1a6a2e - result = clean_urls(['मुम्बई.संगठन']) - assert len(result['incorrect']) == 0 and len(result['correct']) == 1 + result = clean_urls(["मुम्बई.संगठन"]) + assert len(result["incorrect"]) == 0 and len(result["correct"]) == 1 - result = clean_urls([' ', ' ⠀ ', 'eskillsplatform.nl', 'stichtingmediawijzer.nl', '\u200b ']) - assert len(result['incorrect']) == 3 and len(result['correct']) == 2 - assert result['correct'] == ['eskillsplatform.nl', 'stichtingmediawijzer.nl'] + result = clean_urls([" ", " ⠀ ", "eskillsplatform.nl", "stichtingmediawijzer.nl", "\u200b "]) + assert len(result["incorrect"]) == 3 and len(result["correct"]) == 2 + assert result["correct"] == ["eskillsplatform.nl", "stichtingmediawijzer.nl"] diff --git a/dashboard/internet_nl_dashboard/tests/test_config.py b/dashboard/internet_nl_dashboard/tests/test_config.py index efb504ab..34a30513 100644 --- a/dashboard/internet_nl_dashboard/tests/test_config.py +++ b/dashboard/internet_nl_dashboard/tests/test_config.py @@ -4,4 +4,4 @@ def test_config_content(db): data = config_content() - assert data['show']['signup_form'] is False + assert data["show"]["signup_form"] is False diff --git a/dashboard/internet_nl_dashboard/tests/test_dashboard_report.py b/dashboard/internet_nl_dashboard/tests/test_dashboard_report.py index 666b76b1..bf53e20f 100644 --- a/dashboard/internet_nl_dashboard/tests/test_dashboard_report.py +++ b/dashboard/internet_nl_dashboard/tests/test_dashboard_report.py @@ -6,22 +6,36 @@ def test_sum_internet_nl_scores_over_rating(): # a normal calculation: calculation = { "urls": [ - {"url": "acc.dashboard.internet.nl", "endpoints": [ - {"id": 4959, - "ratings": [ - {"type": "internet_nl_web_overall_score", - "explanation": "80 https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/"}] - } - ] + { + "url": "acc.dashboard.internet.nl", + "endpoints": [ + { + "id": 4959, + "ratings": [ + { + "type": "internet_nl_web_overall_score", + "explanation": "80 https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/", + } + ], + } + ], }, - {"url": "acc.dashboard.internet.nl", "endpoints": [ - {"id": 4959, - "ratings": [ - {"type": "internet_nl_web_overall_score", - "explanation": "20 https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/"}] - } - ] - }]} + { + "url": "acc.dashboard.internet.nl", + "endpoints": [ + { + "id": 4959, + "ratings": [ + { + "type": "internet_nl_web_overall_score", + "explanation": "20 https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/", + } + ], + } + ], + }, + ] + } assert sum_internet_nl_scores_over_rating(calculation) == 50 @@ -29,25 +43,39 @@ def test_sum_internet_nl_scores_over_rating(): # a normal calculation: calculation = { "urls": [ - {"url": "acc.dashboard.internet.nl", "endpoints": [ - {"id": 4959, - "ratings": [ - {"type": "internet_nl_web_overall_score", - "explanation": "error https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/"}] - } - ] + { + "url": "acc.dashboard.internet.nl", + "endpoints": [ + { + "id": 4959, + "ratings": [ + { + "type": "internet_nl_web_overall_score", + "explanation": "error https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/", + } + ], + } + ], + }, + { + "url": "acc.dashboard.internet.nl", + "endpoints": [ + { + "id": 4959, + "ratings": [ + { + "type": "internet_nl_web_overall_score", + "explanation": "20 https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/", + } + ], + } + ], }, - {"url": "acc.dashboard.internet.nl", "endpoints": [ - {"id": 4959, - "ratings": [ - {"type": "internet_nl_web_overall_score", - "explanation": "20 https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/"}] - } - ] - }]} + ] + } assert sum_internet_nl_scores_over_rating(calculation) == 20 # and no value at all: - assert sum_internet_nl_scores_over_rating({'urls': []}) == 0 + assert sum_internet_nl_scores_over_rating({"urls": []}) == 0 assert sum_internet_nl_scores_over_rating({}) == 0 diff --git a/dashboard/internet_nl_dashboard/tests/test_deduplication.py b/dashboard/internet_nl_dashboard/tests/test_deduplication.py index 5945fcb7..4e886983 100644 --- a/dashboard/internet_nl_dashboard/tests/test_deduplication.py +++ b/dashboard/internet_nl_dashboard/tests/test_deduplication.py @@ -13,64 +13,83 @@ from dashboard.internet_nl_dashboard.logic.deduplication import dedupe_urls from dashboard.internet_nl_dashboard.models import Account, UrlList -log = logging.getLogger('test') +log = logging.getLogger("test") def test_per_acount_scanner(db) -> None: - account, _ = Account.objects.all().get_or_create(name="test", - internet_nl_api_username='test', - internet_nl_api_password=Account.encrypt_password('test'),) + account, _ = Account.objects.all().get_or_create( + name="test", + internet_nl_api_username="test", + internet_nl_api_password=Account.encrypt_password("test"), + ) # add duplicate (named) urls, the name is the same, the id is different. - url1, _ = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False, - internal_notes='different data, is different id 1') - url2, _ = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False, - internal_notes='different data, is different id 2') - url3, _ = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False, - internal_notes='different data, is different id 3') - url4, _ = Url.objects.get_or_create(url='www.unaffected_url.nl', is_dead=False, not_resolvable=False, - internal_notes='different data, is different id 3') + url1, _ = Url.objects.get_or_create( + url="www.internet.nl", is_dead=False, not_resolvable=False, internal_notes="different data, is different id 1" + ) + url2, _ = Url.objects.get_or_create( + url="www.internet.nl", is_dead=False, not_resolvable=False, internal_notes="different data, is different id 2" + ) + url3, _ = Url.objects.get_or_create( + url="www.internet.nl", is_dead=False, not_resolvable=False, internal_notes="different data, is different id 3" + ) + url4, _ = Url.objects.get_or_create( + url="www.unaffected_url.nl", + is_dead=False, + not_resolvable=False, + internal_notes="different data, is different id 3", + ) # make sure that these three urls actually exist assert Url.objects.all().count() == 4 # with duplicate endpoints, where the endpoints themselves are unique for the url. - ep1, _ = Endpoint.objects.get_or_create(url=url1, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False) - ep2, _ = Endpoint.objects.get_or_create(url=url2, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False) - ep3, _ = Endpoint.objects.get_or_create(url=url3, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False) - ep4, _ = Endpoint.objects.get_or_create(url=url1, protocol='dns_soa', port=0, ip_version=0, is_dead=False) - ep5, _ = Endpoint.objects.get_or_create(url=url2, protocol='dns_soa', port=0, ip_version=0, is_dead=False) - ep6, _ = Endpoint.objects.get_or_create(url=url4, protocol='unaffected', port=0, ip_version=0, is_dead=False) + ep1, _ = Endpoint.objects.get_or_create(url=url1, protocol="dns_a_aaaa", port=0, ip_version=0, is_dead=False) + ep2, _ = Endpoint.objects.get_or_create(url=url2, protocol="dns_a_aaaa", port=0, ip_version=0, is_dead=False) + ep3, _ = Endpoint.objects.get_or_create(url=url3, protocol="dns_a_aaaa", port=0, ip_version=0, is_dead=False) + ep4, _ = Endpoint.objects.get_or_create(url=url1, protocol="dns_soa", port=0, ip_version=0, is_dead=False) + ep5, _ = Endpoint.objects.get_or_create(url=url2, protocol="dns_soa", port=0, ip_version=0, is_dead=False) + ep6, _ = Endpoint.objects.get_or_create(url=url4, protocol="unaffected", port=0, ip_version=0, is_dead=False) # make sure there are five endpoints: assert Endpoint.objects.all().count() == 6 # and duplicate scan results - egs1, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 1', rating='1', - rating_determined_on=datetime.now(timezone.utc)) - egs2, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 2', rating='2', - rating_determined_on=datetime.now(timezone.utc)) - egs3, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 3', rating='3', - rating_determined_on=datetime.now(timezone.utc)) - - egs4, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep2, type='testscan 4', rating='4', - rating_determined_on=datetime.now(timezone.utc)) - egs5, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep2, type='testscan 5', rating='5', - rating_determined_on=datetime.now(timezone.utc)) - - egs6, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep3, type='testscan 6', rating='6', - rating_determined_on=datetime.now(timezone.utc)) + egs1, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep1, type="testscan 1", rating="1", rating_determined_on=datetime.now(timezone.utc) + ) + egs2, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep1, type="testscan 2", rating="2", rating_determined_on=datetime.now(timezone.utc) + ) + egs3, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep1, type="testscan 3", rating="3", rating_determined_on=datetime.now(timezone.utc) + ) + + egs4, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep2, type="testscan 4", rating="4", rating_determined_on=datetime.now(timezone.utc) + ) + egs5, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep2, type="testscan 5", rating="5", rating_determined_on=datetime.now(timezone.utc) + ) + + egs6, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep3, type="testscan 6", rating="6", rating_determined_on=datetime.now(timezone.utc) + ) # some with the same data: - egs7, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 1', rating='1', - rating_determined_on=datetime.now(timezone.utc)) - egs8, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 2', rating='2', - rating_determined_on=datetime.now(timezone.utc)) - egs9, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 3', rating='3', - rating_determined_on=datetime.now(timezone.utc)) - - egs10, _ = EndpointGenericScan.objects.get_or_create(endpoint=ep6, type='unaffected', rating='3', - rating_determined_on=datetime.now(timezone.utc)) + egs7, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep4, type="testscan 1", rating="1", rating_determined_on=datetime.now(timezone.utc) + ) + egs8, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep4, type="testscan 2", rating="2", rating_determined_on=datetime.now(timezone.utc) + ) + egs9, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep4, type="testscan 3", rating="3", rating_determined_on=datetime.now(timezone.utc) + ) + + egs10, _ = EndpointGenericScan.objects.get_or_create( + endpoint=ep6, type="unaffected", rating="3", rating_determined_on=datetime.now(timezone.utc) + ) assert EndpointGenericScan.objects.all().count() == 10 @@ -81,19 +100,19 @@ def test_per_acount_scanner(db) -> None: # the endpoints are merged into one set of unique endpoints. The scans are just attached to their respective # unique endpoint. - urllist, _ = UrlList.objects.all().get_or_create(name='test 1', account=account, scan_type='web') + urllist, _ = UrlList.objects.all().get_or_create(name="test 1", account=account, scan_type="web") urllist.urls.add(url1) urllist.save() - urllist, _ = UrlList.objects.all().get_or_create(name='test 2', account=account, scan_type='web') + urllist, _ = UrlList.objects.all().get_or_create(name="test 2", account=account, scan_type="web") urllist.urls.add(url2) urllist.save() - urllist, _ = UrlList.objects.all().get_or_create(name='test 3', account=account, scan_type='web') + urllist, _ = UrlList.objects.all().get_or_create(name="test 3", account=account, scan_type="web") urllist.urls.add(url3) urllist.save() - urllist, _ = UrlList.objects.all().get_or_create(name='unaffected', account=account, scan_type='web') + urllist, _ = UrlList.objects.all().get_or_create(name="unaffected", account=account, scan_type="web") urllist.urls.add(url4) urllist.save() @@ -101,14 +120,14 @@ def test_per_acount_scanner(db) -> None: dedupe_urls() # assert all urls in the urllists are the same. - list_1 = UrlList.objects.all().filter(name='test 1').first() - list_2 = UrlList.objects.all().filter(name='test 2').first() - list_3 = UrlList.objects.all().filter(name='test 3').first() + list_1 = UrlList.objects.all().filter(name="test 1").first() + list_2 = UrlList.objects.all().filter(name="test 2").first() + list_3 = UrlList.objects.all().filter(name="test 3").first() assert list_1.urls.count() == list_2.urls.count() == list_3.urls.count() assert list_1.urls.first().id == list_2.urls.last().id == list_3.urls.first().id # verify that the unaffected list has a different url than the affected list - list_4 = UrlList.objects.all().filter(name='unaffected').first() + list_4 = UrlList.objects.all().filter(name="unaffected").first() assert list_1.urls.first().id != list_4.urls.last().id # assert that there is only one url in each of these lists @@ -118,7 +137,7 @@ def test_per_acount_scanner(db) -> None: assert list_4.urls.count() == 1 # assert there should only be one url left + the unaffected url, as the rest where identical - assert Url.objects.all().filter(url='www.internet.nl').count() == 1 + assert Url.objects.all().filter(url="www.internet.nl").count() == 1 # the total number of urls should be two now, including one unaffected: assert Url.objects.all().count() == 1 + 1 diff --git a/dashboard/internet_nl_dashboard/tests/test_domain_management.py b/dashboard/internet_nl_dashboard/tests/test_domain_management.py index 9ef8085d..b3e37e62 100644 --- a/dashboard/internet_nl_dashboard/tests/test_domain_management.py +++ b/dashboard/internet_nl_dashboard/tests/test_domain_management.py @@ -65,8 +65,8 @@ def test_add_domains_from_raw_user_data(db, current_path, redis_server): Should give a warning after the limit of N domains is crossed, should add to this limit. """ - file = f'{current_path}/test_domain_management/top_10000_nl_domains.txt' - with open(file, 'rt') as f: + file = f"{current_path}/test_domain_management/top_10000_nl_domains.txt" + with open(file, "rt") as f: domains = f.read() domains = domains.split("\n") @@ -76,48 +76,44 @@ def test_add_domains_from_raw_user_data(db, current_path, redis_server): list_1 = get_or_create_list_by_name(account, "test list 1") # you can't go past the DASHBOARD_MAXIMUM_DOMAINS_PER_LIST - response = add_domains_from_raw_user_data(account, { - 'list_id': list_1.id, - 'urls': ", ".join(domains[:6000]) - }) + response = add_domains_from_raw_user_data(account, {"list_id": list_1.id, "urls": ", ".join(domains[:6000])}) - assert response['error'] is True - assert response['message'] == "too_many_domains" + assert response["error"] is True + assert response["message"] == "too_many_domains" # add an existing url, this should be one query check: - new_url = Url.objects.all().create(url='nu.nl') + new_url = Url.objects.all().create(url="nu.nl") list_1.urls.add(new_url) - response = add_domains_from_raw_user_data(account, { - 'list_id': list_1.id, - 'urls': ", ".join(domains[:100]) - }) - - assert response['success'] is True - assert response['data'] == { - 'incorrect_urls': [], - 'added_to_list': 99, - 'already_in_list': 1, - 'duplicates_removed': 0 + response = add_domains_from_raw_user_data(account, {"list_id": list_1.id, "urls": ", ".join(domains[:100])}) + + assert response["success"] is True + assert response["data"] == { + "incorrect_urls": [], + "added_to_list": 99, + "already_in_list": 1, + "duplicates_removed": 0, } def test_keys_match(): # a, b, c in any object - assert keys_are_present_in_object(expected_keys=['a', 'b', 'c'], any_object={ - 'a': 1, 'b': 2, 'c': 3, 'd': 4}) is True + assert ( + keys_are_present_in_object(expected_keys=["a", "b", "c"], any_object={"a": 1, "b": 2, "c": 3, "d": 4}) is True + ) # Missing b - assert keys_are_present_in_object(expected_keys=['a', 'b', 'c'], any_object={'a': 1, 'c': 3, 'd': 4}) is False + assert keys_are_present_in_object(expected_keys=["a", "b", "c"], any_object={"a": 1, "c": 3, "d": 4}) is False # No expectation should always match - assert keys_are_present_in_object(expected_keys=[], any_object={'a': 1, 'b': 2, 'c': 3, 'd': 4}) is True + assert keys_are_present_in_object(expected_keys=[], any_object={"a": 1, "b": 2, "c": 3, "d": 4}) is True def test_retrieve_urls_from_unfiltered_input() -> None: output, duplicates_removed = retrieve_possible_urls_from_unfiltered_input( - "https://www.apple.com:443/nl/iphone-11/, bing.com, http://nu.nl, nu.nl") - assert output == ['bing.com', 'nu.nl', 'www.apple.com'] + "https://www.apple.com:443/nl/iphone-11/, bing.com, http://nu.nl, nu.nl" + ) + assert output == ["bing.com", "nu.nl", "www.apple.com"] # one nu.nl removed assert duplicates_removed == 1 @@ -136,21 +132,22 @@ def test_retrieve_urls_from_unfiltered_input() -> None: output, duplicates_removed = retrieve_possible_urls_from_unfiltered_input(unsanitized_input) # Zero width space is also seen as a string, and filtered out as a possible domain. See test_clean_urls. # ' ', ' ⠀ ', '\u200b ' - assert output == ['eskillsplatform.nl', 'stichtingmediawijzer.nl'] + assert output == ["eskillsplatform.nl", "stichtingmediawijzer.nl"] # test #410 unsanitized_input = "home­lan­der.nl" output, duplicates_removed = retrieve_possible_urls_from_unfiltered_input(unsanitized_input) - assert output == ['homelander.nl'] + assert output == ["homelander.nl"] def test_retrieve_urls_from_unfiltered_input_email() -> None: # fix 246 and 316 # note that 'info' is seen as a possible url. That will be removed when cleaning the urls. output, duplicates_removed = retrieve_possible_urls_from_unfiltered_input( - "example.com/something, example2.com/something., info@example3.com, info@example4.com") + "example.com/something, example2.com/something., info@example3.com, info@example4.com" + ) # 'info' - assert output == ['example.com', 'example2.com', 'example3.com', 'example4.com'] + assert output == ["example.com", "example2.com", "example3.com", "example4.com"] def test_retrieve_possible_urls_from_unfiltered_input_speed() -> None: @@ -284,19 +281,13 @@ def test_retrieve_possible_urls_from_unfiltered_input_speed() -> None: chess.com, quickconnect.to, split.io, constantcontact.com, msecnd.net, nic.do, creditkarma.com, sportskeeda.com, www.gov.br, spankbang.com, withgoogle.com """ - time = timeit.timeit( - lambda: retrieve_possible_urls_from_unfiltered_input(data), - number=25 - ) + time = timeit.timeit(lambda: retrieve_possible_urls_from_unfiltered_input(data), number=25) # some slower machines... It's < 0.1 on a dev machine assert time < 0.5 domains, _ = retrieve_possible_urls_from_unfiltered_input(data) assert len(domains) == 1000 - time = timeit.timeit( - lambda: clean_urls(domains), - number=25 - ) + time = timeit.timeit(lambda: clean_urls(domains), number=25) assert time < 1.5 # it's 0.102 on a dev machine for 25k @@ -311,7 +302,7 @@ def test_urllists(db, redis_server) -> None: assert list_1 != list_2 list_content = get_urllist_content(account=account, urllist_id=list_1.pk) - assert len(list_content['urls']) == 0 + assert len(list_content["urls"]) == 0 """ We made two lists, so we expect to see two lists returned """ lists = get_urllists_from_account(account=account) @@ -319,67 +310,73 @@ def test_urllists(db, redis_server) -> None: """ Should be no problem to add the same urls, it just has not so much effect. """ added = save_urllist_content_by_name( - account, "test list 1", { - 'test.nl': {'tags': []}, 'internet.nl': {'tags': []}, 'internetcleanup.foundation': {'tags': []}}) - assert added['added_to_list'] == 3 and added['already_in_list'] == 0 and len(added['incorrect_urls']) == 0 + account, + "test list 1", + {"test.nl": {"tags": []}, "internet.nl": {"tags": []}, "internetcleanup.foundation": {"tags": []}}, + ) + assert added["added_to_list"] == 3 and added["already_in_list"] == 0 and len(added["incorrect_urls"]) == 0 already = save_urllist_content_by_name( - account, "test list 1", { - 'test.nl': {'tags': []}, 'internet.nl': {'tags': []}, 'internetcleanup.foundation': {'tags': []}}) - assert already['added_to_list'] == 0 and already['already_in_list'] == 3 and len(already['incorrect_urls']) == 0 + account, + "test list 1", + {"test.nl": {"tags": []}, "internet.nl": {"tags": []}, "internetcleanup.foundation": {"tags": []}}, + ) + assert already["added_to_list"] == 0 and already["already_in_list"] == 3 and len(already["incorrect_urls"]) == 0 list_content = get_urllist_content(account=account, urllist_id=list_1.pk) - assert len(list_content['urls']) == 3 + assert len(list_content["urls"]) == 3 """ Garbage urls should be filtered out and can be displayed as erroneous """ # Impossible to filter out garbage domains, as the tld and domain is checked along the way... and some parts # of the domain like 'info' might be seen as a domain while it isn't - already = save_urllist_content_by_name(account, "test list 1", { - 'test.nonse^': {'tags': []}, 'NONSENSE': {'tags': []}, '127.0.0.1': {'tags': []}}) - assert already['added_to_list'] == 0 and already['already_in_list'] == 0 and len(already['incorrect_urls']) == 0 + already = save_urllist_content_by_name( + account, "test list 1", {"test.nonse^": {"tags": []}, "NONSENSE": {"tags": []}, "127.0.0.1": {"tags": []}} + ) + assert already["added_to_list"] == 0 and already["already_in_list"] == 0 and len(already["incorrect_urls"]) == 0 """ Check if really nothing was added """ list_content = get_urllist_content(account=account, urllist_id=list_1.pk) - assert len(list_content['urls']) == 3 + assert len(list_content["urls"]) == 3 # make sure the url gets deleted from the urllist and not from the database urls_in_database = Url.objects.all().count() assert urls_in_database == 3 """ Delete a a urls from the list: """ - url_got_removed_from_list = delete_url_from_urllist(account, list_1.id, - Url.objects.all().filter(url='test.nl').first().id) + url_got_removed_from_list = delete_url_from_urllist( + account, list_1.id, Url.objects.all().filter(url="test.nl").first().id + ) assert urls_in_database == Url.objects.all().count() assert url_got_removed_from_list is True list_content = get_urllist_content(account=account, urllist_id=list_1.pk) - assert len(list_content['urls']) == 2 + assert len(list_content["urls"]) == 2 """ Delete the entire list, we'll get nothing back, only an empty response. """ - operation_response = delete_list(account=account, user_input={'id': list_1.id}) + operation_response = delete_list(account=account, user_input={"id": list_1.id}) # it deletes two urls and the list itself, makes 3 - assert operation_response['success'] is True + assert operation_response["success"] is True list_content = get_urllist_content(account=account, urllist_id=list_1.pk) # deletion is administrative, so the urls are still connected. - assert len(list_content['urls']) == 2 + assert len(list_content["urls"]) == 2 account2, created = Account.objects.all().get_or_create(name="test 2") """ You cannot delete things from another account """ - operation_response = delete_list(account=account2, user_input={'id': list_1.id}) - assert operation_response['success'] is False + operation_response = delete_list(account=account2, user_input={"id": list_1.id}) + assert operation_response["success"] is False """ A new list will not be created if there are no urls for it...""" added = save_urllist_content_by_name(account, "should be empty", {}) - assert added['added_to_list'] == 0 and added['already_in_list'] == 0 and len(added['incorrect_urls']) == 0 + assert added["added_to_list"] == 0 and added["already_in_list"] == 0 and len(added["incorrect_urls"]) == 0 """ A new list will not be created if there are only nonsensical urls (non valid) for it """ - added = save_urllist_content_by_name(account, "should be empty", {'iuygvb.uighblkj': {'tags': []}}) + added = save_urllist_content_by_name(account, "should be empty", {"iuygvb.uighblkj": {"tags": []}}) list_content = get_urllist_content(account=account, urllist_id=9001) - assert len(list_content['urls']) == 0 + assert len(list_content["urls"]) == 0 # list can be renamed renamed = rename_list(account=account, list_id=list_2.pk, new_name="A new name") @@ -390,13 +387,17 @@ def test_urllists(db, redis_server) -> None: assert renamed is True # lists can have an awfully long name and that will not be a problem, as it is truncated - renamed = rename_list(account=account, list_id=list_2.pk, new_name="alksdnalksdnlaksdnlasdknasldknaldnalskndnlaksn" - "asdnlkansdlknansldknasldnalkndwlkawdnlkdwanlkn" - "aksjdnaksjdndaslkdnlaklwkndlkawndwlakdnwlakkln" - "ansdknlaslkdnlaknwdlknkldawnldkwanlkadwnlkdawn" - "awdnawklnldndawlkndwalkndaklndwaklnwalkdnwakln" - "adlkwndlknawkdlnawldknawlkdnawklndklawnwkalnkn" - "awdlknawlkdnawlkdnalwdnawlkdnawkldnalkwndaklwn") + renamed = rename_list( + account=account, + list_id=list_2.pk, + new_name="alksdnalksdnlaksdnlasdknasldknaldnalskndnlaksn" + "asdnlkansdlknansldknasldnalkndwlkawdnlkdwanlkn" + "aksjdnaksjdndaslkdnlaklwkndlkawndwlakdnwlakkln" + "ansdknlaslkdnlaknwdlknkldawnldkwanlkadwnlkdawn" + "awdnawklnldndawlkndwalkndaklndwaklnwalkdnwakln" + "adlkwndlknawkdlnawldknawlkdnawklndklawnwkalnkn" + "awdlknawlkdnawlkdnalwdnawlkdnawkldnalkwndaklwn", + ) assert renamed is True @@ -409,23 +410,25 @@ def test_delete_url_from_urllist(db, redis_server): a2, _ = Account.objects.all().get_or_create(name="a2") l1 = get_or_create_list_by_name(a1, "l1") l2 = get_or_create_list_by_name(a2, "l2") - save_urllist_content_by_name(a1, "l1", { - 'test.nl': {'tags': []}, 'internet.nl': {'tags': []}, 'internetcleanup.foundation': {'tags': []}}) - save_urllist_content_by_name(a2, "l2", { - 'nu.nl': {'tags': []}, 'nos.nl': {'tags': []}, 'tweakers.net': {'tags': []}}) + save_urllist_content_by_name( + a1, "l1", {"test.nl": {"tags": []}, "internet.nl": {"tags": []}, "internetcleanup.foundation": {"tags": []}} + ) + save_urllist_content_by_name( + a2, "l2", {"nu.nl": {"tags": []}, "nos.nl": {"tags": []}, "tweakers.net": {"tags": []}} + ) assert l1 != l2 assert a1 != a2 assert Url.objects.all().count() == 6 - assert True is delete_url_from_urllist(a1, l1.id, u('test.nl')) + assert True is delete_url_from_urllist(a1, l1.id, u("test.nl")) # double delete results into nothing - assert False is delete_url_from_urllist(a1, l1.id, u('test.nl')) + assert False is delete_url_from_urllist(a1, l1.id, u("test.nl")) # a2 cannot delete something from the lists of a1, even if the url exist in the list from l1 - assert False is delete_url_from_urllist(a2, l1.id, u('test.nl')) + assert False is delete_url_from_urllist(a2, l1.id, u("test.nl")) # no crash on non-existing id's: - assert False is delete_url_from_urllist(a1, 990000, u('test.nl')) + assert False is delete_url_from_urllist(a1, 990000, u("test.nl")) assert False is delete_url_from_urllist(a1, l1.id, 9990000) assert Url.objects.all().count() == 6 diff --git a/dashboard/internet_nl_dashboard/tests/test_lockfile.py b/dashboard/internet_nl_dashboard/tests/test_lockfile.py index efb65279..9f653136 100644 --- a/dashboard/internet_nl_dashboard/tests/test_lockfile.py +++ b/dashboard/internet_nl_dashboard/tests/test_lockfile.py @@ -9,7 +9,7 @@ def test_lockfile(mocker): # Reset environment - remove_lock('test1') + remove_lock("test1") # 2020-01-01 == 1577836800 mocker.patch("os.path.getmtime", return_value=1577836800) @@ -17,24 +17,24 @@ def test_lockfile(mocker): with freeze_time("2020-01-01") as frozen_datetime: # Cannot open the same lock when it's open: - if temporary_file_lock('test1'): + if temporary_file_lock("test1"): sleep(1) - assert temporary_file_lock('test1') is False + assert temporary_file_lock("test1") is False # Remove it and then claim it: - remove_lock('test1') - assert temporary_file_lock('test1') is True + remove_lock("test1") + assert temporary_file_lock("test1") is True # Can't claim it again, unless we're far in the future - assert temporary_file_lock('test1') is False - assert lock_expired('test1', 300) is False + assert temporary_file_lock("test1") is False + assert lock_expired("test1", 300) is False # Now go into the future and see that it can be claimed frozen_datetime.move_to("2020-01-02") - assert lock_expired('test1', 300) is True - assert temporary_file_lock('test1') is True + assert lock_expired("test1", 300) is True + assert temporary_file_lock("test1") is True mocker.patch("os.path.getmtime", return_value=1577923200) - assert lock_expired('test1', 300) is False - assert temporary_file_lock('test1') is False - assert temporary_file_lock('test1') is False + assert lock_expired("test1", 300) is False + assert temporary_file_lock("test1") is False + assert temporary_file_lock("test1") is False diff --git a/dashboard/internet_nl_dashboard/tests/test_mail.py b/dashboard/internet_nl_dashboard/tests/test_mail.py index d6a857a8..32e83d90 100644 --- a/dashboard/internet_nl_dashboard/tests/test_mail.py +++ b/dashboard/internet_nl_dashboard/tests/test_mail.py @@ -5,37 +5,56 @@ from django_mail_admin.models import EmailTemplate, Log, Outbox, OutgoingEmail, TemplateVariable from websecmap.scanners.models import InternetNLV2Scan -from dashboard.internet_nl_dashboard.logic.mail import (email_configration_is_correct, generate_unsubscribe_code, - get_users_to_send_mail_to, send_scan_finished_mails, - unsubscribe) +from dashboard.internet_nl_dashboard.logic.mail import ( + email_configration_is_correct, + generate_unsubscribe_code, + get_users_to_send_mail_to, + send_scan_finished_mails, + unsubscribe, +) from dashboard.internet_nl_dashboard.models import Account, AccountInternetNLScan, DashboardUser, UrlList, UrlListReport def setup_test(): - user = User(**{'first_name': 'test', 'last_name': 'test', 'username': 'test', 'is_active': True}) + user = User(**{"first_name": "test", "last_name": "test", "username": "test", "is_active": True}) user.save() - account = Account(**{'name': 'test'}) + account = Account(**{"name": "test"}) account.save() # use a language that is not supported, so the system will fall back to english... - dashboarduser = DashboardUser(**{'mail_preferred_mail_address': 'info@example.com', 'mail_preferred_language': 'af', - 'mail_send_mail_after_scan_finished': True, 'account': account, 'user': user}) + dashboarduser = DashboardUser( + **{ + "mail_preferred_mail_address": "info@example.com", + "mail_preferred_language": "af", + "mail_send_mail_after_scan_finished": True, + "account": account, + "user": user, + } + ) dashboarduser.save() - urllist = UrlList(**{'name': '', 'account': account}) + urllist = UrlList(**{"name": "", "account": account}) urllist.save() - urllistreport = UrlListReport(**{'urllist': urllist, 'average_internet_nl_score': 42.42, - 'at_when': datetime.now(timezone.utc)}) + urllistreport = UrlListReport( + **{"urllist": urllist, "average_internet_nl_score": 42.42, "at_when": datetime.now(timezone.utc)} + ) urllistreport.save() - internetnlv2scan = InternetNLV2Scan(**{'type': 'web', 'scan_id': '123', 'state': 'finished'}) + internetnlv2scan = InternetNLV2Scan(**{"type": "web", "scan_id": "123", "state": "finished"}) internetnlv2scan.save() accountinternetnlscan = AccountInternetNLScan( - **{'account': account, 'scan': internetnlv2scan, 'urllist': urllist, 'started_on': datetime.now(timezone.utc), - 'report': urllistreport, 'finished_on': datetime.now(timezone.utc) + timedelta(hours=2)}) + **{ + "account": account, + "scan": internetnlv2scan, + "urllist": urllist, + "started_on": datetime.now(timezone.utc), + "report": urllistreport, + "finished_on": datetime.now(timezone.utc) + timedelta(hours=2), + } + ) accountinternetnlscan.save() # first template: @@ -62,7 +81,7 @@ def test_send_scan_finished_mails(db) -> None: assert sent_mail.template == EmailTemplate.objects.get(name="scan_finished_en") # values are saved as TemplateVariable - templatevariable = TemplateVariable.objects.all().filter(name='report_average_internet_nl_score').first() + templatevariable = TemplateVariable.objects.all().filter(name="report_average_internet_nl_score").first() assert templatevariable.name == "report_average_internet_nl_score" assert templatevariable.value == "42.42" assert templatevariable.email == sent_mail @@ -73,13 +92,13 @@ def test_send_scan_finished_mails(db) -> None: assert len(dbu.mail_after_mail_unsubscribe_code) == 128 # fail a unsubscription - unsubscribe('scan_finished', "fake code!") + unsubscribe("scan_finished", "fake code!") dbu = DashboardUser.objects.all().first() assert dbu.mail_send_mail_after_scan_finished is True assert len(dbu.mail_after_mail_unsubscribe_code) == 128 # unsubscribe from the feed - unsubscribe('scan_finished', dbu.mail_after_mail_unsubscribe_code) + unsubscribe("scan_finished", dbu.mail_after_mail_unsubscribe_code) dbu = DashboardUser.objects.all().first() assert len(dbu.mail_after_mail_unsubscribe_code) == 0 assert dbu.mail_send_mail_after_scan_finished is False @@ -152,25 +171,25 @@ def test_email_configration_is_correct(db): def test_urllistreport_get_previous_report(db): - account = Account(**{'name': 'test'}) + account = Account(**{"name": "test"}) account.save() - u = UrlList(**{'name': '', 'account': account}) + u = UrlList(**{"name": "", "account": account}) u.save() - urllistreport1 = UrlListReport(**{'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 1)}) + urllistreport1 = UrlListReport(**{"urllist": u, "average_internet_nl_score": 1, "at_when": datetime(2020, 5, 1)}) urllistreport1.save() - urllistreport2 = UrlListReport(**{'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 2)}) + urllistreport2 = UrlListReport(**{"urllist": u, "average_internet_nl_score": 1, "at_when": datetime(2020, 5, 2)}) urllistreport2.save() - urllistreport3 = UrlListReport(**{'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 3)}) + urllistreport3 = UrlListReport(**{"urllist": u, "average_internet_nl_score": 1, "at_when": datetime(2020, 5, 3)}) urllistreport3.save() - urllistreport4 = UrlListReport(**{'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 4)}) + urllistreport4 = UrlListReport(**{"urllist": u, "average_internet_nl_score": 1, "at_when": datetime(2020, 5, 4)}) urllistreport4.save() - urllistreport5 = UrlListReport(**{'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 5)}) + urllistreport5 = UrlListReport(**{"urllist": u, "average_internet_nl_score": 1, "at_when": datetime(2020, 5, 5)}) urllistreport5.save() assert urllistreport5.get_previous_report_from_this_list() == urllistreport4 diff --git a/dashboard/internet_nl_dashboard/tests/test_models.py b/dashboard/internet_nl_dashboard/tests/test_models.py index c4d3085d..e95c3e6e 100644 --- a/dashboard/internet_nl_dashboard/tests/test_models.py +++ b/dashboard/internet_nl_dashboard/tests/test_models.py @@ -9,7 +9,7 @@ def test_determine_next_scan_moment(): - preference = 'every half year' + preference = "every half year" tests = [ {"input": "2012-01-01", "outcome": datetime(year=2012, month=7, day=1, tzinfo=timezone.utc)}, {"input": "2012-06-30", "outcome": datetime(year=2012, month=7, day=1, tzinfo=timezone.utc)}, diff --git a/dashboard/internet_nl_dashboard/tests/test_password_storage.py b/dashboard/internet_nl_dashboard/tests/test_password_storage.py index 78b39b9b..83871e0b 100644 --- a/dashboard/internet_nl_dashboard/tests/test_password_storage.py +++ b/dashboard/internet_nl_dashboard/tests/test_password_storage.py @@ -12,7 +12,7 @@ def test_password_storage(db) -> None: # pylint: disable=invalid-name,unused-argument # normal usage - secret_password = 'My voice is my password.' + secret_password = "My voice is my password." account, _ = Account.objects.all().get_or_create(name="test") account.internet_nl_api_password = account.encrypt_password(secret_password) account.save() @@ -20,14 +20,14 @@ def test_password_storage(db) -> None: # pylint: disable=invalid-name,unused-ar assert account.decrypt_password() == secret_password # no password set, should throw a valueError - with pytest.raises(ValueError, match=r'.*not set.*'): + with pytest.raises(ValueError, match=r".*not set.*"): account, _ = Account.objects.all().get_or_create(name="value error") account.decrypt_password() # can create at once: account2 = Account() - account2.name = 'test 2' - account2.internet_nl_api_username = 'test 2' + account2.name = "test 2" + account2.internet_nl_api_username = "test 2" account2.internet_nl_api_password = Account.encrypt_password(secret_password) account2.save() @@ -36,7 +36,7 @@ def test_password_storage(db) -> None: # pylint: disable=invalid-name,unused-ar # If the field is stored as CharField instead of BinaryField, the type has become a string. # It's a bit hard to cast that string back into bytes. # verify that when retrieving all accounts, the password fields are still bytes. - accounts = Account.objects.all().filter(name__in=['test', 'test 2']) + accounts = Account.objects.all().filter(name__in=["test", "test 2"]) for account in accounts: # assert type(account.internet_nl_api_password) is bytes diff --git a/dashboard/internet_nl_dashboard/tests/test_release.py b/dashboard/internet_nl_dashboard/tests/test_release.py index 88619e36..8a9fca30 100644 --- a/dashboard/internet_nl_dashboard/tests/test_release.py +++ b/dashboard/internet_nl_dashboard/tests/test_release.py @@ -1,6 +1,7 @@ # stuff needed for releasing new versions of the dashboard. # the tutorial must work. import logging + from django.core.management import call_command log = logging.getLogger(__name__) @@ -16,9 +17,9 @@ def test_loading_fixtures(db, django_db_reset_sequences): # say 'what' foreign key or 'what' key was being attempted to be referred to. Because that would be convenient. log.debug("Start Loading fixtures") - call_command('loaddata', 'dashboard_production_default_account') - call_command('loaddata', 'dashboard_production_example_email_templates') - call_command('loaddata', 'dashboard_production_periodic_tasks') - call_command('loaddata', 'dashboard_production_default_scanner_configuration') - call_command('loaddata', 'dashboard_production_default_scan_policy') + call_command("loaddata", "dashboard_production_default_account") + call_command("loaddata", "dashboard_production_example_email_templates") + call_command("loaddata", "dashboard_production_periodic_tasks") + call_command("loaddata", "dashboard_production_default_scanner_configuration") + call_command("loaddata", "dashboard_production_default_scan_policy") log.debug("Done Loading fixtures") diff --git a/dashboard/internet_nl_dashboard/tests/test_report_comparison.py b/dashboard/internet_nl_dashboard/tests/test_report_comparison.py index 20d534cc..c0ddc397 100644 --- a/dashboard/internet_nl_dashboard/tests/test_report_comparison.py +++ b/dashboard/internet_nl_dashboard/tests/test_report_comparison.py @@ -5,19 +5,25 @@ from copy import deepcopy from dashboard.internet_nl_dashboard.logic.mail_admin_templates import store_template -from dashboard.internet_nl_dashboard.logic.report_comparison import (compare_report_in_detail, - determine_changes_in_ratings, - filter_comparison_report, key_calculation, - render_comparison_view) +from dashboard.internet_nl_dashboard.logic.report_comparison import ( + compare_report_in_detail, + determine_changes_in_ratings, + filter_comparison_report, + key_calculation, + render_comparison_view, +) from dashboard.internet_nl_dashboard.tests.common import get_json_file def test_key_report(current_path): calculation = key_calculation(get_json_file(f"{current_path}/test_report_comparison/report_1.json")) - assert calculation['calculation']['urls_by_url']['acc.dashboard.internet.nl']['endpoints_by_key'][ - 'dns_a_aaaa/0 IPv0']['ratings_by_type']['internet_nl_web_legacy_ipv6_webserver'][ - 'test_result'] == "failed" + assert ( + calculation["calculation"]["urls_by_url"]["acc.dashboard.internet.nl"]["endpoints_by_key"]["dns_a_aaaa/0 IPv0"][ + "ratings_by_type" + ]["internet_nl_web_legacy_ipv6_webserver"]["test_result"] + == "failed" + ) def test_compare_report_in_detail_equal_reports(current_path): @@ -26,168 +32,149 @@ def test_compare_report_in_detail_equal_reports(current_path): # there should be no differences when the report is compared to itself: result = compare_report_in_detail(calculation, calculation) assert result == { - 'comparison': { - 'acc.dashboard.internet.nl': { - - 'changes': { - 'improvement': 0, - 'neutral': 31, - 'regression': 0, - 'neutral_metrics': ['internet_nl_web_https_http_hsts', - 'internet_nl_web_ipv6_ws_reach', - 'internet_nl_web_https_tls_version', - 'internet_nl_web_appsecpriv_x_frame_options', - 'internet_nl_web_https_tls_cipherorder', - 'internet_nl_web_appsecpriv_x_content_type_options', - 'internet_nl_web_https_tls_ciphers', - 'internet_nl_web_appsecpriv_referrer_policy', - 'internet_nl_web_https_dane_exist', - 'internet_nl_web_https_tls_ocsp', - 'internet_nl_web_appsecpriv_csp', - 'internet_nl_web_ipv6_ns_address', - 'internet_nl_web_https_tls_clientreneg', - 'internet_nl_web_ipv6_ns_reach', - 'internet_nl_web_https_http_compress', - 'internet_nl_web_https_cert_sig', - 'internet_nl_web_https_http_redirect', - 'internet_nl_web_https_dane_valid', - 'internet_nl_web_ipv6_ws_address', - 'internet_nl_web_https_tls_0rtt', - 'internet_nl_web_dnssec_exist', - 'internet_nl_web_https_cert_domain', - 'internet_nl_web_https_tls_keyexchangehash', - 'internet_nl_web_https_tls_secreneg', - 'internet_nl_web_ipv6_ws_similar', - 'internet_nl_web_dnssec_valid', - 'internet_nl_web_https_tls_compress', - 'internet_nl_web_https_cert_chain', - 'internet_nl_web_https_cert_pubkey', - 'internet_nl_web_https_http_available', - 'internet_nl_web_https_tls_keyexchange'], - 'regressed_metrics': [], - 'improved_metrics': [], - }, - 'computed_domain': 'internet', - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'acc.dashboard', - 'computed_suffix': 'nl', - 'test_results_from_internet_nl_available': True, - 'new': { - 'report': 'https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/', - 'score': 79 - }, - 'old': { - 'report': 'https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/', - 'score': 79 + "comparison": { + "acc.dashboard.internet.nl": { + "changes": { + "improvement": 0, + "neutral": 31, + "regression": 0, + "neutral_metrics": [ + "internet_nl_web_https_http_hsts", + "internet_nl_web_ipv6_ws_reach", + "internet_nl_web_https_tls_version", + "internet_nl_web_appsecpriv_x_frame_options", + "internet_nl_web_https_tls_cipherorder", + "internet_nl_web_appsecpriv_x_content_type_options", + "internet_nl_web_https_tls_ciphers", + "internet_nl_web_appsecpriv_referrer_policy", + "internet_nl_web_https_dane_exist", + "internet_nl_web_https_tls_ocsp", + "internet_nl_web_appsecpriv_csp", + "internet_nl_web_ipv6_ns_address", + "internet_nl_web_https_tls_clientreneg", + "internet_nl_web_ipv6_ns_reach", + "internet_nl_web_https_http_compress", + "internet_nl_web_https_cert_sig", + "internet_nl_web_https_http_redirect", + "internet_nl_web_https_dane_valid", + "internet_nl_web_ipv6_ws_address", + "internet_nl_web_https_tls_0rtt", + "internet_nl_web_dnssec_exist", + "internet_nl_web_https_cert_domain", + "internet_nl_web_https_tls_keyexchangehash", + "internet_nl_web_https_tls_secreneg", + "internet_nl_web_ipv6_ws_similar", + "internet_nl_web_dnssec_valid", + "internet_nl_web_https_tls_compress", + "internet_nl_web_https_cert_chain", + "internet_nl_web_https_cert_pubkey", + "internet_nl_web_https_http_available", + "internet_nl_web_https_tls_keyexchange", + ], + "regressed_metrics": [], + "improved_metrics": [], }, - 'url': 'acc.dashboard.internet.nl' + "computed_domain": "internet", + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "acc.dashboard", + "computed_suffix": "nl", + "test_results_from_internet_nl_available": True, + "new": {"report": "https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/", "score": 79}, + "old": {"report": "https://batch.internet.nl/site/acc.dashboard.internet.nl/200719/", "score": 79}, + "url": "acc.dashboard.internet.nl", }, - # this is an entirely empty report, thus there will be no data here... # Should we skip it in it's entirity? No final report should say something about no metrics. # just like on the dashboard. - 'pl.internet.nl': { + "pl.internet.nl": { # There is no endpoint to get changes from. - 'changes': { - 'improvement': 0, - 'neutral': 0, - 'regression': 0, - 'neutral_metrics': [], - 'regressed_metrics': [], - 'improved_metrics': [], + "changes": { + "improvement": 0, + "neutral": 0, + "regression": 0, + "neutral_metrics": [], + "regressed_metrics": [], + "improved_metrics": [], }, - 'computed_domain': 'internet', - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'pl', - 'computed_suffix': 'nl', - 'test_results_from_internet_nl_available': False, + "computed_domain": "internet", + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "pl", + "computed_suffix": "nl", + "test_results_from_internet_nl_available": False, # There is no endpoint and thus no comparison - 'new': { - 'report': '', - 'score': 0 - }, - 'old': { - 'report': '', - 'score': 0 - }, - 'url': 'pl.internet.nl' + "new": {"report": "", "score": 0}, + "old": {"report": "", "score": 0}, + "url": "pl.internet.nl", }, - # www.internet.nl is missing a few ratings, so the neutral should be lower # It misses 2: internet_nl_web_https_http_hsts, internet_nl_web_https_tls_ocsp - 'www.internet.nl': { - 'changes': { - 'improvement': 0, - 'neutral': 29, - 'regression': 0, - 'neutral_metrics': ['internet_nl_web_https_tls_version', - 'internet_nl_web_ipv6_ns_address', - 'internet_nl_web_https_tls_clientreneg', - 'internet_nl_web_ipv6_ns_reach', - 'internet_nl_web_https_http_compress', - 'internet_nl_web_https_cert_sig', - 'internet_nl_web_https_http_redirect', - 'internet_nl_web_https_dane_valid', - 'internet_nl_web_appsecpriv_x_frame_options', - 'internet_nl_web_https_tls_cipherorder', - 'internet_nl_web_ipv6_ws_address', - 'internet_nl_web_appsecpriv_x_content_type_options', - 'internet_nl_web_https_tls_0rtt', - 'internet_nl_web_dnssec_exist', - 'internet_nl_web_https_cert_domain', - 'internet_nl_web_https_tls_keyexchangehash', - 'internet_nl_web_https_dane_exist', - 'internet_nl_web_https_tls_secreneg', - 'internet_nl_web_ipv6_ws_similar', - 'internet_nl_web_dnssec_valid', - 'internet_nl_web_https_tls_compress', - 'internet_nl_web_https_cert_chain', - 'internet_nl_web_https_tls_ciphers', - 'internet_nl_web_https_cert_pubkey', - 'internet_nl_web_ipv6_ws_reach', - 'internet_nl_web_https_http_available', - 'internet_nl_web_https_tls_keyexchange', - 'internet_nl_web_appsecpriv_csp', - 'internet_nl_web_appsecpriv_referrer_policy'], - 'regressed_metrics': [], - 'improved_metrics': [], - }, - 'computed_domain': 'internet', - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'www', - 'computed_suffix': 'nl', - 'test_results_from_internet_nl_available': True, - 'new': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/200737/', - 'score': 100 - }, - 'old': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/200737/', - 'score': 100 + "www.internet.nl": { + "changes": { + "improvement": 0, + "neutral": 29, + "regression": 0, + "neutral_metrics": [ + "internet_nl_web_https_tls_version", + "internet_nl_web_ipv6_ns_address", + "internet_nl_web_https_tls_clientreneg", + "internet_nl_web_ipv6_ns_reach", + "internet_nl_web_https_http_compress", + "internet_nl_web_https_cert_sig", + "internet_nl_web_https_http_redirect", + "internet_nl_web_https_dane_valid", + "internet_nl_web_appsecpriv_x_frame_options", + "internet_nl_web_https_tls_cipherorder", + "internet_nl_web_ipv6_ws_address", + "internet_nl_web_appsecpriv_x_content_type_options", + "internet_nl_web_https_tls_0rtt", + "internet_nl_web_dnssec_exist", + "internet_nl_web_https_cert_domain", + "internet_nl_web_https_tls_keyexchangehash", + "internet_nl_web_https_dane_exist", + "internet_nl_web_https_tls_secreneg", + "internet_nl_web_ipv6_ws_similar", + "internet_nl_web_dnssec_valid", + "internet_nl_web_https_tls_compress", + "internet_nl_web_https_cert_chain", + "internet_nl_web_https_tls_ciphers", + "internet_nl_web_https_cert_pubkey", + "internet_nl_web_ipv6_ws_reach", + "internet_nl_web_https_http_available", + "internet_nl_web_https_tls_keyexchange", + "internet_nl_web_appsecpriv_csp", + "internet_nl_web_appsecpriv_referrer_policy", + ], + "regressed_metrics": [], + "improved_metrics": [], }, - 'url': 'www.internet.nl' - } - + "computed_domain": "internet", + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "www", + "computed_suffix": "nl", + "test_results_from_internet_nl_available": True, + "new": {"report": "https://batch.internet.nl/site/www.internet.nl/200737/", "score": 100}, + "old": {"report": "https://batch.internet.nl/site/www.internet.nl/200737/", "score": 100}, + "url": "www.internet.nl", + }, }, - - 'new': { - 'average_internet_nl_score': 96.54, - 'data_from': '2020-10-15 00:19:28.788709+00:00', - 'number_of_urls': 25, - 'report_id': 1267, - 'urllist_id': 1 + "new": { + "average_internet_nl_score": 96.54, + "data_from": "2020-10-15 00:19:28.788709+00:00", + "number_of_urls": 25, + "report_id": 1267, + "urllist_id": 1, }, - 'old': { - 'average_internet_nl_score': 96.54, - 'data_from': '2020-10-15 00:19:28.788709+00:00', - 'number_of_urls': 25, - 'report_id': 1267, - 'urllist_id': 1 + "old": { + "average_internet_nl_score": 96.54, + "data_from": "2020-10-15 00:19:28.788709+00:00", + "number_of_urls": 25, + "report_id": 1267, + "urllist_id": 1, }, - 'summary': {'improvement': 0, 'neutral': 60, 'regression': 0}, + "summary": {"improvement": 0, "neutral": 60, "regression": 0}, # Should be an empty list, as both reports are the same - 'urls_exclusive_in_new_report': [], - 'urls_exclusive_in_old_report': [] + "urls_exclusive_in_new_report": [], + "urls_exclusive_in_old_report": [], } @@ -211,21 +198,21 @@ def test_compare_report_detail_differences(): "ratings_by_type": { "internet_nl_web_https_tls_version": { "test_result": "passed", - "simple_progression": 400 + "simple_progression": 400, }, # this is intentionally missing in the negative report, to verify # that the correct keys are used from the correct report. "internet_nl_web_https_tls_cipherorder": { "test_result": "passed", - "simple_progression": 400 + "simple_progression": 400, }, "internet_nl_score": { "internet_nl_score": 11.11, - "internet_nl_url": "https://batch.internet.nl/site/www.internet.nl/1/" - } - } + "internet_nl_url": "https://batch.internet.nl/site/www.internet.nl/1/", + }, + }, } - } + }, }, # This is an extra domain that will be missing in the negative report. # This will create a neutral rating. @@ -240,24 +227,24 @@ def test_compare_report_detail_differences(): "ratings_by_type": { "internet_nl_web_https_tls_version": { "test_result": "passed", - "simple_progression": 400 + "simple_progression": 400, }, # this is intentionally missing in the negative report, to verify # that the correct keys are used from the correct report. "internet_nl_web_https_tls_cipherorder": { "test_result": "passed", - "simple_progression": 400 + "simple_progression": 400, }, "internet_nl_score": { "internet_nl_score": 11.11, - "internet_nl_url": "https://batch.internet.nl/site/www.internet.nl/1/" - } - } + "internet_nl_url": "https://batch.internet.nl/site/www.internet.nl/1/", + }, + }, } - } - } + }, + }, } - } + }, } negative_report = { @@ -279,138 +266,124 @@ def test_compare_report_detail_differences(): "ratings_by_type": { "internet_nl_web_https_tls_version": { "test_result": "failed", - "simple_progression": 100 + "simple_progression": 100, }, "internet_nl_score": { "internet_nl_score": 22.22, - "internet_nl_url": "https://batch.internet.nl/site/www.internet.nl/2/" - } - } + "internet_nl_url": "https://batch.internet.nl/site/www.internet.nl/2/", + }, + }, } - } + }, } } - } + }, } comparison = compare_report_in_detail(positive_report, negative_report) assert comparison == { - 'comparison': { - 'www.internet.nl': { - 'changes': { - 'improvement': 1, - 'neutral': 1, - 'regression': 0, - 'improved_metrics': ['internet_nl_web_https_tls_version'], - 'regressed_metrics': [], - 'neutral_metrics': ['internet_nl_web_https_tls_cipherorder'] - }, - 'computed_domain': 'internet', - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'www', - 'computed_suffix': 'nl', - 'test_results_from_internet_nl_available': True, - 'new': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/1/', - 'score': 11.11 + "comparison": { + "www.internet.nl": { + "changes": { + "improvement": 1, + "neutral": 1, + "regression": 0, + "improved_metrics": ["internet_nl_web_https_tls_version"], + "regressed_metrics": [], + "neutral_metrics": ["internet_nl_web_https_tls_cipherorder"], }, - 'old': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/2/', - 'score': 22.22 - }, - 'url': 'www.internet.nl' + "computed_domain": "internet", + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "www", + "computed_suffix": "nl", + "test_results_from_internet_nl_available": True, + "new": {"report": "https://batch.internet.nl/site/www.internet.nl/1/", "score": 11.11}, + "old": {"report": "https://batch.internet.nl/site/www.internet.nl/2/", "score": 22.22}, + "url": "www.internet.nl", }, # The extra domain is not in the negative report, it will thus not have 'old' data. - 'extradomain.internet.nl': { - 'changes': { - 'improvement': 0, - 'neutral': 2, - 'regression': 0, - 'improved_metrics': [], - 'regressed_metrics': [], - 'neutral_metrics': ['internet_nl_web_https_tls_version', 'internet_nl_web_https_tls_cipherorder'] - }, - 'computed_domain': 'internet', - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'extradomain', - 'computed_suffix': 'nl', - 'test_results_from_internet_nl_available': True, - 'new': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/1/', - 'score': 11.11 + "extradomain.internet.nl": { + "changes": { + "improvement": 0, + "neutral": 2, + "regression": 0, + "improved_metrics": [], + "regressed_metrics": [], + "neutral_metrics": ["internet_nl_web_https_tls_version", "internet_nl_web_https_tls_cipherorder"], }, - 'old': { - 'report': '', - 'score': 0, + "computed_domain": "internet", + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "extradomain", + "computed_suffix": "nl", + "test_results_from_internet_nl_available": True, + "new": {"report": "https://batch.internet.nl/site/www.internet.nl/1/", "score": 11.11}, + "old": { + "report": "", + "score": 0, }, - 'url': 'extradomain.internet.nl' - } + "url": "extradomain.internet.nl", + }, }, - 'new': { - 'average_internet_nl_score': 11.11, - 'data_from': '2020-10-10 10:10:10.101010+00:00', - 'number_of_urls': 1, - 'report_id': 1, - 'urllist_id': 1 + "new": { + "average_internet_nl_score": 11.11, + "data_from": "2020-10-10 10:10:10.101010+00:00", + "number_of_urls": 1, + "report_id": 1, + "urllist_id": 1, }, - 'old': { - 'average_internet_nl_score': 22.22, - 'data_from': '2020-02-02 20:20:20.202020+00:00', - 'number_of_urls': 2, - 'report_id': 2, - 'urllist_id': 2 + "old": { + "average_internet_nl_score": 22.22, + "data_from": "2020-02-02 20:20:20.202020+00:00", + "number_of_urls": 2, + "report_id": 2, + "urllist_id": 2, }, - 'summary': {'improvement': 1, 'neutral': 3, 'regression': 0}, - 'urls_exclusive_in_new_report': ['extradomain.internet.nl'], - 'urls_exclusive_in_old_report': []} + "summary": {"improvement": 1, "neutral": 3, "regression": 0}, + "urls_exclusive_in_new_report": ["extradomain.internet.nl"], + "urls_exclusive_in_old_report": [], + } # now let's swap it around: comparison = compare_report_in_detail(negative_report, positive_report) assert comparison == { - 'comparison': { - 'www.internet.nl': { - 'changes': { + "comparison": { + "www.internet.nl": { + "changes": { # only a regression, there are no further matching fields - 'improvement': 0, - 'neutral': 0, - 'regression': 1, - 'improved_metrics': [], - 'regressed_metrics': ['internet_nl_web_https_tls_version'], - 'neutral_metrics': [] - }, - 'computed_domain': 'internet', - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'www', - 'computed_suffix': 'nl', - 'test_results_from_internet_nl_available': True, - 'new': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/2/', - 'score': 22.22 + "improvement": 0, + "neutral": 0, + "regression": 1, + "improved_metrics": [], + "regressed_metrics": ["internet_nl_web_https_tls_version"], + "neutral_metrics": [], }, - 'old': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/1/', - 'score': 11.11 - }, - 'url': 'www.internet.nl' + "computed_domain": "internet", + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "www", + "computed_suffix": "nl", + "test_results_from_internet_nl_available": True, + "new": {"report": "https://batch.internet.nl/site/www.internet.nl/2/", "score": 22.22}, + "old": {"report": "https://batch.internet.nl/site/www.internet.nl/1/", "score": 11.11}, + "url": "www.internet.nl", }, }, - 'new': { - 'average_internet_nl_score': 22.22, - 'data_from': '2020-02-02 20:20:20.202020+00:00', - 'number_of_urls': 2, - 'report_id': 2, - 'urllist_id': 2 + "new": { + "average_internet_nl_score": 22.22, + "data_from": "2020-02-02 20:20:20.202020+00:00", + "number_of_urls": 2, + "report_id": 2, + "urllist_id": 2, }, - 'old': { - 'average_internet_nl_score': 11.11, - 'data_from': '2020-10-10 10:10:10.101010+00:00', - 'number_of_urls': 1, - 'report_id': 1, - 'urllist_id': 1 + "old": { + "average_internet_nl_score": 11.11, + "data_from": "2020-10-10 10:10:10.101010+00:00", + "number_of_urls": 1, + "report_id": 1, + "urllist_id": 1, }, - 'summary': {'improvement': 0, 'neutral': 0, 'regression': 1}, - 'urls_exclusive_in_new_report': [], - 'urls_exclusive_in_old_report': ['extradomain.internet.nl'] + "summary": {"improvement": 0, "neutral": 0, "regression": 1}, + "urls_exclusive_in_new_report": [], + "urls_exclusive_in_old_report": ["extradomain.internet.nl"], } @@ -422,63 +395,63 @@ def test_filter_comparison_report(): # the comparison reports have been made a little smaller, so they can be maintained # more easily and there is less code. comparison = { - 'comparison': { - 'www.internet.nl': { - 'changes': { - 'improvement': 1, - 'improved_metrics': ['internet_nl_web_https_tls_version'], + "comparison": { + "www.internet.nl": { + "changes": { + "improvement": 1, + "improved_metrics": ["internet_nl_web_https_tls_version"], }, - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'www', - 'url': 'www.internet.nl' + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "www", + "url": "www.internet.nl", }, - 'internet.nl': { - 'changes': { - 'improvement': 1, - 'improved_metrics': ['internet_nl_web_https_tls_version'], + "internet.nl": { + "changes": { + "improvement": 1, + "improved_metrics": ["internet_nl_web_https_tls_version"], }, - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': '', - 'url': 'internet.nl' + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "", + "url": "internet.nl", }, - 'example.com': { - 'changes': { - 'improvement': 1, - 'improved_metrics': ['internet_nl_web_https_tls_version'], + "example.com": { + "changes": { + "improvement": 1, + "improved_metrics": ["internet_nl_web_https_tls_version"], }, - 'computed_domain_and_suffix': 'example.com', - 'computed_subdomain': '', - 'url': 'example.com' + "computed_domain_and_suffix": "example.com", + "computed_subdomain": "", + "url": "example.com", }, - 'ignoreddomain.example.com': { - 'changes': { - 'improvement': 0, - 'regression': 1, - 'improved_metrics': [], + "ignoreddomain.example.com": { + "changes": { + "improvement": 0, + "regression": 1, + "improved_metrics": [], }, - 'computed_domain_and_suffix': 'example.com', - 'computed_subdomain': '', - 'url': 'ignoreddomain.example.com' + "computed_domain_and_suffix": "example.com", + "computed_subdomain": "", + "url": "ignoreddomain.example.com", }, - 'afterinternet.nl.wexample.com': { - 'changes': { - 'improvement': 1, - 'improved_metrics': ['internet_nl_web_https_tls_version'], + "afterinternet.nl.wexample.com": { + "changes": { + "improvement": 1, + "improved_metrics": ["internet_nl_web_https_tls_version"], }, - 'computed_domain_and_suffix': 'wexample.com', - 'computed_subdomain': 'afterinternet.nl', - 'url': 'afterinternet.nl.wexample.com' - } + "computed_domain_and_suffix": "wexample.com", + "computed_subdomain": "afterinternet.nl", + "url": "afterinternet.nl.wexample.com", + }, } } # Expected: # ignoreddomain.example.com will be ignored output = filter_comparison_report(comparison, impact="improvement") - assert output[0]['url'] == "example.com" - assert output[1]['url'] == "internet.nl" - assert output[2]['url'] == "www.internet.nl" - assert output[3]['url'] == "afterinternet.nl.wexample.com" + assert output[0]["url"] == "example.com" + assert output[1]["url"] == "internet.nl" + assert output[2]["url"] == "www.internet.nl" + assert output[3]["url"] == "afterinternet.nl.wexample.com" assert len(output) == 4 @@ -501,65 +474,55 @@ def test_render_comparison_view(db): {% endfor %} - """ + """, ) # no input, no output and no crashes: - output = render_comparison_view( - [], - impact="improvement", - language="nl" - ) + output = render_comparison_view([], impact="improvement", language="nl") assert output == "" # See that rendering is happening, for both improvement and regression. # The point is to see that the templating stuff works, not so much what HTML is being generated. # And that translations are correct. comparison = { - 'comparison': { - 'www.internet.nl': { - 'changes': { - 'improvement': 1, - 'neutral': 0, - 'regression': 1, - 'improved_metrics': ['internet_nl_web_https_tls_version'], - 'regressed_metrics': ['internet_nl_web_https_tls_version'], - 'neutral_metrics': [] - }, - 'computed_domain': 'internet', - 'computed_domain_and_suffix': 'internet.nl', - 'computed_subdomain': 'www', - 'computed_suffix': 'nl', - 'test_results_from_internet_nl_available': True, - 'new': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/2/', - 'score': 22.22 + "comparison": { + "www.internet.nl": { + "changes": { + "improvement": 1, + "neutral": 0, + "regression": 1, + "improved_metrics": ["internet_nl_web_https_tls_version"], + "regressed_metrics": ["internet_nl_web_https_tls_version"], + "neutral_metrics": [], }, - 'old': { - 'report': 'https://batch.internet.nl/site/www.internet.nl/1/', - 'score': 11.11 - }, - 'url': 'www.internet.nl' + "computed_domain": "internet", + "computed_domain_and_suffix": "internet.nl", + "computed_subdomain": "www", + "computed_suffix": "nl", + "test_results_from_internet_nl_available": True, + "new": {"report": "https://batch.internet.nl/site/www.internet.nl/2/", "score": 22.22}, + "old": {"report": "https://batch.internet.nl/site/www.internet.nl/1/", "score": 11.11}, + "url": "www.internet.nl", }, }, - 'new': { - 'average_internet_nl_score': 22.22, - 'data_from': '2020-02-02 20:20:20.202020+00:00', - 'number_of_urls': 2, - 'report_id': 2, - 'urllist_id': 2 + "new": { + "average_internet_nl_score": 22.22, + "data_from": "2020-02-02 20:20:20.202020+00:00", + "number_of_urls": 2, + "report_id": 2, + "urllist_id": 2, }, - 'old': { - 'average_internet_nl_score': 11.11, - 'data_from': '2020-10-10 10:10:10.101010+00:00', - 'number_of_urls': 1, - 'report_id': 1, - 'urllist_id': 1 + "old": { + "average_internet_nl_score": 11.11, + "data_from": "2020-10-10 10:10:10.101010+00:00", + "number_of_urls": 1, + "report_id": 1, + "urllist_id": 1, }, - 'summary': {'improvement': 0, 'neutral': 0, 'regression': 1}, - 'urls_exclusive_in_new_report': [], - 'urls_exclusive_in_old_report': ['extradomain.internet.nl'] + "summary": {"improvement": 0, "neutral": 0, "regression": 1}, + "urls_exclusive_in_new_report": [], + "urls_exclusive_in_old_report": ["extradomain.internet.nl"], } # the first time called the correct translation language is used, the second time it's not... @@ -567,11 +530,7 @@ def test_render_comparison_view(db): # code running in the same thread. # # Do not _change_ the comparison report itself, as it might be used for different things. Or reused in a testcase. data = filter_comparison_report(deepcopy(comparison), "improvement") - output = render_comparison_view( - data, - impact="improvement", - language="nl" - ) + output = render_comparison_view(data, impact="improvement", language="nl") # template engines add newlines and whitespace, which makes comparison harder. output = output.strip() @@ -591,108 +550,87 @@ def test_render_comparison_view(db): # test that translating to english also works. data = filter_comparison_report(deepcopy(comparison), "improvement") - output = render_comparison_view( - data, - impact="improvement", - language="en" - ) + output = render_comparison_view(data, impact="improvement", language="en") assert "TLS version" in output def test_determine_changes(current_path): set_1 = { # The only metric that is included - "internet_nl_web_https_http_hsts": { - "test_result": "failed", - "simple_progression": 100 - }, + "internet_nl_web_https_http_hsts": {"test_result": "failed", "simple_progression": 100}, # Categories are ignored - "internet_nl_web_ipv6": { - "test_result": "failed", - "simple_progression": 100 - }, + "internet_nl_web_ipv6": {"test_result": "failed", "simple_progression": 100}, # legacy fields are ignored - "internet_nl_web_legacy_category_ipv6": { - "test_result": "failed", - "simple_progression": 100 - }, + "internet_nl_web_legacy_category_ipv6": {"test_result": "failed", "simple_progression": 100}, # score fields are ignored: "internet_nl_score": { "test_result": 100, }, # deleted field, will also be ignored: - "internet_nl_web_appsecpriv_x_xss_protection": { - "ok": 1, - "scan": 51, - "simple_progression": 0 - } + "internet_nl_web_appsecpriv_x_xss_protection": {"ok": 1, "scan": 51, "simple_progression": 0}, } - assert determine_changes_in_ratings( - set_1, - set_1 - ) == {'improvement': 0, 'neutral': 1, 'regression': 0, - 'improved_metrics': [], 'regressed_metrics': [], 'neutral_metrics': ['internet_nl_web_https_http_hsts']} + assert determine_changes_in_ratings(set_1, set_1) == { + "improvement": 0, + "neutral": 1, + "regression": 0, + "improved_metrics": [], + "regressed_metrics": [], + "neutral_metrics": ["internet_nl_web_https_http_hsts"], + } # Test improvement assert determine_changes_in_ratings( - { - "internet_nl_web_https_http_hsts": { - "test_result": "passed", - "simple_progression": 400 - } - }, - { - "internet_nl_web_https_http_hsts": { - "test_result": "failed", - "simple_progression": 100 - } - } - ) == {'improvement': 1, 'neutral': 0, 'regression': 0, - 'improved_metrics': ['internet_nl_web_https_http_hsts'], 'regressed_metrics': [], 'neutral_metrics': []} + {"internet_nl_web_https_http_hsts": {"test_result": "passed", "simple_progression": 400}}, + {"internet_nl_web_https_http_hsts": {"test_result": "failed", "simple_progression": 100}}, + ) == { + "improvement": 1, + "neutral": 0, + "regression": 0, + "improved_metrics": ["internet_nl_web_https_http_hsts"], + "regressed_metrics": [], + "neutral_metrics": [], + } # test regression assert determine_changes_in_ratings( - { - "internet_nl_web_https_http_hsts": { - "test_result": "failed", - "simple_progression": 100 - } - }, - { - "internet_nl_web_https_http_hsts": { - "test_result": "passed", - "simple_progression": 400 - } - } - ) == {'improvement': 0, 'neutral': 0, 'regression': 1, - 'improved_metrics': [], 'regressed_metrics': ['internet_nl_web_https_http_hsts'], 'neutral_metrics': []} + {"internet_nl_web_https_http_hsts": {"test_result": "failed", "simple_progression": 100}}, + {"internet_nl_web_https_http_hsts": {"test_result": "passed", "simple_progression": 400}}, + ) == { + "improvement": 0, + "neutral": 0, + "regression": 1, + "improved_metrics": [], + "regressed_metrics": ["internet_nl_web_https_http_hsts"], + "neutral_metrics": [], + } # test missing key will result in neutral result assert determine_changes_in_ratings( - { - "internet_nl_web_https_http_hsts": { - "test_result": "failed", - "simple_progression": 100 - } - }, + {"internet_nl_web_https_http_hsts": {"test_result": "failed", "simple_progression": 100}}, { "this_is_a_non_matching_result_and_will_return_into_no_comparison_thus_neutral": { "test_result": "passed", - "simple_progression": 400 + "simple_progression": 400, } - } - ) == {'improvement': 0, 'neutral': 1, 'regression': 0, - 'improved_metrics': [], 'regressed_metrics': [], 'neutral_metrics': ['internet_nl_web_https_http_hsts']} + }, + ) == { + "improvement": 0, + "neutral": 1, + "regression": 0, + "improved_metrics": [], + "regressed_metrics": [], + "neutral_metrics": ["internet_nl_web_https_http_hsts"], + } # and test missing key with an entirely empty old report assert determine_changes_in_ratings( - { - "internet_nl_web_https_http_hsts": { - "test_result": "failed", - "simple_progression": 100 - } - }, - {} - ) == {'improvement': 0, 'neutral': 1, 'regression': 0, - 'improved_metrics': [], 'regressed_metrics': [], 'neutral_metrics': ['internet_nl_web_https_http_hsts']} + {"internet_nl_web_https_http_hsts": {"test_result": "failed", "simple_progression": 100}}, {} + ) == { + "improvement": 0, + "neutral": 1, + "regression": 0, + "improved_metrics": [], + "regressed_metrics": [], + "neutral_metrics": ["internet_nl_web_https_http_hsts"], + } diff --git a/dashboard/internet_nl_dashboard/tests/test_report_upgrade.py b/dashboard/internet_nl_dashboard/tests/test_report_upgrade.py index 4187aee0..a2249431 100644 --- a/dashboard/internet_nl_dashboard/tests/test_report_upgrade.py +++ b/dashboard/internet_nl_dashboard/tests/test_report_upgrade.py @@ -9,8 +9,12 @@ from websecmap.organizations.models import Url from websecmap.reporting.diskreport import retrieve_report, store_report -from dashboard.internet_nl_dashboard.logic.report import (add_keyed_ratings, add_percentages_to_statistics, - add_statistics_over_ratings, remove_comply_or_explain) +from dashboard.internet_nl_dashboard.logic.report import ( + add_keyed_ratings, + add_percentages_to_statistics, + add_statistics_over_ratings, + remove_comply_or_explain, +) from dashboard.internet_nl_dashboard.models import Account, AccountInternetNLScan, UrlList, UrlListReport from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import upgrade_report_with_unscannable_urls @@ -20,9 +24,21 @@ def test_report_upgrade(db, monkeypatch) -> None: # megaupload.com will never be scannable, and the rest can have an endpoint and might be in the report # already because of this (but without endpoints) - urls = ['akamaihd.net', 'apple.com', 'bp.blogspot.com', 'clickbank.net', 'cocolog-nifty.com', 'fda.gov', - 'geocities.jp', 'ggpht.com', 'googleusercontent.com', 'megaupload.com', 'nhk.or.jp', - 'ssl-images-amazon.com', 'ytimg.com'] + urls = [ + "akamaihd.net", + "apple.com", + "bp.blogspot.com", + "clickbank.net", + "cocolog-nifty.com", + "fda.gov", + "geocities.jp", + "ggpht.com", + "googleusercontent.com", + "megaupload.com", + "nhk.or.jp", + "ssl-images-amazon.com", + "ytimg.com", + ] # create the list, code from test domain management: account, created = Account.objects.all().get_or_create(name="test") @@ -116,7 +132,7 @@ def test_report_upgrade(db, monkeypatch) -> None: "comply_or_explain_explanation_valid_until": "", "comply_or_explain_valid_at_time_of_report": False, "scan": 114575, - "scan_type": "internet_nl_web_ipv6_ws_address" + "scan_type": "internet_nl_web_ipv6_ws_address", }, { "type": "internet_nl_web_dnssec_valid", @@ -136,7 +152,7 @@ def test_report_upgrade(db, monkeypatch) -> None: "comply_or_explain_explanation_valid_until": "", "comply_or_explain_valid_at_time_of_report": False, "scan": 114556, - "scan_type": "internet_nl_web_dnssec_valid" + "scan_type": "internet_nl_web_dnssec_valid", }, ], "high": 19, @@ -145,7 +161,7 @@ def test_report_upgrade(db, monkeypatch) -> None: "ok": 15, "explained_high": 0, "explained_medium": 0, - "explained_low": 0 + "explained_low": 0, } ], "total_issues": 26, @@ -181,11 +197,11 @@ def test_report_upgrade(db, monkeypatch) -> None: "explained_total_endpoint_issues": 0, "explained_endpoint_issues_high": 0, "explained_endpoint_issues_medium": 0, - "explained_endpoint_issues_low": 0 + "explained_endpoint_issues_low": 0, } ], "total_issues": 26, - "name": "Unscannable Web + one scannable" + "name": "Unscannable Web + one scannable", } fake_report = UrlListReport() @@ -196,13 +212,13 @@ def test_report_upgrade(db, monkeypatch) -> None: # First check if we are removing the comply_or_explain keys, mainly to save data: remove_comply_or_explain(fake_calculation) - assert "explained_endpoint_issues_high" not in fake_calculation['urls'][0] - assert "comply_or_explain_explanation" not in fake_calculation['urls'][0]['endpoints'][0]["ratings"][0] + assert "explained_endpoint_issues_high" not in fake_calculation["urls"][0] + assert "comply_or_explain_explanation" not in fake_calculation["urls"][0]["endpoints"][0]["ratings"][0] # Now add ratings based on keys, which makes direct access possible: add_keyed_ratings(fake_calculation) - assert "ratings_by_type" in fake_calculation['urls'][0]['endpoints'][0] - assert "internet_nl_web_ipv6_ws_address" in fake_calculation['urls'][0]['endpoints'][0]['ratings_by_type'] + assert "ratings_by_type" in fake_calculation["urls"][0]["endpoints"][0] + assert "internet_nl_web_ipv6_ws_address" in fake_calculation["urls"][0]["endpoints"][0]["ratings_by_type"] # Add graph statistics, so the graphs can be instantly created based on report data add_statistics_over_ratings(fake_calculation) @@ -217,14 +233,14 @@ def test_report_upgrade(db, monkeypatch) -> None: # and make sure the report is complete: meaning that all urls requested are present, even though they # could not be scanned. So a top 100 stays a top 100. - assert (len(fake_calculation['urls']) == 1) + assert len(fake_calculation["urls"]) == 1 store_report(fake_report.id, "UrlListReport", fake_calculation) upgrade_report_with_unscannable_urls(fake_report.id, scan.id) fake_report = UrlListReport.objects.all().first() fake_report_calculation = retrieve_report(fake_report.id, "UrlListReport") - assert (len(fake_report_calculation['urls']) == len(urls)) + assert len(fake_report_calculation["urls"]) == len(urls) # the first url should still be by apple: - assert fake_report_calculation['urls'][0]['url'] == "apple.com" + assert fake_report_calculation["urls"][0]["url"] == "apple.com" # assert fake_report_calculation['urls'][1]['url'] == "apple.com" diff --git a/dashboard/internet_nl_dashboard/tests/test_scan.py b/dashboard/internet_nl_dashboard/tests/test_scan.py index 2ad1da78..f076db79 100644 --- a/dashboard/internet_nl_dashboard/tests/test_scan.py +++ b/dashboard/internet_nl_dashboard/tests/test_scan.py @@ -10,7 +10,7 @@ def test_update_state(db): account = Account() account.save() - urllist = UrlList(**{'name': '', 'account': account}) + urllist = UrlList(**{"name": "", "account": account}) urllist.save() scan = AccountInternetNLScan() diff --git a/dashboard/internet_nl_dashboard/tests/test_scan_internet_nl_per_account.py b/dashboard/internet_nl_dashboard/tests/test_scan_internet_nl_per_account.py index b65aa7ad..dc59ac12 100644 --- a/dashboard/internet_nl_dashboard/tests/test_scan_internet_nl_per_account.py +++ b/dashboard/internet_nl_dashboard/tests/test_scan_internet_nl_per_account.py @@ -8,8 +8,11 @@ from websecmap.scanners.models import Endpoint, EndpointGenericScan, InternetNLV2Scan from dashboard.internet_nl_dashboard.models import Account, AccountInternetNLScan, UrlList, UrlListReport -from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import (creating_report, monitor_timeout, - processing_scan_results) +from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import ( + creating_report, + monitor_timeout, + processing_scan_results, +) log = logging.getLogger(__package__) @@ -17,14 +20,14 @@ def test_monitor_timeout(db): a = Account.objects.create() - recoverable_states = ['discovering endpoints', 'retrieving scannable urls', 'registering scan at internet.nl'] + recoverable_states = ["discovering endpoints", "retrieving scannable urls", "registering scan at internet.nl"] for rec_state in recoverable_states: scan = AccountInternetNLScan.objects.create( account=a, scan=InternetNLV2Scan.objects.create(), urllist=UrlList.objects.create(account=a), state=rec_state, - state_changed_on=datetime(1996, 1, 1) + state_changed_on=datetime(1996, 1, 1), ) monitor_timeout(scan.id) updated_scan = AccountInternetNLScan.objects.get(id=scan.id) @@ -44,95 +47,118 @@ def test_creating_report(db, redis_server, default_policy, default_scan_metadata """ # preparing the data to use in this test: - user, c = User.objects.all().get_or_create(first_name='testuser', last_name='test', username='test', is_active=True) - account, c = Account.objects.all().get_or_create(name='testaccount') - urllist, c = UrlList.objects.all().get_or_create(name='testlist', account=account) - internetnlv2scan, c = InternetNLV2Scan.objects.all().get_or_create(type='web', scan_id=1, - state='scan results stored') + user, c = User.objects.all().get_or_create(first_name="testuser", last_name="test", username="test", is_active=True) + account, c = Account.objects.all().get_or_create(name="testaccount") + urllist, c = UrlList.objects.all().get_or_create(name="testlist", account=account) + internetnlv2scan, c = InternetNLV2Scan.objects.all().get_or_create( + type="web", scan_id=1, state="scan results stored" + ) # fcwaalwijk is missing in the results, which is intended # note: also set discovered on and created on to a date in the past, otherwise they will not be in the report # as they are not seen as relevant for the moment the report is made. - for domain in ['vitesse.nl', 'dierenscheboys.nl', 'fcwaalwijk.nl']: + for domain in ["vitesse.nl", "dierenscheboys.nl", "fcwaalwijk.nl"]: mydomain, c = Url.objects.all().get_or_create( - url=domain, is_dead=False, not_resolvable=False, created_on=datetime(2010, 1, 1)) + url=domain, is_dead=False, not_resolvable=False, created_on=datetime(2010, 1, 1) + ) myendpoint, c = Endpoint.objects.all().get_or_create( - url=mydomain, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False, discovered_on=datetime(2010, 1, 1) + url=mydomain, protocol="dns_a_aaaa", port=0, ip_version=0, is_dead=False, discovered_on=datetime(2010, 1, 1) ) urllist.urls.add(mydomain) urllist.save() internetnlv2scan.subject_urls.add() accountinternetnlscan, c = AccountInternetNLScan.objects.all().get_or_create( - account=account, scan=internetnlv2scan, urllist=urllist, started_on=datetime.now(timezone.utc), - report=None, finished_on=None, state='scan results ready') + account=account, + scan=internetnlv2scan, + urllist=urllist, + started_on=datetime.now(timezone.utc), + report=None, + finished_on=None, + state="scan results ready", + ) # Add a separate endpoint to verify that error scores do not hinder creating reports or statistics: # 'internet_nl_mail_dashboard_overall_score', 'internet_nl_web_overall_score' - vitesse_endpoint = Endpoint.objects.all().filter(url__url='vitesse.nl').first() + vitesse_endpoint = Endpoint.objects.all().filter(url__url="vitesse.nl").first() EndpointGenericScan.objects.all().get_or_create( - type='internet_nl_web_overall_score', rating="error", evidence="", endpoint=vitesse_endpoint, - rating_determined_on=datetime(2010, 1, 1) + type="internet_nl_web_overall_score", + rating="error", + evidence="", + endpoint=vitesse_endpoint, + rating_determined_on=datetime(2010, 1, 1), ) EndpointGenericScan.objects.all().get_or_create( - type='web_https_tls_version', rating="failed", evidence="", endpoint=vitesse_endpoint, - rating_determined_on=datetime(2010, 1, 1) + type="web_https_tls_version", + rating="failed", + evidence="", + endpoint=vitesse_endpoint, + rating_determined_on=datetime(2010, 1, 1), ) # here, vitesse.nl has an error from the API. This occurs when a crash happens in the api scanner, which is # rare, but can happen due to all kinds of weird setups it can encounter. internetnlv2scan.retrieved_scan_report = { - 'vitesse.nl': {'status': 'error'}, - 'dierenscheboys.nl': { - 'status': 'ok', - 'report': {'url': 'https://batch.internet.nl/site/dierenscheboys.nl/219843/'}, - 'scoring': {'percentage': 76}, 'results': { - 'categories': {'web_ipv6': {'verdict': 'failed', 'status': 'failed'}, - 'web_dnssec': {'verdict': 'passed', 'status': 'passed'}, - 'web_https': {'verdict': 'failed', 'status': 'failed'}, - 'web_appsecpriv': {'verdict': 'warning', 'status': 'warning'}}, - 'tests': {'web_ipv6_ns_address': {'status': 'passed', 'verdict': 'good'}, - 'web_ipv6_ns_reach': {'status': 'passed', 'verdict': 'good'}, - 'web_ipv6_ws_address': {'status': 'failed', 'verdict': 'bad'}, - 'web_ipv6_ws_reach': {'status': 'not_tested', 'verdict': 'not-tested'}, - 'web_ipv6_ws_similar': {'status': 'not_tested', 'verdict': 'not-tested'}, - 'web_dnssec_exist': {'status': 'passed', 'verdict': 'good'}, - 'web_dnssec_valid': {'status': 'passed', 'verdict': 'good'}, - 'web_https_http_available': {'status': 'passed', 'verdict': 'good'}, - 'web_https_http_redirect': {'status': 'passed', 'verdict': 'good'}, - 'web_https_http_hsts': {'status': 'failed', 'verdict': 'bad'}, - 'web_https_http_compress': {'status': 'passed', 'verdict': 'good'}, - 'web_https_tls_keyexchange': {'status': 'failed', 'verdict': 'bad'}, - 'web_https_tls_ciphers': {'status': 'passed', 'verdict': 'good'}, - 'web_https_tls_cipherorder': {'status': 'warning', 'verdict': 'warning'}, - 'web_https_tls_version': {'status': 'warning', 'verdict': 'phase-out'}, - 'web_https_tls_compress': {'status': 'passed', 'verdict': 'good'}, - 'web_https_tls_secreneg': {'status': 'passed', 'verdict': 'good'}, - 'web_https_tls_clientreneg': {'status': 'passed', 'verdict': 'good'}, - 'web_https_cert_chain': {'status': 'passed', 'verdict': 'good'}, - 'web_https_cert_pubkey': {'status': 'passed', 'verdict': 'good'}, - 'web_https_cert_sig': {'status': 'passed', 'verdict': 'good'}, - 'web_https_cert_domain': {'status': 'passed', 'verdict': 'good'}, - 'web_https_dane_exist': {'status': 'info', 'verdict': 'bad'}, - 'web_https_dane_valid': {'status': 'not_tested', 'verdict': 'not-tested'}, - 'web_https_tls_0rtt': {'status': 'passed', 'verdict': 'na'}, - 'web_https_tls_ocsp': {'status': 'info', 'verdict': 'ok'}, - 'web_https_tls_keyexchangehash': {'status': 'passed', 'verdict': 'good'}, - 'web_appsecpriv_x_frame_options': {'status': 'warning', 'verdict': 'bad'}, - 'web_appsecpriv_referrer_policy': {'status': 'warning', 'verdict': 'bad'}, - 'web_appsecpriv_csp': {'status': 'info', 'verdict': 'bad'}, - 'web_appsecpriv_x_content_type_options': {'status': 'warning', 'verdict': 'bad'}}, - 'custom': {'tls_1_3_support': 'no'}, 'calculated_results': { - 'web_legacy_dnssec': {'status': 'passed', 'verdict': 'passed', 'technical_details': []}, - 'web_legacy_tls_available': {'status': 'passed', 'verdict': 'passed', 'technical_details': []}, - 'web_legacy_tls_ncsc_web': {'status': 'failed', 'verdict': 'failed', 'technical_details': []}, - 'web_legacy_https_enforced': {'status': 'passed', 'verdict': 'passed', 'technical_details': []}, - 'web_legacy_hsts': {'status': 'failed', 'verdict': 'failed', 'technical_details': []}, - 'web_legacy_ipv6_nameserver': {'status': 'passed', 'verdict': 'passed', 'technical_details': []}, - 'web_legacy_ipv6_webserver': {'status': 'failed', 'verdict': 'failed', 'technical_details': []}, - 'web_legacy_dane': {'status': 'info', 'verdict': 'info', 'technical_details': []}, - 'web_legacy_tls_1_3': {'status': 'failed', 'verdict': 'failed', 'technical_details': []}, - 'web_legacy_category_ipv6': {'status': 'failed', 'verdict': 'failed', 'technical_details': []}}}} + "vitesse.nl": {"status": "error"}, + "dierenscheboys.nl": { + "status": "ok", + "report": {"url": "https://batch.internet.nl/site/dierenscheboys.nl/219843/"}, + "scoring": {"percentage": 76}, + "results": { + "categories": { + "web_ipv6": {"verdict": "failed", "status": "failed"}, + "web_dnssec": {"verdict": "passed", "status": "passed"}, + "web_https": {"verdict": "failed", "status": "failed"}, + "web_appsecpriv": {"verdict": "warning", "status": "warning"}, + }, + "tests": { + "web_ipv6_ns_address": {"status": "passed", "verdict": "good"}, + "web_ipv6_ns_reach": {"status": "passed", "verdict": "good"}, + "web_ipv6_ws_address": {"status": "failed", "verdict": "bad"}, + "web_ipv6_ws_reach": {"status": "not_tested", "verdict": "not-tested"}, + "web_ipv6_ws_similar": {"status": "not_tested", "verdict": "not-tested"}, + "web_dnssec_exist": {"status": "passed", "verdict": "good"}, + "web_dnssec_valid": {"status": "passed", "verdict": "good"}, + "web_https_http_available": {"status": "passed", "verdict": "good"}, + "web_https_http_redirect": {"status": "passed", "verdict": "good"}, + "web_https_http_hsts": {"status": "failed", "verdict": "bad"}, + "web_https_http_compress": {"status": "passed", "verdict": "good"}, + "web_https_tls_keyexchange": {"status": "failed", "verdict": "bad"}, + "web_https_tls_ciphers": {"status": "passed", "verdict": "good"}, + "web_https_tls_cipherorder": {"status": "warning", "verdict": "warning"}, + "web_https_tls_version": {"status": "warning", "verdict": "phase-out"}, + "web_https_tls_compress": {"status": "passed", "verdict": "good"}, + "web_https_tls_secreneg": {"status": "passed", "verdict": "good"}, + "web_https_tls_clientreneg": {"status": "passed", "verdict": "good"}, + "web_https_cert_chain": {"status": "passed", "verdict": "good"}, + "web_https_cert_pubkey": {"status": "passed", "verdict": "good"}, + "web_https_cert_sig": {"status": "passed", "verdict": "good"}, + "web_https_cert_domain": {"status": "passed", "verdict": "good"}, + "web_https_dane_exist": {"status": "info", "verdict": "bad"}, + "web_https_dane_valid": {"status": "not_tested", "verdict": "not-tested"}, + "web_https_tls_0rtt": {"status": "passed", "verdict": "na"}, + "web_https_tls_ocsp": {"status": "info", "verdict": "ok"}, + "web_https_tls_keyexchangehash": {"status": "passed", "verdict": "good"}, + "web_appsecpriv_x_frame_options": {"status": "warning", "verdict": "bad"}, + "web_appsecpriv_referrer_policy": {"status": "warning", "verdict": "bad"}, + "web_appsecpriv_csp": {"status": "info", "verdict": "bad"}, + "web_appsecpriv_x_content_type_options": {"status": "warning", "verdict": "bad"}, + }, + "custom": {"tls_1_3_support": "no"}, + "calculated_results": { + "web_legacy_dnssec": {"status": "passed", "verdict": "passed", "technical_details": []}, + "web_legacy_tls_available": {"status": "passed", "verdict": "passed", "technical_details": []}, + "web_legacy_tls_ncsc_web": {"status": "failed", "verdict": "failed", "technical_details": []}, + "web_legacy_https_enforced": {"status": "passed", "verdict": "passed", "technical_details": []}, + "web_legacy_hsts": {"status": "failed", "verdict": "failed", "technical_details": []}, + "web_legacy_ipv6_nameserver": {"status": "passed", "verdict": "passed", "technical_details": []}, + "web_legacy_ipv6_webserver": {"status": "failed", "verdict": "failed", "technical_details": []}, + "web_legacy_dane": {"status": "info", "verdict": "info", "technical_details": []}, + "web_legacy_tls_1_3": {"status": "failed", "verdict": "failed", "technical_details": []}, + "web_legacy_category_ipv6": {"status": "failed", "verdict": "failed", "technical_details": []}, + }, + }, + }, } internetnlv2scan.save() @@ -150,14 +176,14 @@ def test_creating_report(db, redis_server, default_policy, default_scan_metadata # todo: fix this test, it has never worked on the build because redis was not available. Now that redis is # available it returns 4 instead of the 3 we expect (and see locally during test). # assert UrlReport.objects.all().count() == 3 - first_urlreport = UrlReport.objects.all().filter(url__url='dierenscheboys.nl').first() + first_urlreport = UrlReport.objects.all().filter(url__url="dierenscheboys.nl").first() # this fluctuates if settings such as forum_standardisation metrics are enabled/disabled. # since the default changed on 20240618 this amount is lowered from 12 to 6. assert first_urlreport.high == 6 assert first_urlreport.medium == 6 assert first_urlreport.low == 3 assert first_urlreport.ok == 18 - assert UrlReport.objects.all().filter(url__url='vitesse.nl').count() == 2 + assert UrlReport.objects.all().filter(url__url="vitesse.nl").count() == 2 # To see what the calculation is exactly: assert first_urlreport.calculation == {} # vitesse was added, has a very old one and a new report with the error update. diff --git a/dashboard/internet_nl_dashboard/tests/test_shared_report_lists.py b/dashboard/internet_nl_dashboard/tests/test_shared_report_lists.py index 26e31533..db340549 100644 --- a/dashboard/internet_nl_dashboard/tests/test_shared_report_lists.py +++ b/dashboard/internet_nl_dashboard/tests/test_shared_report_lists.py @@ -13,8 +13,8 @@ def test_get_publicly_shared_lists_per_account(db): # todo: perhaps mandate that a list-id is given, and perhaps that the list id has to be a guid. # no content no crash - response = c.get('/data/report/public/account/1/lists/all/') - assert response.content == b'[]' + response = c.get("/data/report/public/account/1/lists/all/") + assert response.content == b"[]" # create a list with some reports account = Account.objects.create(name="share_account") @@ -25,48 +25,48 @@ def test_get_publicly_shared_lists_per_account(db): expected_response = [ { - 'account': {'public_name': ''}, - 'list': { - 'id': urllist.id, - 'name': urllist.name, - 'scan_type': 'web', - 'automatically_share_new_reports': False, - 'automated_scan_frequency': 'disabled' + "account": {"public_name": ""}, + "list": { + "id": urllist.id, + "name": urllist.name, + "scan_type": "web", + "automatically_share_new_reports": False, + "automated_scan_frequency": "disabled", }, - 'number_of_reports': 1, - 'reports': [ + "number_of_reports": 1, + "reports": [ { - 'id': urllistreport.id, - 'at_when': '2020-01-01T00:00:00Z', - 'report_type': "web", + "id": urllistreport.id, + "at_when": "2020-01-01T00:00:00Z", + "report_type": "web", "has_public_share_code": False, "average_internet_nl_score": 0.0, - "public_report_code": '', - 'total_urls': 0, - 'urllist__name': urllist.name + "public_report_code": "", + "total_urls": 0, + "urllist__name": urllist.name, } - ] + ], } ] - response = c.get(f'/data/report/public/account/{account.id}/lists/all/') + response = c.get(f"/data/report/public/account/{account.id}/lists/all/") # remove randomness: assert response.status_code == 200 json_data = response.json() - json_data[0]['reports'][0]['public_report_code'] = "" + json_data[0]["reports"][0]["public_report_code"] = "" assert json_data == expected_response # non existing list: - response = c.get(f'/data/report/public/account/{account.id}/lists/781263187/') - assert response.content == b'[]' + response = c.get(f"/data/report/public/account/{account.id}/lists/781263187/") + assert response.content == b"[]" # non existing account: - response = c.get(f'/data/report/public/account/781263187/lists/{urllist.id}/') - assert response.content == b'[]' + response = c.get(f"/data/report/public/account/781263187/lists/{urllist.id}/") + assert response.content == b"[]" - response = c.get(f'/data/report/public/account/{account.id}/lists/{urllist.id}/') + response = c.get(f"/data/report/public/account/{account.id}/lists/{urllist.id}/") json_data = response.json() - json_data[0]['reports'][0]['public_report_code'] = "" + json_data[0]["reports"][0]["public_report_code"] = "" assert json_data == expected_response assert response.status_code == 200 diff --git a/dashboard/internet_nl_dashboard/tests/test_spreadsheet.py b/dashboard/internet_nl_dashboard/tests/test_spreadsheet.py index daec1d50..246cfba8 100644 --- a/dashboard/internet_nl_dashboard/tests/test_spreadsheet.py +++ b/dashboard/internet_nl_dashboard/tests/test_spreadsheet.py @@ -4,9 +4,15 @@ from django.contrib.auth.models import User -from dashboard.internet_nl_dashboard.logic.spreadsheet import (get_data, get_upload_history, is_file, - is_valid_extension, is_valid_mimetype, - log_spreadsheet_upload, save_data) +from dashboard.internet_nl_dashboard.logic.spreadsheet import ( + get_data, + get_upload_history, + is_file, + is_valid_extension, + is_valid_mimetype, + log_spreadsheet_upload, + save_data, +) from dashboard.internet_nl_dashboard.models import Account, DashboardUser, TaggedUrlInUrllist # the 5000 urls has been skipped, it adds nothing to the test cases, only to the load for the UI. Use it for UI @@ -41,37 +47,39 @@ def test_valid(file): assert len(data) == 2 # testsite contains these three - assert data['testsites'] == {'aaenmaas.nl': {'tags': ['test']}, - 'hdsr.nl': {'tags': ['test', ' waterschap', ' extra']}, - 'zuiderzeeland.nl': {'tags': ['extra']}} + assert data["testsites"] == { + "aaenmaas.nl": {"tags": ["test"]}, + "hdsr.nl": {"tags": ["test", " waterschap", " extra"]}, + "zuiderzeeland.nl": {"tags": ["extra"]}, + } # waterschappen should contain 24 items (including two compound items(!)) - assert len(data['waterschappen']) == 24 + assert len(data["waterschappen"]) == 24 saved = save_data(account, data) # the test is run multiple times with data, so after the first time, already_in_list is set. - assert saved['testsites']['added_to_list'] == 3 or saved['testsites']['already_in_list'] == 3 + assert saved["testsites"]["added_to_list"] == 3 or saved["testsites"]["already_in_list"] == 3 # Tags have also been added assert TaggedUrlInUrllist.objects.all().count() == 27 - data = log_spreadsheet_upload(account.dashboarduser_set.first(), file=file, status='test', message="Test") - assert len(data['original_filename']) > 5 + data = log_spreadsheet_upload(account.dashboarduser_set.first(), file=file, status="test", message="Test") + assert len(data["original_filename"]) > 5 upload_history = get_upload_history(account) assert len(upload_history) > 0 # Try to work with a malformed CSV file - file = f'{path}/test spreadsheet uploads/incorrect.csv' + file = f"{path}/test spreadsheet uploads/incorrect.csv" data = get_data(file) assert len(data) == 0 # Run all tests on a valid file, that complies with the standard etc... - file = f'{path}/test spreadsheet uploads/waterschappen.ods' + file = f"{path}/test spreadsheet uploads/waterschappen.ods" test_valid(file) # should also work for excel files - file = f'{path}/test spreadsheet uploads/waterschappen.xlsx' + file = f"{path}/test spreadsheet uploads/waterschappen.xlsx" test_valid(file) # We skip this test as it takes long and doesn't really add anything. We could improve it's speed perhaps. @@ -80,11 +88,11 @@ def test_valid(file): # test_valid(file) # Should also be able to handle CSV files, as if they are spreadsheets - file = f'{path}/test spreadsheet uploads/waterschappen.csv' + file = f"{path}/test spreadsheet uploads/waterschappen.csv" test_valid(file) # Now let's see what happens if another octet/stream is uploaded, like an .exe file. - file = f'{path}/test spreadsheet uploads/tracie/tracie.exe' + file = f"{path}/test spreadsheet uploads/tracie/tracie.exe" # this will fail because of the file extension valid = is_valid_extension(file) @@ -92,12 +100,12 @@ def test_valid(file): # Let's be evil and rename it to .xlsx, so we bypass both the mime check and the extension check # Nope, it finds it's a 'application/x-dosexec' - file = f'{path}/test spreadsheet uploads/tracie/tracie.xlsx' + file = f"{path}/test spreadsheet uploads/tracie/tracie.xlsx" valid = is_valid_mimetype(file) assert valid is False # Let's instead use a corrupted xlsx that should not work when parsing. - file = f'{path}/test spreadsheet uploads/waterschappen_corrupted.xlsx' + file = f"{path}/test spreadsheet uploads/waterschappen_corrupted.xlsx" valid = is_valid_mimetype(file) assert valid is True @@ -108,13 +116,13 @@ def test_valid(file): # Mixing datatypes (ints, booleans etc) should just work # one of the datatypes is split into two, therefore there are 9 items. # The point is to check that importing doesn't crash and some casting to strings work. - file = f'{path}/test spreadsheet uploads/mixed_datatypes.ods' + file = f"{path}/test spreadsheet uploads/mixed_datatypes.ods" data = get_data(file) assert len(data) == 9 # the original filename can be retrieved (using a heuristic, not exact) # the file was already uploaded above, so it should now be renamed internally. - file = f'{path}/test spreadsheet uploads/waterschappen_WFgL3uS.ods' - data = log_spreadsheet_upload(dashboarduser, file=file, status='Test', message="Test") - assert data['original_filename'] == "waterschappen.ods" - assert data['internal_filename'] != "waterschappen.ods" + file = f"{path}/test spreadsheet uploads/waterschappen_WFgL3uS.ods" + data = log_spreadsheet_upload(dashboarduser, file=file, status="Test", message="Test") + assert data["original_filename"] == "waterschappen.ods" + assert data["internal_filename"] != "waterschappen.ods" diff --git a/dashboard/internet_nl_dashboard/tests/test_translation.py b/dashboard/internet_nl_dashboard/tests/test_translation.py index eb84234e..321c5a9a 100644 --- a/dashboard/internet_nl_dashboard/tests/test_translation.py +++ b/dashboard/internet_nl_dashboard/tests/test_translation.py @@ -7,9 +7,13 @@ from requests import Response -from dashboard.internet_nl_dashboard.logic.internet_nl_translations import (convert_vue_i18n_format, get_locale_content, - get_po_as_dictionary_v2, load_as_po_file, - translate_field) +from dashboard.internet_nl_dashboard.logic.internet_nl_translations import ( + convert_vue_i18n_format, + get_locale_content, + get_po_as_dictionary_v2, + load_as_po_file, + translate_field, +) path = Path(__file__).parent @@ -21,7 +25,7 @@ def file_get_contents(filepath): - with open(filepath, 'r') as content_file: + with open(filepath, "r") as content_file: return content_file.read() @@ -31,22 +35,22 @@ def __init__(self, content, status_code): self.content = content self.status_code = status_code - if args[0] == 'https://raw.githubusercontent.com/internetstandards/Internet.nl/master/translations/nl/main.po': - return MockResponse(file_get_contents(f'{path}/translation files/main.po').encode(), 200) + if args[0] == "https://raw.githubusercontent.com/internetstandards/Internet.nl/master/translations/nl/main.po": + return MockResponse(file_get_contents(f"{path}/translation files/main.po").encode(), 200) return MockResponse(None, 404) -@mock.patch('requests.get', side_effect=mocked_requests_get) +@mock.patch("requests.get", side_effect=mocked_requests_get) def test_convert_vue_i18n_format(db, tmpdir) -> None: - content = get_locale_content('nl') + content = get_locale_content("nl") assert len(content) > 1000 list_po_content = load_as_po_file(content) assert len(list_po_content) > 10 - formatted = convert_vue_i18n_format('nl', list_po_content) + formatted = convert_vue_i18n_format("nl", list_po_content) assert len(formatted) > 1000 # create the expected directory structure @@ -63,16 +67,16 @@ def test_convert_vue_i18n_format(db, tmpdir) -> None: def test_field_translation(): # Load dutch translation for a field - translation_dictionary = get_po_as_dictionary_v2('nl') - translated = translate_field('internet_nl_web_https_tls_version', translation_dictionary=translation_dictionary) + translation_dictionary = get_po_as_dictionary_v2("nl") + translated = translate_field("internet_nl_web_https_tls_version", translation_dictionary=translation_dictionary) assert translated == "TLS-versie" # Now verify that we can live switch to an english translation - translation_dictionary = get_po_as_dictionary_v2('en') - translated = translate_field('internet_nl_web_https_tls_version', translation_dictionary=translation_dictionary) + translation_dictionary = get_po_as_dictionary_v2("en") + translated = translate_field("internet_nl_web_https_tls_version", translation_dictionary=translation_dictionary) assert translated == "TLS version" # And switch back to dutch again - translation_dictionary = get_po_as_dictionary_v2('nl') - translated = translate_field('internet_nl_web_https_tls_version', translation_dictionary=translation_dictionary) + translation_dictionary = get_po_as_dictionary_v2("nl") + translated = translate_field("internet_nl_web_https_tls_version", translation_dictionary=translation_dictionary) assert translated == "TLS-versie" diff --git a/dashboard/internet_nl_dashboard/tests/test_two_factor_integration.py b/dashboard/internet_nl_dashboard/tests/test_two_factor_integration.py index 3cc02ade..24bcec75 100644 --- a/dashboard/internet_nl_dashboard/tests/test_two_factor_integration.py +++ b/dashboard/internet_nl_dashboard/tests/test_two_factor_integration.py @@ -10,6 +10,6 @@ def test_two_factor_integration(db) -> None: - """ The first page you visit when logged out should contain something that is used only in the theme. """ - response = client.get('/', follow=True) + """The first page you visit when logged out should contain something that is used only in the theme.""" + response = client.get("/", follow=True) assert b"Internet Standards Platform" in response.content diff --git a/dashboard/internet_nl_dashboard/tests/test_user.py b/dashboard/internet_nl_dashboard/tests/test_user.py index 9876d5f0..f2779a9c 100644 --- a/dashboard/internet_nl_dashboard/tests/test_user.py +++ b/dashboard/internet_nl_dashboard/tests/test_user.py @@ -22,37 +22,37 @@ def test_user_editing(db): dashboarduser.user = user dashboarduser.account = account dashboarduser.mail_send_mail_after_scan_finished = True - dashboarduser.mail_preferred_language = 'en' - dashboarduser.mail_preferred_mail_address = 'info@example.com' + dashboarduser.mail_preferred_language = "en" + dashboarduser.mail_preferred_mail_address = "info@example.com" dashboarduser.save() data = get_user_settings(1) assert data == { - 'first_name': 'test', - 'last_name': 'test', - 'date_joined': None, - 'last_login': None, - 'account_id': 1, - 'account_name': "test account", - 'mail_preferred_mail_address': "info@example.com", - 'mail_preferred_language': Country(code='en'), - 'mail_send_mail_after_scan_finished': True + "first_name": "test", + "last_name": "test", + "date_joined": None, + "last_login": None, + "account_id": 1, + "account_name": "test account", + "mail_preferred_mail_address": "info@example.com", + "mail_preferred_language": Country(code="en"), + "mail_send_mail_after_scan_finished": True, } # make a correct change on all fields that we can change. new_data = { - 'first_name': 'example', - 'last_name': 'example', - 'mail_preferred_mail_address': 'example@example.com', - 'mail_preferred_language': 'nl', - 'mail_send_mail_after_scan_finished': False + "first_name": "example", + "last_name": "example", + "mail_preferred_mail_address": "example@example.com", + "mail_preferred_language": "nl", + "mail_send_mail_after_scan_finished": False, } response = save_user_settings(1, new_data) - del (response['timestamp']) + del response["timestamp"] expected_response = operation_response(success=True, message="save_user_settings_success") - del (expected_response['timestamp']) + del expected_response["timestamp"] assert response == expected_response @@ -60,7 +60,7 @@ def test_user_editing(db): assert user.first_name == "example" assert user.last_name == "example" assert user.dashboarduser.mail_preferred_mail_address == "example@example.com" - assert user.dashboarduser.mail_preferred_language == Country(code='nl') + assert user.dashboarduser.mail_preferred_language == Country(code="nl") assert user.dashboarduser.mail_send_mail_after_scan_finished is False # todo: make all error situations happen. diff --git a/dashboard/internet_nl_dashboard/urls.py b/dashboard/internet_nl_dashboard/urls.py index b9273b46..761540a4 100644 --- a/dashboard/internet_nl_dashboard/urls.py +++ b/dashboard/internet_nl_dashboard/urls.py @@ -6,14 +6,29 @@ # We have to import the signals somewhere..?! import dashboard.internet_nl_dashboard.signals # noqa # pylint: disable=unused-import -from dashboard.internet_nl_dashboard.views import (account, app, domains, download_spreadsheet, logout_view, mail, - powertools, report, scan_monitor, session, signup, spreadsheet, - subdomains, tags, usage, user) +from dashboard.internet_nl_dashboard.views import ( + account, + app, + domains, + download_spreadsheet, + logout_view, + mail, + powertools, + report, + scan_monitor, + session, + signup, + spreadsheet, + subdomains, + tags, + usage, + user, +) class SpreadsheetFileTypeConverter: # Supports {"key": "value", "key2": "value2"} syntax. - regex = '(xlsx|ods|csv)' + regex = "(xlsx|ods|csv)" @staticmethod def to_python(value): @@ -21,115 +36,105 @@ def to_python(value): @staticmethod def to_url(value): - return f'{value}' + return f"{value}" -register_converter(SpreadsheetFileTypeConverter, 'spreadsheet_filetype') +register_converter(SpreadsheetFileTypeConverter, "spreadsheet_filetype") urlpatterns = [ - path('', lambda request: redirect(config.DASHBOARD_FRONTEND_URL)), - path('data/config/', app.config), - + path("", lambda request: redirect(config.DASHBOARD_FRONTEND_URL)), + path("data/config/", app.config), # The SPA is not reachable anymore. # path('spa/', powertools.spa), - path('data/powertools/get_accounts/', powertools.get_accounts), - path('data/powertools/set_account/', powertools.set_account), - path('data/powertools/save_instant_account/', powertools.save_instant_account), - path('logout/', logout_view), - + path("data/powertools/get_accounts/", powertools.get_accounts), + path("data/powertools/set_account/", powertools.set_account), + path("data/powertools/save_instant_account/", powertools.save_instant_account), + path("logout/", logout_view), # domain management - path('data/urllists/get/', domains.get_lists), - path('data/urllist_content/get//', domains.get_urllist_content_), - path('data/urllist/save_list_content/', domains.save_list_content), - path('data/urllist/update_list_settings/', domains.update_list_settings_), - path('data/urllist/get_scan_status_of_list//', domains.get_scan_status_of_list_), - path('data/urllist/create_list/', domains.create_list_), - path('data/urllist/delete/', domains.delete_list_), - path('data/urllist/scan_now/', domains.scan_now_), - path('data/urllist/discover-subdomains//', subdomains.request_subdomain_discovery_scan_), - path('data/urllist/discover-subdomains-status//', subdomains.subdomain_discovery_scan_status_), - path('data/urllist/url/save/', domains.alter_url_in_urllist_), - path('data/urllist/url/add/', domains.add_urls_to_urllist), - path('data/urllist/url/delete/', domains.delete_url_from_urllist_), - path('data/urllist/download/', domains.download_list_), - path('data/urllist/upload//', spreadsheet.upload_list_), - path('data/urllist/suggest-subdomains/', domains.suggest_subdomains_), - - path('data/urllist/tag/add/', tags.add_tag_), - path('data/urllist/tag/remove/', tags.remove_tag_), - path('data/urllist/tag/list//', tags.tags_in_urllist_), - + path("data/urllists/get/", domains.get_lists), + path("data/urllist_content/get//", domains.get_urllist_content_), + path("data/urllist/save_list_content/", domains.save_list_content), + path("data/urllist/update_list_settings/", domains.update_list_settings_), + path("data/urllist/get_scan_status_of_list//", domains.get_scan_status_of_list_), + path("data/urllist/create_list/", domains.create_list_), + path("data/urllist/delete/", domains.delete_list_), + path("data/urllist/scan_now/", domains.scan_now_), + path("data/urllist/discover-subdomains//", subdomains.request_subdomain_discovery_scan_), + path("data/urllist/discover-subdomains-status//", subdomains.subdomain_discovery_scan_status_), + path("data/urllist/url/save/", domains.alter_url_in_urllist_), + path("data/urllist/url/add/", domains.add_urls_to_urllist), + path("data/urllist/url/delete/", domains.delete_url_from_urllist_), + path("data/urllist/download/", domains.download_list_), + path("data/urllist/upload//", spreadsheet.upload_list_), + path("data/urllist/suggest-subdomains/", domains.suggest_subdomains_), + path("data/urllist/tag/add/", tags.add_tag_), + path("data/urllist/tag/remove/", tags.remove_tag_), + path("data/urllist/tag/list//", tags.tags_in_urllist_), # account management: - path('data/account/report_settings/get/', account.get_report_settings_), - path('data/account/report_settings/save/', account.save_report_settings_), - - path('data/user/get/', user.get_user_settings_), - path('data/user/save/', user.save_user_settings_), - + path("data/account/report_settings/get/", account.get_report_settings_), + path("data/account/report_settings/save/", account.save_report_settings_), + path("data/user/get/", user.get_user_settings_), + path("data/user/save/", user.save_user_settings_), # uploads of domains - path('upload/', spreadsheet.upload), - path('data/upload-spreadsheet/', spreadsheet.upload_spreadsheet), - path('data/upload-history/', spreadsheet.upload_history), - + path("upload/", spreadsheet.upload), + path("data/upload-spreadsheet/", spreadsheet.upload_spreadsheet), + path("data/upload-history/", spreadsheet.upload_history), # scans / scan monitor - path('data/scan-monitor/', scan_monitor.running_scans), - path('data/scan/cancel/', domains.cancel_scan_), - - path('data/report/get//', report.get_report_), - path('data/report/shared//', report.get_shared_report_), - path('data/report/public/', report.get_public_reports_), - - path('data/report/ad_hoc//', report.get_ad_hoc_tagged_report_), - path('data/report/ad_hoc_save//', report.save_ad_hoc_tagged_report_), - path('data/report/get_improvements_and_regressions//', - report.improvement_regressions_compared_to_previous_report_), - - path('data/report/share/share/', report.x_share), - path('data/report/share/unshare/', report.x_unshare), - path('data/report/share/update_share_code/', report.x_update_share_code), - path('data/report/share/update_report_code/', report.x_update_report_code), - - path('data/report/public/account//lists/all/', report.get_publicly_shared_lists_per_account_), - path('data/report/public/account//lists//', - report.get_publicly_shared_lists_per_account_and_list_id_), - path('data/report/public/lists//latest/', - report.get_latest_report_id_from_list), - path('data/report/public/lists//latest//', - report.get_latest_report_id_from_list_and_type_), - - - - path('data/report/differences_compared_to_current_list//', - report.get_report_differences_compared_to_current_list_), - path('data/report/get_previous///', report.get_previous_report_), - - path('data/report/recent/', report.get_recent_reports_), - path('data/report/urllist_timeline_graph///', - report.get_urllist_report_graph_data_), - path('data/download-spreadsheet///', - download_spreadsheet.download_spreadsheet), - - path('data/usage/', usage.usage_), - - path('mail/unsubscribe///', mail.unsubscribe_), - - path('data/signup/', signup.process_application), - + path("data/scan-monitor/", scan_monitor.running_scans), + path("data/scan/cancel/", domains.cancel_scan_), + path("data/report/get//", report.get_report_), + path("data/report/shared//", report.get_shared_report_), + path("data/report/public/", report.get_public_reports_), + path("data/report/ad_hoc//", report.get_ad_hoc_tagged_report_), + path("data/report/ad_hoc_save//", report.save_ad_hoc_tagged_report_), + path( + "data/report/get_improvements_and_regressions//", + report.improvement_regressions_compared_to_previous_report_, + ), + path("data/report/share/share/", report.x_share), + path("data/report/share/unshare/", report.x_unshare), + path("data/report/share/update_share_code/", report.x_update_share_code), + path("data/report/share/update_report_code/", report.x_update_report_code), + path("data/report/public/account//lists/all/", report.get_publicly_shared_lists_per_account_), + path( + "data/report/public/account//lists//", + report.get_publicly_shared_lists_per_account_and_list_id_, + ), + path("data/report/public/lists//latest/", report.get_latest_report_id_from_list), + path( + "data/report/public/lists//latest//", + report.get_latest_report_id_from_list_and_type_, + ), + path( + "data/report/differences_compared_to_current_list//", + report.get_report_differences_compared_to_current_list_, + ), + path("data/report/get_previous///", report.get_previous_report_), + path("data/report/recent/", report.get_recent_reports_), + path( + "data/report/urllist_timeline_graph///", report.get_urllist_report_graph_data_ + ), + path( + "data/download-spreadsheet///", + download_spreadsheet.download_spreadsheet, + ), + path("data/usage/", usage.usage_), + path("mail/unsubscribe///", mail.unsubscribe_), + path("data/signup/", signup.process_application), # session management # logging in via javascript is not possible, because the CSRF is tied to the session cookie. # The session cookie cannot be requested by javascript, and we're not going to use JWT because # the second factor part is also django only, and not implmented as REST methods. # So there is currently no way to move to rest based auth _including_ second factor authentication. # of course except OAUTH, but there is no knowledge for that yet. - path('session/login/', session.session_login), - path('session/logout/', session.session_logout), - path('session/status/', session.session_status), + path("session/login/", session.session_login), + path("session/logout/", session.session_logout), + path("session/status/", session.session_status), # Would you enable the below login form, you will bypass all second factor authentication. Therefore do not enable # this url (!) # url(r'^login/$', auth_views.LoginView.as_view(template_name='internet_nl_dashboard/registration/login.html'), # name='login'), - path("security.txt", security_txt, name="security_txt"), path(".well-known/security.txt", security_txt, name="well_known_security_txt"), ] diff --git a/dashboard/internet_nl_dashboard/views/__init__.py b/dashboard/internet_nl_dashboard/views/__init__.py index 8226756f..375ab244 100644 --- a/dashboard/internet_nl_dashboard/views/__init__.py +++ b/dashboard/internet_nl_dashboard/views/__init__.py @@ -9,7 +9,7 @@ from dashboard.internet_nl_dashboard.models import Account, DashboardUser log = logging.getLogger(__package__) -LOGIN_URL = '/account/login/' +LOGIN_URL = "/account/login/" """ Todo: csrf via API calls... @@ -20,7 +20,7 @@ def logout_view(request) -> HttpResponse: logout(request) - return redirect('/') + return redirect("/") def get_account(request) -> Account: @@ -40,7 +40,7 @@ def empty_response() -> JsonResponse: def error_response(message: str) -> JsonResponse: - return JsonResponse({'status': 'error', 'message': message}) + return JsonResponse({"status": "error", "message": message}) def get_json_body(request): diff --git a/dashboard/internet_nl_dashboard/views/app.py b/dashboard/internet_nl_dashboard/views/app.py index 6d9afeda..f7774992 100644 --- a/dashboard/internet_nl_dashboard/views/app.py +++ b/dashboard/internet_nl_dashboard/views/app.py @@ -24,7 +24,7 @@ def config_content(): }, "layout": configuration["SITE_LAYOUT_NAME"], "supported_languages": configuration["SUPPORTED_LANGUAGES"], - } + }, } diff --git a/dashboard/internet_nl_dashboard/views/domains.py b/dashboard/internet_nl_dashboard/views/domains.py index c4c810a0..ec5a77fd 100644 --- a/dashboard/internet_nl_dashboard/views/domains.py +++ b/dashboard/internet_nl_dashboard/views/domains.py @@ -6,12 +6,22 @@ from django.views.decorators.http import require_http_methods from websecmap.app.common import JSEncoder -from dashboard.internet_nl_dashboard.logic.domains import (add_domains_from_raw_user_data, alter_url_in_urllist, - cancel_scan, create_list, delete_list, - delete_url_from_urllist, download_as_spreadsheet, - get_scan_status_of_list, get_urllist_content, - get_urllists_from_account, save_urllist_content_by_name, - scan_now, suggest_subdomains, update_list_settings) +from dashboard.internet_nl_dashboard.logic.domains import ( + add_domains_from_raw_user_data, + alter_url_in_urllist, + cancel_scan, + create_list, + delete_list, + delete_url_from_urllist, + download_as_spreadsheet, + get_scan_status_of_list, + get_urllist_content, + get_urllists_from_account, + save_urllist_content_by_name, + scan_now, + update_list_settings, +) +from dashboard.internet_nl_dashboard.logic.suggestions import suggest_subdomains from dashboard.internet_nl_dashboard.views import LOGIN_URL, get_account, get_json_body @@ -79,8 +89,8 @@ def add_urls_to_urllist(request): def delete_url_from_urllist_(request): account = get_account(request) request = get_json_body(request) - item_deleted = delete_url_from_urllist(account, request.get('urllist_id', None), request.get('url_id', None)) - return JsonResponse({'items_deleted': None, 'success': item_deleted}) + item_deleted = delete_url_from_urllist(account, request.get("urllist_id", None), request.get("url_id", None)) + return JsonResponse({"items_deleted": None, "success": item_deleted}) @login_required(login_url=LOGIN_URL) @@ -92,7 +102,7 @@ def scan_now_(request): def cancel_scan_(request): account = get_account(request) request = get_json_body(request) - response = cancel_scan(account, request.get('id')) + response = cancel_scan(account, request.get("id")) return JsonResponse(response) @@ -100,4 +110,4 @@ def cancel_scan_(request): @require_http_methods(["POST"]) def download_list_(request): params = get_json_body(request) - return download_as_spreadsheet(get_account(request), params.get('list-id', None), params.get('file-type', None)) + return download_as_spreadsheet(get_account(request), params.get("list-id", None), params.get("file-type", None)) diff --git a/dashboard/internet_nl_dashboard/views/download_spreadsheet.py b/dashboard/internet_nl_dashboard/views/download_spreadsheet.py index 927a29a5..e6961498 100644 --- a/dashboard/internet_nl_dashboard/views/download_spreadsheet.py +++ b/dashboard/internet_nl_dashboard/views/download_spreadsheet.py @@ -44,7 +44,7 @@ def create_spreadsheet_download(file_name: str, spreadsheet_data: Any, file_type } if file_type == "xlsx-openpyxl": - with open(spreadsheet_data.name, 'rb') as file_handle: + with open(spreadsheet_data.name, "rb") as file_handle: output: HttpResponse = HttpResponse(file_handle.read()) file_type = "xlsx" else: diff --git a/dashboard/internet_nl_dashboard/views/powertools.py b/dashboard/internet_nl_dashboard/views/powertools.py index 932c436c..6acf66c4 100644 --- a/dashboard/internet_nl_dashboard/views/powertools.py +++ b/dashboard/internet_nl_dashboard/views/powertools.py @@ -35,7 +35,7 @@ def is_powertool_user(user): @user_passes_test(is_powertool_user, login_url=LOGIN_URL) def set_account(request) -> HttpResponse: request_data = get_json_body(request) - selected_account_id: int = request_data['id'] + selected_account_id: int = request_data["id"] if not selected_account_id: return JsonResponse(operation_response(error=True, message="No account supplied.")) @@ -44,16 +44,13 @@ def set_account(request) -> HttpResponse: # very new users don't have the dashboarduser fields filled in, and are thus not connected to an account. if not dashboard_user: - dashboard_user = DashboardUser(**{'account': Account.objects.all().first(), 'user': request.user}) + dashboard_user = DashboardUser(**{"account": Account.objects.all().first(), "user": request.user}) dashboard_user.account = Account.objects.get(id=selected_account_id) dashboard_user.save() - return JsonResponse(operation_response( - success=True, - message="switched_account", - data={'account_name': dashboard_user.account.name} - ) + return JsonResponse( + operation_response(success=True, message="switched_account", data={"account_name": dashboard_user.account.name}) ) @@ -62,75 +59,73 @@ def get_accounts(request) -> HttpResponse: myaccount = get_account(request) scans = AccountInternetNLScan.objects.all().filter(account=myaccount.id).count() lists = UrlList.objects.all().filter(account=myaccount.id).count() - users = list(User.objects.all().filter(dashboarduser__account=myaccount.id).values_list('username', flat=True)) + users = list(User.objects.all().filter(dashboarduser__account=myaccount.id).values_list("username", flat=True)) current_account = { - 'id': myaccount.id, - 'name': myaccount.name, - 'scans': scans, - 'lists': lists, - 'users': users, - 'label': f"{myaccount.id}: {myaccount.name} (Lists: {lists}, Scans: {scans}, Users: {len(users)})" + "id": myaccount.id, + "name": myaccount.name, + "scans": scans, + "lists": lists, + "users": users, + "label": f"{myaccount.id}: {myaccount.name} (Lists: {lists}, Scans: {scans}, Users: {len(users)})", } - accounts = Account.objects.all().values_list('id', 'name').order_by('id') + accounts = Account.objects.all().values_list("id", "name").order_by("id") account_data = [] # add some metadata to the accounts, so it's more clear where you are switching to: for account in accounts: scans = AccountInternetNLScan.objects.all().filter(account=account[0]).count() lists = UrlList.objects.all().filter(account=account[0]).count() - users = list(User.objects.all().filter(dashboarduser__account=account[0]).values_list('username', flat=True)) + users = list(User.objects.all().filter(dashboarduser__account=account[0]).values_list("username", flat=True)) account_information = { - 'id': account[0], - 'name': account[1], - 'scans': scans, - 'lists': lists, - 'users': users, - 'label': f"{account[0]}: {account[1]} (Lists: {lists}, Scans: {scans}, Users: {len(users)})" + "id": account[0], + "name": account[1], + "scans": scans, + "lists": lists, + "users": users, + "label": f"{account[0]}: {account[1]} (Lists: {lists}, Scans: {scans}, Users: {len(users)})", } account_data.append(account_information) - return JsonResponse( - { - 'current_account': current_account, - 'accounts': account_data - } - ) + return JsonResponse({"current_account": current_account, "accounts": account_data}) @user_passes_test(is_powertool_user, login_url=LOGIN_URL) def save_instant_account(request) -> HttpResponse: request = get_json_body(request) - username = request['username'] - password = request['password'] + username = request["username"] + password = request["password"] if User.objects.all().filter(username=username).exists(): return JsonResponse(operation_response(error=True, message=f"User with username '{username}' already exists.")) if Account.objects.all().filter(name=username).exists(): - return JsonResponse(operation_response(error=True, - message=f"Account with username {username}' already exists.")) + return JsonResponse( + operation_response(error=True, message=f"Account with username {username}' already exists.") + ) # Extremely arbitrary password requirements. Just to make sure a password has been filled in. if len(password) < 5: return JsonResponse(operation_response(error=True, message="Password not filled in or not long enough.")) # all seems fine, let's add the user - user = User(**{'username': username}) + user = User(**{"username": username}) user.set_password(password) user.is_active = True user.save() - account = Account(**{ - 'name': username, - 'internet_nl_api_username': username, - 'internet_nl_api_password': Account.encrypt_password(password), - 'can_connect_to_internet_nl_api': Account.connect_to_internet_nl_api(username, password) - }) + account = Account( + **{ + "name": username, + "internet_nl_api_username": username, + "internet_nl_api_password": Account.encrypt_password(password), + "can_connect_to_internet_nl_api": Account.connect_to_internet_nl_api(username, password), + } + ) account.save() - dashboarduser = DashboardUser(**{'user': user, 'account': account}) + dashboarduser = DashboardUser(**{"user": user, "account": account}) dashboarduser.save() return JsonResponse(operation_response(success=True, message=f"Account and user with name '{username}' created!")) diff --git a/dashboard/internet_nl_dashboard/views/report.py b/dashboard/internet_nl_dashboard/views/report.py index a8b97da5..f1d7d9f7 100644 --- a/dashboard/internet_nl_dashboard/views/report.py +++ b/dashboard/internet_nl_dashboard/views/report.py @@ -8,15 +8,26 @@ from websecmap.app.common import JSEncoder from dashboard.internet_nl_dashboard.logic.mail import values_from_previous_report -from dashboard.internet_nl_dashboard.logic.report import (ad_hoc_tagged_report, get_previous_report, get_public_reports, - get_recent_reports, get_report, - get_report_differences_compared_to_current_list, - get_shared_report, get_urllist_timeline_graph, - save_ad_hoc_tagged_report, share, unshare, update_report_code, - update_share_code) +from dashboard.internet_nl_dashboard.logic.report import ( + ad_hoc_tagged_report, + get_previous_report, + get_public_reports, + get_recent_reports, + get_report, + get_report_differences_compared_to_current_list, + get_shared_report, + get_urllist_timeline_graph, + save_ad_hoc_tagged_report, + share, + unshare, + update_report_code, + update_share_code, +) from dashboard.internet_nl_dashboard.logic.shared_report_lists import ( - get_latest_report_id_from_list_and_type, get_publicly_shared_lists_per_account, - get_publicly_shared_lists_per_account_and_list_id) + get_latest_report_id_from_list_and_type, + get_publicly_shared_lists_per_account, + get_publicly_shared_lists_per_account_and_list_id, +) from dashboard.internet_nl_dashboard.models import UrlListReport from dashboard.internet_nl_dashboard.views import LOGIN_URL, get_account, get_json_body @@ -25,8 +36,7 @@ def get_report_(request, report_id) -> HttpResponse: # Explicitly NOT use jsonresponse as this loads the json data into an encoder which is extremely slow on large files return HttpResponse( # pylint: disable=http-response-with-content-type-json - get_report(get_account(request), report_id), - content_type="application/json" + get_report(get_account(request), report_id), content_type="application/json" ) # return JsonResponse(get_report(get_account(request), report_id), encoder=JSEncoder, safe=False) @@ -34,8 +44,9 @@ def get_report_(request, report_id) -> HttpResponse: @login_required(login_url=LOGIN_URL) def get_report_differences_compared_to_current_list_(request, report_id): - return JsonResponse(get_report_differences_compared_to_current_list(get_account(request), report_id), - encoder=JSEncoder, safe=True) + return JsonResponse( + get_report_differences_compared_to_current_list(get_account(request), report_id), encoder=JSEncoder, safe=True + ) @login_required(login_url=LOGIN_URL) @@ -46,7 +57,7 @@ def get_previous_report_(request, urllist_id, at_when): @login_required(login_url=LOGIN_URL) def get_ad_hoc_tagged_report_(request, report_id: int): data = get_json_body(request) - tags = data.get('tags', []) + tags = data.get("tags", []) try: at_when: Optional[datetime] = datetime.fromisoformat(f"{data.get('custom_date')} {data.get('custom_time')}") @@ -54,15 +65,14 @@ def get_ad_hoc_tagged_report_(request, report_id: int): at_when = None return HttpResponse( # pylint: disable=http-response-with-content-type-json - ad_hoc_tagged_report(get_account(request), report_id, tags, at_when), - content_type="application/json" + ad_hoc_tagged_report(get_account(request), report_id, tags, at_when), content_type="application/json" ) @login_required(login_url=LOGIN_URL) def save_ad_hoc_tagged_report_(request, report_id: int): data = get_json_body(request) - tags = data.get('tags', []) + tags = data.get("tags", []) try: at_when: Optional[datetime] = datetime.fromisoformat(f"{data.get('custom_date')} {data.get('custom_time')}") except ValueError: @@ -77,8 +87,9 @@ def get_recent_reports_(request) -> JsonResponse: @login_required(login_url=LOGIN_URL) def get_urllist_report_graph_data_(request, urllist_ids, report_type: str = "web") -> JsonResponse: - return JsonResponse(get_urllist_timeline_graph(get_account(request), urllist_ids, report_type), - encoder=JSEncoder, safe=False) + return JsonResponse( + get_urllist_timeline_graph(get_account(request), urllist_ids, report_type), encoder=JSEncoder, safe=False + ) # No login required: reports via this method are public @@ -89,8 +100,7 @@ def get_shared_report_(request, report_code: str) -> HttpResponse: data = get_json_body(request) return HttpResponse( # pylint: disable=http-response-with-content-type-json - get_shared_report(report_code, data.get('share_code', '')), - content_type="application/json" + get_shared_report(report_code, data.get("share_code", "")), content_type="application/json" ) @@ -103,29 +113,30 @@ def get_public_reports_(request) -> JsonResponse: def x_share(request): data = get_json_body(request) account = get_account(request) - return JsonResponse(share(account, data.get('report_id', -1), data.get('public_share_code', '')), safe=False) + return JsonResponse(share(account, data.get("report_id", -1), data.get("public_share_code", "")), safe=False) @login_required(login_url=LOGIN_URL) def x_unshare(request): data = get_json_body(request) account = get_account(request) - return JsonResponse(unshare(account, data.get('report_id', -1)), safe=False) + return JsonResponse(unshare(account, data.get("report_id", -1)), safe=False) @login_required(login_url=LOGIN_URL) def x_update_share_code(request): data = get_json_body(request) account = get_account(request) - return JsonResponse(update_share_code(account, data.get('report_id', -1), data.get('public_share_code', '')), - safe=False) + return JsonResponse( + update_share_code(account, data.get("report_id", -1), data.get("public_share_code", "")), safe=False + ) @login_required(login_url=LOGIN_URL) def x_update_report_code(request): data = get_json_body(request) account = get_account(request) - return JsonResponse(update_report_code(account, data.get('report_id', -1)), safe=False) + return JsonResponse(update_report_code(account, data.get("report_id", -1)), safe=False) def get_publicly_shared_lists_per_account_(request, account_id) -> JsonResponse: @@ -137,7 +148,7 @@ def get_publicly_shared_lists_per_account_and_list_id_(request, account_id, urll def get_latest_report_id_from_list(request, urllist_id) -> JsonResponse: - return JsonResponse(get_latest_report_id_from_list_and_type(urllist_id, ''), safe=False) + return JsonResponse(get_latest_report_id_from_list_and_type(urllist_id, ""), safe=False) def get_latest_report_id_from_list_and_type_(request, urllist_id, report_type) -> JsonResponse: diff --git a/dashboard/internet_nl_dashboard/views/session.py b/dashboard/internet_nl_dashboard/views/session.py index 2e77ac34..d5392f3a 100644 --- a/dashboard/internet_nl_dashboard/views/session.py +++ b/dashboard/internet_nl_dashboard/views/session.py @@ -30,15 +30,15 @@ def session_login_(request): :return: """ # taken from: https://stackoverflow.com/questions/11891322/setting-up-a-user-login-in-python-django-using-json-and- - if request.method != 'POST': + if request.method != "POST": sleep(2) return operation_response(error=True, message="post_only") # get the json data: parameters = get_json_body(request) - username = parameters.get('username', '').strip() - password = parameters.get('password', '').strip() + username = parameters.get("username", "").strip() + password = parameters.get("password", "").strip() if not username or not password: sleep(2) @@ -69,7 +69,7 @@ def session_logout_(request): # The preferred method of detecting anonymous users is to see if they are authenticated, according to: # https://docs.djangoproject.com/en/3.1/ref/contrib/auth/ if not request.user.is_authenticated: - log.debug('User is not authenticated...') + log.debug("User is not authenticated...") return operation_response(success=True, message="logged_out") logout(request) @@ -78,24 +78,24 @@ def session_logout_(request): def session_status_(request): """ - Returns a dictionary of permissions the user has. We keep it simple and only distinct - :param request: - :return: + Returns a dictionary of permissions the user has. We keep it simple and only distinct + :param request: + :return: """ if not request.user.is_authenticated: return { - 'is_authenticated': False, - 'is_superuser': False, - 'second_factor_enabled': False, - 'account_name': '', + "is_authenticated": False, + "is_superuser": False, + "second_factor_enabled": False, + "account_name": "", } return { - 'is_authenticated': request.user.is_authenticated, - 'is_superuser': request.user.is_superuser, - 'account_name': request.user.dashboarduser.account.name, - 'account_id': request.user.dashboarduser.account.id, + "is_authenticated": request.user.is_authenticated, + "is_superuser": request.user.is_superuser, + "account_name": request.user.dashboarduser.account.name, + "account_id": request.user.dashboarduser.account.id, } diff --git a/dashboard/internet_nl_dashboard/views/signup.py b/dashboard/internet_nl_dashboard/views/signup.py index 0f656e32..8d7ea0ff 100644 --- a/dashboard/internet_nl_dashboard/views/signup.py +++ b/dashboard/internet_nl_dashboard/views/signup.py @@ -22,9 +22,19 @@ def process_application(request): data = get_json_body(request) - required_fields = ["access", "name", "email", "mobile_phone_number", "organization_name", "nature_of_organization", - "chamber_of_commerce_number", "reason_for_application", "intended_usage_frequency", - "terms_of_use", "captcha"] + required_fields = [ + "access", + "name", + "email", + "mobile_phone_number", + "organization_name", + "nature_of_organization", + "chamber_of_commerce_number", + "reason_for_application", + "intended_usage_frequency", + "terms_of_use", + "captcha", + ] form_data = data.get("form_data", {}) @@ -73,7 +83,7 @@ def send_backoffice_mail_async(form_data): recipients=email_addresses, subject=email_subject, # how to set the reply to: https://github.com/Bearle/django_mail_admin/blob/master/docs/usage.rst#L148 - headers={'Reply-to': form_data.get('email', config.EMAIL_NOTIFICATION_SENDER)}, + headers={"Reply-to": form_data.get("email", config.EMAIL_NOTIFICATION_SENDER)}, message=email_content.replace("
", ""), priority=models.PRIORITY.now, html_message=email_content, @@ -85,7 +95,7 @@ def send_signup_received_mail_to_requester(form_data): # perform some validation that this looks like a valid mail address # https://stackoverflow.com/questions/8022530/how-to-check-for-valid-email-address - if not re.fullmatch(r"[^@]+@[^@]+\.[^@]+", form_data['email']): + if not re.fullmatch(r"[^@]+@[^@]+\.[^@]+", form_data["email"]): return # also set some sort of rate limit in case someone smart wants to send 1000's of mails quickly. @@ -93,10 +103,7 @@ def send_signup_received_mail_to_requester(form_data): # the backoffice team will see a mail coming in every request, so don't add those addresses here... mail.send( sender=config.EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP, - recipients=form_data['email'], # List of email addresses also accepted - template=xget_template( - template_name="signup_thank_you", - preferred_language="nl" - ), - variable_dict=form_data + recipients=form_data["email"], # List of email addresses also accepted + template=xget_template(template_name="signup_thank_you", preferred_language="nl"), + variable_dict=form_data, ) diff --git a/dashboard/internet_nl_dashboard/views/spreadsheet.py b/dashboard/internet_nl_dashboard/views/spreadsheet.py index 9c8a38ee..880e5bf4 100644 --- a/dashboard/internet_nl_dashboard/views/spreadsheet.py +++ b/dashboard/internet_nl_dashboard/views/spreadsheet.py @@ -9,9 +9,13 @@ from django.views.decorators.http import require_http_methods from websecmap.app.common import JSEncoder -from dashboard.internet_nl_dashboard.logic.spreadsheet import (get_upload_history, import_step_2, - log_spreadsheet_upload, save_file, - upload_domain_spreadsheet_to_list) +from dashboard.internet_nl_dashboard.logic.spreadsheet import ( + get_upload_history, + import_step_2, + log_spreadsheet_upload, + save_file, + upload_domain_spreadsheet_to_list, +) from dashboard.internet_nl_dashboard.views import LOGIN_URL, get_account, get_dashboarduser log = logging.getLogger(__package__) @@ -20,11 +24,15 @@ @login_required(login_url=LOGIN_URL) def upload(request) -> HttpResponse: - response: HttpResponse = render(request, 'internet_nl_dashboard/templates/internet_nl_dashboard/upload.html', { - 'menu_item_addressmanager': "current", - 'max_lists': int(config.DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET), - 'max_urls': int(config.DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET) - }) + response: HttpResponse = render( + request, + "internet_nl_dashboard/templates/internet_nl_dashboard/upload.html", + { + "menu_item_addressmanager": "current", + "max_lists": int(config.DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET), + "max_urls": int(config.DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET), + }, + ) return response @@ -42,18 +50,18 @@ def upload_spreadsheet(request) -> HttpResponse: user = get_dashboarduser(request) # happens when no file is sent - if 'file' not in request.FILES: + if "file" not in request.FILES: return response # a request of 25k domains will take 12 seconds, which is already too long for interaction. # so all steps are now parallelized. - if request.method == 'POST' and request.FILES['file']: + if request.method == "POST" and request.FILES["file"]: log.debug("Saving file") - file = save_file(request.FILES['file']) + file = save_file(request.FILES["file"]) upload_data = log_spreadsheet_upload( - user=user, file=file, status='[1/3] Initializing', message="[1/3] Initializing upload..." + user=user, file=file, status="[1/3] Initializing", message="[1/3] Initializing upload..." ) - uploadlog_id = upload_data['id'] + uploadlog_id = upload_data["id"] group(import_step_2.si(user.id, file, uploadlog_id)).apply_async() @@ -75,9 +83,8 @@ def upload_history(request) -> JsonResponse: @require_http_methods(["POST"]) def upload_list_(request, list_id): # params = get_json_body(request) - return JsonResponse(upload_domain_spreadsheet_to_list( - get_account(request), - get_dashboarduser(request), - list_id, - request.FILES.get('file', None) - )) + return JsonResponse( + upload_domain_spreadsheet_to_list( + get_account(request), get_dashboarduser(request), list_id, request.FILES.get("file", None) + ) + ) diff --git a/dashboard/internet_nl_dashboard/views/tags.py b/dashboard/internet_nl_dashboard/views/tags.py index 1fc4b24c..269f6cc1 100644 --- a/dashboard/internet_nl_dashboard/views/tags.py +++ b/dashboard/internet_nl_dashboard/views/tags.py @@ -13,9 +13,9 @@ def add_tag_(request): data = get_json_body(request) add_tag( account=get_account(request), - urllist_id=data.get('urllist_id', []), - url_ids=data.get('url_ids', []), - tag=data.get('tag', "") + urllist_id=data.get("urllist_id", []), + url_ids=data.get("url_ids", []), + tag=data.get("tag", ""), ) return JsonResponse(operation_response(success=True), safe=False) @@ -25,9 +25,9 @@ def remove_tag_(request): data = get_json_body(request) remove_tag( account=get_account(request), - urllist_id=data.get('urllist_id', []), - url_ids=data.get('url_ids', []), - tag=data.get('tag', "") + urllist_id=data.get("urllist_id", []), + url_ids=data.get("url_ids", []), + tag=data.get("tag", ""), ) return JsonResponse(operation_response(success=True), safe=False) diff --git a/dashboard/lockfile.py b/dashboard/lockfile.py index af69e3f5..2c9d5ff7 100644 --- a/dashboard/lockfile.py +++ b/dashboard/lockfile.py @@ -8,8 +8,8 @@ def renew_lock(process_name: str) -> bool: lockfile = f"{settings.LOCKFILE_DIR}{process_name}.lock" - with open(lockfile, 'wt', encoding="UTF-8") as handle: - return handle.write('locked') > 1 + with open(lockfile, "wt", encoding="UTF-8") as handle: + return handle.write("locked") > 1 def remove_lock(process_name: str) -> None: diff --git a/dashboard/security.py b/dashboard/security.py index bc258bd4..ca44b0d1 100755 --- a/dashboard/security.py +++ b/dashboard/security.py @@ -3,11 +3,12 @@ def confirm_keys_are_changed(): - if not settings.DEBUG and settings.SECRET_KEY == '_dzlo^9d#ox6!7c9rju@=u8+4^sprqocy3s*l*ejc2yr34@&98': # nosec - raise ValueError('SECRET_KEY contains a debugging value. Set a sane secret key.') + if not settings.DEBUG and settings.SECRET_KEY == "_dzlo^9d#ox6!7c9rju@=u8+4^sprqocy3s*l*ejc2yr34@&98": # nosec + raise ValueError("SECRET_KEY contains a debugging value. Set a sane secret key.") - if not settings.DEBUG and settings.FIELD_ENCRYPTION_KEY == b'JjvHNnFMfEaGd7Y0SAHBRNZYGGpNs7ydEp-ixmKSvkQ=': # nosec + if not settings.DEBUG and settings.FIELD_ENCRYPTION_KEY == b"JjvHNnFMfEaGd7Y0SAHBRNZYGGpNs7ydEp-ixmKSvkQ=": # nosec raise ValueError( - 'FIELD_ENCRYPTION_KEY has to be configured on the OS level, and needs to be different than the ' - 'default key provided. Please create a new key. Instructions are listed here:' - 'https://github.com/pyca/cryptography. In short, run: key = Fernet.generate_key()') + "FIELD_ENCRYPTION_KEY has to be configured on the OS level, and needs to be different than the " + "default key provided. Please create a new key. Instructions are listed here:" + "https://github.com/pyca/cryptography. In short, run: key = Fernet.generate_key()" + ) diff --git a/dashboard/settings.py b/dashboard/settings.py index 15ddae35..dfb47218 100644 --- a/dashboard/settings.py +++ b/dashboard/settings.py @@ -9,10 +9,12 @@ from sentry_sdk.integrations.redis import RedisIntegration from . import __version__ + from .settings_constance import (CONSTANCE_ADDITIONAL_FIELDS, CONSTANCE_BACKEND, CONSTANCE_CONFIG, CONSTANCE_CONFIG_FIELDSETS) # noqa # pylint: disable=unused-import from .settings_jet import JET_SIDE_MENU_COMPACT, JET_SIDE_MENU_ITEMS # noqa # pylint: disable=unused-import + # import all of this and don't auto-lint it away because there are no references here. # most code refers to these settings. @@ -43,98 +45,84 @@ SECRET_KEY = get_secret_key_from_file_or_env() # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = os.environ.get('DEBUG', False) +DEBUG = os.environ.get("DEBUG", False) if DEBUG: - print('Django debugging is enabled.') + print("Django debugging is enabled.") -ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', 'localhost,127.0.0.1,::1').split(',') +ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "localhost,127.0.0.1,::1").split(",") # Application definition INSTALLED_APPS = [ # Constance - 'constance', + "constance", # not needed since 3.0 # 'constance.backends.database', - # Jet - 'jet.dashboard', - 'jet', - 'nested_admin', - + "jet.dashboard", + "jet", + "nested_admin", "taggit", - # Import Export - 'import_export', - + "import_export", # Standard Django - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.humanize', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.humanize", # Allow a client to access the data: - 'corsheaders', - + "corsheaders", # Periodic tasks - 'django_celery_beat', - + "django_celery_beat", # Web Security Map (todo: minimize the subset) # The reason (model) why it's included is in the comments. - 'websecmap.app', # Job - 'websecmap.api', - 'websecmap.organizations', # Url - 'websecmap.scanners', # Endpoint, EndpointGenericScan, UrlGenericScan - 'websecmap.reporting', # Various reporting functions (might be not needed) - 'websecmap.map', # because some scanners are intertwined with map configurations. That needs to go. - 'websecmap.game', - + "websecmap.app", # Job + "websecmap.api", + "websecmap.organizations", # Url + "websecmap.scanners", # Endpoint, EndpointGenericScan, UrlGenericScan + "websecmap.reporting", # Various reporting functions (might be not needed) + "websecmap.map", # because some scanners are intertwined with map configurations. That needs to go. + "websecmap.game", # Custom Apps # These apps overwrite whatever is declared above, for example the user information. # Yet, it does not overwrite management commands. - 'dashboard.internet_nl_dashboard', - + "dashboard.internet_nl_dashboard", # Two factor auth - 'phonenumber_field', - 'django_otp', - 'django_otp.plugins.otp_static', - 'django_otp.plugins.otp_totp', - 'two_factor', + "phonenumber_field", + "django_otp", + "django_otp.plugins.otp_static", + "django_otp.plugins.otp_totp", + "two_factor", # Django activity stream # https://django-activity-stream.readthedocs.io/en/latest/installation.html - 'django.contrib.sites', - 'actstream', - + "django.contrib.sites", + "actstream", # Sending templated and translatable emails - 'django_mail_admin', - + "django_mail_admin", # allauth - 'allauth', - 'allauth.account', - 'allauth.socialaccount', - + "allauth", + "allauth.account", + "allauth.socialaccount", ] # django activity stream wants a site-id: SITE_ID = 1 MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.locale.LocaleMiddleware', - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.locale.LocaleMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", # Two factor Auth - 'django_otp.middleware.OTPMiddleware', - + "django_otp.middleware.OTPMiddleware", # https://docs.djangoproject.com/en/4.2/ref/middleware/#django.middleware.gzip.GZipMiddleware # This middleware should be placed before any other middleware that need to read or write the response body so # that compression happens afterward. @@ -144,93 +132,95 @@ "allauth.account.middleware.AccountMiddleware", ] -ROOT_URLCONF = 'dashboard.urls' +ROOT_URLCONF = "dashboard.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [f'{BASE_DIR}/'], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'constance.context_processors.config', - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'dashboard.internet_nl_dashboard.context_processors.template_settings_processor', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [f"{BASE_DIR}/"], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "constance.context_processors.config", + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + "dashboard.internet_nl_dashboard.context_processors.template_settings_processor", ], }, } ] -WSGI_APPLICATION = 'dashboard.wsgi.application' +WSGI_APPLICATION = "dashboard.wsgi.application" # Database # https://docs.djangoproject.com/en/2.1/ref/settings/#databases DATABASE_OPTIONS = { - 'mysql': {'init_command': "SET character_set_connection=utf8," - "collation_connection=utf8_unicode_ci," - "sql_mode='STRICT_ALL_TABLES';"}, + "mysql": { + "init_command": "SET character_set_connection=utf8," + "collation_connection=utf8_unicode_ci," + "sql_mode='STRICT_ALL_TABLES';" + }, } -DB_ENGINE = os.environ.get('DB_ENGINE', 'mysql') +DB_ENGINE = os.environ.get("DB_ENGINE", "mysql") DATABASE_ENGINES = { - 'mysql': 'dashboard.app.backends.mysql', + "mysql": "dashboard.app.backends.mysql", } DATABASES_SETTINGS = { # persist local database used during development (runserver) - 'dev': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.environ.get('DB_NAME', 'db.sqlite3'), + "dev": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.environ.get("DB_NAME", "db.sqlite3"), }, # sqlite memory database for running tests without - 'test': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.environ.get('DB_NAME', 'db.sqlite3'), + "test": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.environ.get("DB_NAME", "db.sqlite3"), }, # for production get database settings from environment (eg: docker) - 'production': { - 'ENGINE': DATABASE_ENGINES.get(DB_ENGINE, f'django.db.backends.{DB_ENGINE}'), - 'NAME': os.environ.get('DB_NAME', 'dashboard'), - 'USER': os.environ.get('DB_USER', 'dashboard'), - 'PASSWORD': os.environ.get('DB_PASSWORD', 'dashboard'), - 'HOST': os.environ.get('DB_HOST', 'mysql'), - 'OPTIONS': DATABASE_OPTIONS.get(os.environ.get('DB_ENGINE', 'mysql'), {}), + "production": { + "ENGINE": DATABASE_ENGINES.get(DB_ENGINE, f"django.db.backends.{DB_ENGINE}"), + "NAME": os.environ.get("DB_NAME", "dashboard"), + "USER": os.environ.get("DB_USER", "dashboard"), + "PASSWORD": os.environ.get("DB_PASSWORD", "dashboard"), + "HOST": os.environ.get("DB_HOST", "mysql"), + "OPTIONS": DATABASE_OPTIONS.get(os.environ.get("DB_ENGINE", "mysql"), {}), }, } # allow database to be selected through environment variables -DATABASE = os.environ.get('DJANGO_DATABASE', 'dev') -DATABASES = {'default': DATABASES_SETTINGS[DATABASE]} +DATABASE = os.environ.get("DJANGO_DATABASE", "dev") +DATABASES = {"default": DATABASES_SETTINGS[DATABASE]} # Password validation # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", # fix 514 "OPTIONS": { "min_length": 16, }, }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] # Internationalization # https://docs.djangoproject.com/en/2.1/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -239,99 +229,95 @@ USE_TZ = True -LOCALE_PATHS = ['locale'] +LOCALE_PATHS = ["locale"] -LANGUAGE_COOKIE_NAME = 'dashboard_language' +LANGUAGE_COOKIE_NAME = "dashboard_language" # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.1/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = "/static/" # Absolute path to aggregate to and serve static file from. if DEBUG: - STATIC_ROOT = 'static' + STATIC_ROOT = "static" else: - STATIC_ROOT = os.environ.get('STATIC_ROOT', '/srv/dashboard/static/') # noqa + STATIC_ROOT = os.environ.get("STATIC_ROOT", "/srv/dashboard/static/") # noqa MEDIA_ROOT = os.environ.get("MEDIA_ROOT", f"{os.path.abspath(os.path.dirname(__file__))}/uploads/") -UPLOAD_ROOT: str = os.environ.get('MEDIA_ROOT', f'{os.path.abspath(os.path.dirname(__file__))}/uploads/') +UPLOAD_ROOT: str = os.environ.get("MEDIA_ROOT", f"{os.path.abspath(os.path.dirname(__file__))}/uploads/") # Two factor auth LOGIN_URL = "two_factor:login" LOGIN_REDIRECT_URL = "/" LOGOUT_REDIRECT_URL = LOGIN_URL -TWO_FACTOR_QR_FACTORY = 'qrcode.image.pil.PilImage' +TWO_FACTOR_QR_FACTORY = "qrcode.image.pil.PilImage" # 6 supports google authenticator TWO_FACTOR_TOTP_DIGITS = 6 TWO_FACTOR_PATCH_ADMIN = True # something from websecmap -TOOLS = {'organizations': {'import_data_dir': ''}} +TOOLS = {"organizations": {"import_data_dir": ""}} # Encrypted fields FIELD_ENCRYPTION_KEY: bytes = get_field_encryption_key_from_file_or_env() LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', # sys.stdout - 'formatter': 'color', + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": { + "class": "logging.StreamHandler", # sys.stdout + "formatter": "color", }, }, - 'formatters': { - 'debug': { - 'format': '%(asctime)s\t%(levelname)-8s - %(filename)-20s:%(lineno)-4s - ' - '%(funcName)20s() - %(message)s', + "formatters": { + "debug": { + "format": "%(asctime)s\t%(levelname)-8s - %(filename)-20s:%(lineno)-4s - " "%(funcName)20s() - %(message)s", }, - 'color': { - '()': 'colorlog.ColoredFormatter', - 'format': '%(log_color)s%(asctime)s\t%(levelname)-8s - ' - '%(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S', - 'log_colors': { - 'DEBUG': 'green', - 'INFO': 'white', - 'WARNING': 'yellow', - 'ERROR': 'red', - 'CRITICAL': 'bold_red', + "color": { + "()": "colorlog.ColoredFormatter", + "format": "%(log_color)s%(asctime)s\t%(levelname)-8s - " "%(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S", + "log_colors": { + "DEBUG": "green", + "INFO": "white", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "bold_red", }, - } + }, }, - 'loggers': { + "loggers": { # Used when there is no log defined or loaded. Disabled given we always use __package__ to log. # Would you enable it, all logging messages will be logged twice. # '': { # 'handlers': ['console'], # 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), # }, - # Default Django logging, we expect django to work, and therefore only show INFO messages. # It can be smart to sometimes want to see what's going on here, but not all the time. # https://docs.djangoproject.com/en/2.1/topics/logging/#django-s-logging-extensions - 'django': { - 'handlers': ['console'], - 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + "django": { + "handlers": ["console"], + "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"), }, - - 'celery.app.trace': { - 'handlers': ['console'], - 'level': os.getenv('CELERY_LOG_LEVEL', 'INFO'), + "celery.app.trace": { + "handlers": ["console"], + "level": os.getenv("CELERY_LOG_LEVEL", "INFO"), }, # logging defaults to INFO, DEBUG logging is enabled in development using .envrc - 'dashboard': { - 'handlers': ['console'], - 'level': os.getenv('APP_LOG_LEVEL', 'INFO'), + "dashboard": { + "handlers": ["console"], + "level": os.getenv("APP_LOG_LEVEL", "INFO"), }, - 'websecmap': { - 'handlers': ['console'], - 'level': os.getenv('APP_LOG_LEVEL', 'INFO'), + "websecmap": { + "handlers": ["console"], + "level": os.getenv("APP_LOG_LEVEL", "INFO"), }, - 'test': { - 'handlers': ['console'], - 'level': os.getenv('APP_LOG_LEVEL', 'INFO'), + "test": { + "handlers": ["console"], + "level": os.getenv("APP_LOG_LEVEL", "INFO"), }, }, } @@ -350,11 +336,11 @@ CELERY_EVENT_SERIALIZER = "json" # Celery config -CELERY_BROKER_URL = os.environ.get('BROKER', 'redis://localhost:6379/0') +CELERY_BROKER_URL = os.environ.get("BROKER", "redis://localhost:6379/0") CELERY_ENABLE_UTC = True -CELERY_TIMEZONE = 'UTC' +CELERY_TIMEZONE = "UTC" -CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" CELERY_BROKER_CONNECTION_RETRY = True CELERY_BROKER_CONNECTION_MAX_RETRIES = 400 @@ -402,19 +388,21 @@ # Settings for statsd metrics collection. Statsd defaults over UDP port 8125. # https://django-statsd.readthedocs.io/en/latest/#celery-signals-integration -STATSD_HOST = os.environ.get('STATSD_HOST', '127.0.0.1') +STATSD_HOST = os.environ.get("STATSD_HOST", "127.0.0.1") STATSD_PORT = os.environ.get("STATSD_PORT", "8125") -STATSD_PREFIX = 'dashboard' +STATSD_PREFIX = "dashboard" STATSD_TELEGRAF = True # register hooks for selery tasks STATSD_CELERY_SIGNALS = True # send database query metric (in production, in development we have debug toolbar for this) if not DEBUG: - STATSD_PATCHES = ['django_statsd.patches.db', ] + STATSD_PATCHES = [ + "django_statsd.patches.db", + ] -OUTPUT_DIR = os.environ.get('OUTPUT_DIR', f'{os.path.abspath(os.path.dirname(__file__))}/') -VENDOR_DIR = os.environ.get('VENDOR_DIR', f"{os.path.abspath(f'{os.path.dirname(__file__)}/../vendor/')}/") +OUTPUT_DIR = os.environ.get("OUTPUT_DIR", f"{os.path.abspath(os.path.dirname(__file__))}/") +VENDOR_DIR = os.environ.get("VENDOR_DIR", f"{os.path.abspath(f'{os.path.dirname(__file__)}/../vendor/')}/") if DEBUG: # too many sql variables.... @@ -424,42 +412,47 @@ # Security options if not DEBUG: SESSION_COOKIE_SECURE = True # insecure by default - SESSION_COOKIE_SAMESITE = 'Lax' + SESSION_COOKIE_SAMESITE = "Lax" SESSION_COOKIE_AGE = 1209600 # two weeks, could be longer CSRF_COOKIE_SECURE = True # insecure by default -if SENTRY_DSN := os.environ.get('SENTRY_DSN'): +if SENTRY_DSN := os.environ.get("SENTRY_DSN"): # new sentry_sdk implementation, with hopes to also get exceptions from workers. sentry_sdk.init( # pylint: disable=abstract-class-instantiated # (following the documentation) # type: ignore dsn=SENTRY_DSN, integrations=[CeleryIntegration(), DjangoIntegration(), RedisIntegration()], - release=__version__, send_default_pii=False) + release=__version__, + send_default_pii=False, + ) -SENTRY_ORGANIZATION = os.environ.get("SENTRY_ORGANIZATION", 'internet-cleanup-foundation') -SENTRY_PROJECT = os.environ.get("SENTRY_PROJECT", 'internet-nl-dashboard') -SENTRY_PROJECT_URL = f'https://sentry.io/{SENTRY_ORGANIZATION}/{SENTRY_PROJECT}' +SENTRY_ORGANIZATION = os.environ.get("SENTRY_ORGANIZATION", "internet-cleanup-foundation") +SENTRY_PROJECT = os.environ.get("SENTRY_PROJECT", "internet-nl-dashboard") +SENTRY_PROJECT_URL = f"https://sentry.io/{SENTRY_ORGANIZATION}/{SENTRY_PROJECT}" # Copied from internet.nl # Supported languages. # NOTE: Make sure that a DNS record for each language exists. # More information can be found in the README file. -LANGUAGES = sorted([ - ('nl', 'Dutch'), - ('en', 'English'), -], key=lambda x: x[0]) +LANGUAGES = sorted( + [ + ("nl", "Dutch"), + ("en", "English"), + ], + key=lambda x: x[0], +) # email settings... # django_mail_admin.backends.OutboxEmailBackend = Store sent mails in outbox, so we know what has been sent. # It's not a log -> this is just a way to test things, and it will hang send_queued_mail. EMAIL_USE_SSL = False EMAIL_USE_TLS = False -EMAIL_HOST_PASSWORD = '' # nosec -EMAIL_HOST_USER = '' -EMAIL_PORT = '' +EMAIL_HOST_PASSWORD = "" # nosec +EMAIL_HOST_USER = "" +EMAIL_PORT = "" # As there are sanity checks, these settings need to be present during debugging too. -EMAIL_HOST = '' -EMAIL_BACKEND = 'django_mail_admin.backends.CustomEmailBackend' +EMAIL_HOST = "" +EMAIL_BACKEND = "django_mail_admin.backends.CustomEmailBackend" if DEBUG: # 25 megs for importing reports from live situations DATA_UPLOAD_MAX_MEMORY_SIZE = 26214400 @@ -476,26 +469,26 @@ # See # https://github.com/adamchainz/django-cors-headers CORS_ALLOWED_ORIGINS = [ - os.environ.get("CORS_ALLOWED_ACCEPT_DOMAIN", 'https://acc.dashboard.internet.nl'), - os.environ.get("CORS_ALLOWED_DOMAIN", 'https://dashboard.internet.nl'), + os.environ.get("CORS_ALLOWED_ACCEPT_DOMAIN", "https://acc.dashboard.internet.nl"), + os.environ.get("CORS_ALLOWED_DOMAIN", "https://dashboard.internet.nl"), "http://localhost:8080", "http://127.0.0.1:8080", - "http://127.0.0.1:8081" + "http://127.0.0.1:8081", ] # as soon as this is set, the vue post stuff doesn't work anymore. # CSRF_HEADER_NAME = 'X-CSRFToken' CORS_ALLOW_HEADERS = list(default_headers) + [ - 'cache-control', - 'X-CSRFToken', - 'csrfmiddlewaretoken', + "cache-control", + "X-CSRFToken", + "csrfmiddlewaretoken", ] # allow cookies to be sent as well, we have to, because there are logins and such. CORS_ALLOW_CREDENTIALS = True -LOCKFILE_DIR = os.environ.get('LOCKFILE_DIR', f'{os.path.abspath(os.path.dirname(__file__))}/lockfiles/') +LOCKFILE_DIR = os.environ.get("LOCKFILE_DIR", f"{os.path.abspath(os.path.dirname(__file__))}/lockfiles/") TAGGIT_CASE_INSENSITIVE = True @@ -523,24 +516,23 @@ "http://::1", "http://localhost:8080", "http://localhost:8081", - os.environ.get("CSRF_TRUSTED_ORIGINS_DEFAULT_DOMAIN", 'https://internet.nl'), - os.environ.get("CSRF_TRUSTED_ORIGINS_WILDCARD_DOMAIN", 'https://*.internet.nl') + os.environ.get("CSRF_TRUSTED_ORIGINS_DEFAULT_DOMAIN", "https://internet.nl"), + os.environ.get("CSRF_TRUSTED_ORIGINS_WILDCARD_DOMAIN", "https://*.internet.nl"), ] REPORT_STORAGE_DIR = os.environ.get("REPORT_STORAGE_DIR", f"{MEDIA_ROOT}diskreports/") # todo: should be handled better, this is a fix to know for sure reports can be written to disk... # check diskreport dir, todo: move to django file field -full_report_storage_dir = f"{REPORT_STORAGE_DIR}original/UrlListReport/" -if not os.path.isdir(full_report_storage_dir): - os.makedirs(full_report_storage_dir, exist_ok=True) +FULL_REPORT_STORAGE_DIR = f"{REPORT_STORAGE_DIR}original/UrlListReport/" +if not os.path.isdir(FULL_REPORT_STORAGE_DIR): + os.makedirs(FULL_REPORT_STORAGE_DIR, exist_ok=True) AUTHENTICATION_BACKENDS = [ # Needed to login by username in Django admin, regardless of `allauth` - 'django.contrib.auth.backends.ModelBackend', - + "django.contrib.auth.backends.ModelBackend", # `allauth` specific authentication methods, such as login by email - 'allauth.account.auth_backends.AuthenticationBackend', + "allauth.account.auth_backends.AuthenticationBackend", ] # https://docs.allauth.org/en/latest/installation/quickstart.html diff --git a/dashboard/settings_constance.py b/dashboard/settings_constance.py index 37b813b1..6fe8a962 100644 --- a/dashboard/settings_constance.py +++ b/dashboard/settings_constance.py @@ -1,7 +1,7 @@ import os from collections import OrderedDict -CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' +CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend" CONSTANCE_ADDITIONAL_FIELDS = { "json": ["django.forms.fields.JSONField", {"required": False}], @@ -9,59 +9,70 @@ CONSTANCE_CONFIG = { # general settings - 'DASHBOARD_FRONTEND_URL': ( - os.environ.get("DASHBOARD_FRONTEND_URL", 'http://localhost:8000'), - 'Url where the frontend is reachable on for end users. This url is references in a few parts of the frontend.', - str + "DASHBOARD_FRONTEND_URL": ( + os.environ.get("DASHBOARD_FRONTEND_URL", "http://localhost:8000"), + "Url where the frontend is reachable on for end users. This url is references in a few parts of the frontend.", + str, ), - 'DASHBOARD_MAXIMUM_DOMAINS_PER_LIST': ( + "DASHBOARD_MAXIMUM_DOMAINS_PER_LIST": ( 1000, - 'The maximum amount of domains that can be in a list. There will be no crash when somebody imports more ' - 'via a spreadsheet: domains will be added but the list will refuse to scan and show a warning.' - 'In normal use cases these limits will not be reached as average lists are about 300 domains. Lists ' - 'with 600 domains are unusual. Lists with 10.000+ domains are exceptional.', - int + "The maximum amount of domains that can be in a list. There will be no crash when somebody imports more " + "via a spreadsheet: domains will be added but the list will refuse to scan and show a warning." + "In normal use cases these limits will not be reached as average lists are about 300 domains. Lists " + "with 600 domains are unusual. Lists with 10.000+ domains are exceptional.", + int, ), - 'DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET': ( + "DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET": ( 1000, - 'The maximum amount of domains that can be imported via a spreadsheet at one time. ' - 'In normal use cases these limits will not be reached.', - int + "The maximum amount of domains that can be imported via a spreadsheet at one time. " + "In normal use cases these limits will not be reached.", + int, ), - 'DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET': ( + "DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET": ( 20, - 'The maximum amount of lists that can be imported via a spreadsheet at one time. ' - 'In normal usec ases these limits will not be reached.', - int + "The maximum amount of lists that can be imported via a spreadsheet at one time. " + "In normal usec ases these limits will not be reached.", + int, ), - 'DASHBOARD_FRONT_PAGE_URL_LISTS': ( - '', - 'Comma separated list of urllists of which all reports will automatically be shared on the front page. ' - 'For example: 1,2,3. No data means the front page will not show any lists, just the usual information.', - str + "DASHBOARD_FRONT_PAGE_URL_LISTS": ( + "", + "Comma separated list of urllists of which all reports will automatically be shared on the front page. " + "For example: 1,2,3. No data means the front page will not show any lists, just the usual information.", + str, ), - # scan settings - 'SCAN_AT_ALL': ( + "SCAN_AT_ALL": ( True, - 'This enables or disabled all scans. Note that scans that are picked up will still be processed.', - bool + "This enables or disabled all scans. Note that scans that are picked up will still be processed.", + bool, ), - 'INTERNET_NL_API_URL': ( - 'http://localhost:8080/api/batch/v2', + "INTERNET_NL_API_URL": ( + "http://localhost:8080/api/batch/v2", 'The internet address for the Internet.nl API installation. This is commonly called a "batch server".', - str + str, ), - 'INTERNET_NL_SCAN_TRACKING_NAME': ( - '', - 'This setting is used when sending API requests for tracking purposes. Setting this value make it clear who ' - 'is sending API requests. A good setting contains something unique about this installation, such as an ' - 'organization name. The maximum length is 40 characters.', - str + "INTERNET_NL_SCAN_TRACKING_NAME": ( + "", + "This setting is used when sending API requests for tracking purposes. Setting this value make it clear who " + "is sending API requests. A good setting contains something unique about this installation, such as an " + "organization name. The maximum length is 40 characters.", + str, ), "SCANNER_NAMESERVERS": ( - ["193.17.47.1", "185.43.135.1", "193.110.81.0", "185.253.5.0", "9.9.9.9", "149.112.112.112", - "2001:148f:ffff::1", "2001:148f:fffe::1", "2a0f:fc80::", "2a0f:fc81::", "2620:fe::fe", "2620:fe::9"], + [ + "193.17.47.1", + "185.43.135.1", + "193.110.81.0", + "185.253.5.0", + "9.9.9.9", + "149.112.112.112", + "2001:148f:ffff::1", + "2001:148f:fffe::1", + "2a0f:fc80::", + "2a0f:fc81::", + "2620:fe::fe", + "2620:fe::9", + ], "Nameservers used during scans (dns endpoints and subdomains). This string is loaded as JSON, but not validated" " due to limitations of this settings library. Be careful when editing(!). " "This information is cached and loaded only once every 10 minutes.", @@ -72,64 +83,60 @@ "The url where internet.nl api credentials are checked. This is usually the bare INTERNET_NL_API_URL endpoint. " "This feature is used in the admin interface at account management. " "There the option 'check credentials' can be performed for each account.", - str + str, ), - # email settings - 'EMAIL_NOTIFICATION_SENDER': ( - 'noreply@localhost', - 'The sender of email report notification: this is the e-mail that contains the current scan results and a ' - 'summary. It also compares the result to the previous results. Use an e-mail address that is in use.', - str - ), - 'EMAIL_FALLBACK_LANGUAGE': ( - 'en', - 'Default language used for templates. Template should end with _en in lowercase. Example e-mail templates are ' - 'included and can be found in the menu of the admin interface.', - str - ), - 'EMAIL_TEST_RECIPIENT': ( - 'info@localhost', + "EMAIL_NOTIFICATION_SENDER": ( + "noreply@localhost", + "The sender of email report notification: this is the e-mail that contains the current scan results and a " + "summary. It also compares the result to the previous results. Use an e-mail address that is in use.", + str, + ), + "EMAIL_FALLBACK_LANGUAGE": ( + "en", + "Default language used for templates. Template should end with _en in lowercase. Example e-mail templates are " + "included and can be found in the menu of the admin interface.", + str, + ), + "EMAIL_TEST_RECIPIENT": ( + "info@localhost", 'Which e-mail address receives the testmail from the command "dashboard send_testmail". This command tests if ' - 'the e-mail outbox is properly configured.', - str + "the e-mail outbox is properly configured.", + str, ), - 'EMAIL_DASHBOARD_ADDRESS': ( - 'http://localhost:8000', - 'The address of the dashboard, can be set to any url. Available in email template at {{dashboard_address}}. ' - 'This is probably the same as the DASHBOARD_FRONTEND_URL. Only in rare cases this would differ.', - str + "EMAIL_DASHBOARD_ADDRESS": ( + "http://localhost:8000", + "The address of the dashboard, can be set to any url. Available in email template at {{dashboard_address}}. " + "This is probably the same as the DASHBOARD_FRONTEND_URL. Only in rare cases this would differ.", + str, ), - # security.txt "SECURITY_TXT_IS_REDIRECTED": ( False, "Security.txt is used to allow security researchers to report vulnerabilities. This can be either set to a " "redirect to an existing security.txt or configured with your own security.txt policy.", - bool + bool, ), "SECURITY_TXT_REDIRECT_URL": ( "http://localhost:8000/.well-known/security.txt", "The url where the security.txt files redirect to. This is usually an external site.", - str + str, ), "SECURITY_TXT_CONTENT": ( "", "The content of the security.txt file, located at .well-known/security.txt. Only " "used when redirect is disabled. Go to securitytxt.org to create a configuration " "for this installation.", - str + str, ), - # frontend "SITE_LAYOUT_NAME": ( - '', - 'The name of the layout, when internet_nl is used, logos, footer and styling from internet.nl is used. When ' - 'this field is empty all references to internet.nl disappear while still using the same color scheme. ' - 'Supported values: internet_nl, [empty]', - str + "", + "The name of the layout, when internet_nl is used, logos, footer and styling from internet.nl is used. When " + "this field is empty all references to internet.nl disappear while still using the same color scheme. " + "Supported values: internet_nl, [empty]", + str, ), - # signup settings "SHOW_SIGNUP_FORM": ( False, @@ -137,74 +144,48 @@ "only internet.nl signup questions are available. So this might not be useful for most installations.", bool, ), - 'EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP': ( - 'noreply@localhost', + "EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP": ( + "noreply@localhost", 'The sender of the "thank you" e-mail after signing up. The template for this e-mail can be found in the ' - 'E-Mail templates menu of the admin interface.', - str + "E-Mail templates menu of the admin interface.", + str, ), - 'DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES': ( - 'support@localhost', - 'Comma separated list of email addresses to notify about new signups. Don\'t add extra spaces in between.', - str + "DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES": ( + "support@localhost", + "Comma separated list of email addresses to notify about new signups. Don't add extra spaces in between.", + str, ), - # timeouts - "SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS": ( - 10000, - 'timeout for phase DISCOVERING_ENDPOINTS', - int - ), - "SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS": ( - 1440, - 'timeout for phase RETRIEVING_SCANABLE_URLS', - int - ), + "SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS": (10000, "timeout for phase DISCOVERING_ENDPOINTS", int), + "SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS": (1440, "timeout for phase RETRIEVING_SCANABLE_URLS", int), "SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL": ( 1440, - 'timeout for phase REGISTERING_SCAN_AT_INTERNET_NL', - int - ), - "SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS": ( - 10000, - 'timeout for phase IMPORTING_SCAN_RESULTS', - int - ), - "SCAN_TIMEOUT_MINUTES_CREATING_REPORT": ( - 10000, - 'timeout for phase CREATING_REPORT', - int - ), - "SCAN_TIMEOUT_MINUTES_SENDING_MAIL": ( - 1440, - 'timeout for phase SENDING_MAIL', - int - ), - "SCAN_TIMEOUT_MINUTES_SERVER_ERROR": ( - 1440, - 'timeout for phase SERVER_ERROR', - int + "timeout for phase REGISTERING_SCAN_AT_INTERNET_NL", + int, ), - + "SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS": (10000, "timeout for phase IMPORTING_SCAN_RESULTS", int), + "SCAN_TIMEOUT_MINUTES_CREATING_REPORT": (10000, "timeout for phase CREATING_REPORT", int), + "SCAN_TIMEOUT_MINUTES_SENDING_MAIL": (1440, "timeout for phase SENDING_MAIL", int), + "SCAN_TIMEOUT_MINUTES_SERVER_ERROR": (1440, "timeout for phase SERVER_ERROR", int), # other stuff - 'INTERNET_NL_API_USERNAME': ( - 'dummy', - 'Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep ' - 'this value set to dummy for legacy reasons.', - str), + "INTERNET_NL_API_USERNAME": ( + "dummy", + "Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep " + "this value set to dummy for legacy reasons.", + str, + ), # this is defaulting to dummy as otherwise the scanner wil give an error that no credential has been configured. - 'INTERNET_NL_API_PASSWORD': ( - 'dummy', - 'Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep ' - 'this value set to dummy for legacy reasons.', - str + "INTERNET_NL_API_PASSWORD": ( + "dummy", + "Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep " + "this value set to dummy for legacy reasons.", + str, ), - 'INTERNET_NL_MAXIMUM_URLS': ( + "INTERNET_NL_MAXIMUM_URLS": ( 1000, - 'The maximum amount of domains per scan, not relevant for dashboard, only for websecmap.', - int + "The maximum amount of domains per scan, not relevant for dashboard, only for websecmap.", + int, ), - "SCANNER_LOG_PLANNED_SCANS": ( False, "Used when debugging, logs all changes to planned scans to a separate table. Causes millions of records a day", @@ -250,12 +231,7 @@ "Do not send in subdomains. To reduce the number of tests while still getting an impression on a broader scope", bool, ), - "PROJECT_WEBSITE": ( - "", - "", - str - ), - + "PROJECT_WEBSITE": ("", "", str), "SUBDOMAIN_SUGGESTION_ENABLED": ( False, "Do you want subdomain suggestions to become available in the web interface?", @@ -277,110 +253,96 @@ "The amount of days to extend the range to search for available subdomains", int, ), - "SUPPORTED_LANGUAGES": ( ["en", "nl"], "Languages supported in the front end, if you need a new language, add it to the dashboard project.", "json", ), - } CONSTANCE_CONFIG_FIELDSETS = OrderedDict( [ ( - 'General Dashboard Settings', ( - 'DASHBOARD_FRONTEND_URL', - 'DASHBOARD_MAXIMUM_DOMAINS_PER_LIST', - 'DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET', - 'DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET', - 'DASHBOARD_FRONT_PAGE_URL_LISTS' - ) - ), - - ( - 'Front End Settings', ( - "SITE_LAYOUT_NAME", - "SUPPORTED_LANGUAGES" - ) + "General Dashboard Settings", + ( + "DASHBOARD_FRONTEND_URL", + "DASHBOARD_MAXIMUM_DOMAINS_PER_LIST", + "DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET", + "DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET", + "DASHBOARD_FRONT_PAGE_URL_LISTS", + ), ), - + ("Front End Settings", ("SITE_LAYOUT_NAME", "SUPPORTED_LANGUAGES")), ( - 'Internet.nl Scan Settings', ( - 'SCAN_AT_ALL', - 'INTERNET_NL_API_URL', + "Internet.nl Scan Settings", + ( + "SCAN_AT_ALL", + "INTERNET_NL_API_URL", "INTERNET_NL_SCAN_TRACKING_NAME", "SCANNER_NAMESERVERS", "CREDENTIAL_CHECK_URL", - ) - ), - - ( - 'E-Mail Settings', ( - 'EMAIL_NOTIFICATION_SENDER', - 'EMAIL_FALLBACK_LANGUAGE', - 'EMAIL_TEST_RECIPIENT', - 'EMAIL_DASHBOARD_ADDRESS', ), ), - ( - "Security.txt", ( - "SECURITY_TXT_IS_REDIRECTED", - "SECURITY_TXT_REDIRECT_URL", - "SECURITY_TXT_CONTENT" - ) + "E-Mail Settings", + ( + "EMAIL_NOTIFICATION_SENDER", + "EMAIL_FALLBACK_LANGUAGE", + "EMAIL_TEST_RECIPIENT", + "EMAIL_DASHBOARD_ADDRESS", + ), ), - + ("Security.txt", ("SECURITY_TXT_IS_REDIRECTED", "SECURITY_TXT_REDIRECT_URL", "SECURITY_TXT_CONTENT")), ( - "Subdomain suggestions", ( + "Subdomain suggestions", + ( "SUBDOMAIN_SUGGESTION_ENABLED", "SUBDOMAIN_SUGGESTION_SERVER_ADDRESS", "SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD", - "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD" - ) + "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD", + ), ), - ( - 'Signup Settings (internet.nl only)', ( - 'SHOW_SIGNUP_FORM', - 'EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP', - 'DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES' - ) + "Signup Settings (internet.nl only)", + ( + "SHOW_SIGNUP_FORM", + "EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP", + "DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES", + ), ), - ( - "Timeouts (advanced)", ( - 'SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS', - 'SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS', - 'SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL', - 'SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS', - 'SCAN_TIMEOUT_MINUTES_CREATING_REPORT', - 'SCAN_TIMEOUT_MINUTES_SENDING_MAIL', - 'SCAN_TIMEOUT_MINUTES_SERVER_ERROR', - ) + "Timeouts (advanced)", + ( + "SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS", + "SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS", + "SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL", + "SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS", + "SCAN_TIMEOUT_MINUTES_CREATING_REPORT", + "SCAN_TIMEOUT_MINUTES_SENDING_MAIL", + "SCAN_TIMEOUT_MINUTES_SERVER_ERROR", + ), ), - ( - "Logging settings (advanced)", ( + "Logging settings (advanced)", + ( "SCANNER_LOG_PLANNED_SCANS", "SCANNER_AUTO_PURGE_FINISHED_SCANS", - ) + ), ), - ( - "Unused / Expert settings", ( - 'INTERNET_NL_API_USERNAME', - 'INTERNET_NL_API_PASSWORD', - 'INTERNET_NL_MAXIMUM_URLS', + "Unused / Expert settings", + ( + "INTERNET_NL_API_USERNAME", + "INTERNET_NL_API_PASSWORD", + "INTERNET_NL_MAXIMUM_URLS", "INTERNET_NL_ADD_CALCULATED_RESULTS_WEBSECMAP", "INTERNET_NL_ADD_CALCULATED_RESULTS_FORUM_STANDAARDISATIE", "INTERNET_NL_ADD_CALCULATED_RESULTS_VNG_V6", "INTERNET_NL_WEB_ONLY_TOP_LEVEL", "IPV6_TEST_DOMAIN", "CONNECTIVITY_TEST_DOMAIN", - "PROJECT_WEBSITE" - ) - ) + "PROJECT_WEBSITE", + ), + ), ] ) diff --git a/dashboard/settings_jet.py b/dashboard/settings_jet.py index c7548403..99482c1c 100644 --- a/dashboard/settings_jet.py +++ b/dashboard/settings_jet.py @@ -1,58 +1,74 @@ from django.utils.translation import gettext_lazy as _ JET_SIDE_MENU_ITEMS = [ - - {'label': '', 'items': [ - {'name': 'constance.config', 'label': '🎛️ Dashboard Configuration'}, - {'name': 'django_mail_admin.emailtemplate', 'label': '📨 E-Mail Templates'}, - {'name': 'django_mail_admin.outbox', 'label': '📨 Outboxes'}, - {'name': 'django_celery_beat.periodictask', 'label': '⏰ Periodic Tasks'}, - {'name': 'auth.user', 'label': '👤 Users'}, - {'name': 'internet_nl_dashboard.account', 'label': '🏢 Accounts'}, - {'name': 'otp_totp.totpdevice', 'label': '📱 TOTP Devices'}, - ]}, - - {'label': _('📘 Dashboard'), 'items': [ - {'name': 'internet_nl_dashboard.urllist', 'label': "Domain lists"}, - {'name': 'internet_nl_dashboard.taggedurlinurllist', 'label': 'Tagged Url'}, - {'name': 'internet_nl_dashboard.uploadlog', 'label': 'Uploads'}, - ]}, - - {'label': _('🔬 Scan'), 'items': [ - {'name': 'scanners.internetnlscaninspection', 'label': 'Scan Inspections'}, - {'name': 'internet_nl_dashboard.accountinternetnlscan'}, - {'name': 'internet_nl_dashboard.accountinternetnlscanlog'}, - {'name': 'scanners.internetnlv2scan', 'label': 'Internet.nl Scans Tasks'}, - {'name': 'scanners.internetnlv2statelog', 'label': 'Internet.nl Scans Log'}, - {'name': 'internet_nl_dashboard.subdomaindiscoveryscan', 'label': 'Subdomain Discovery'} - ]}, - - {'label': _('💽 Data'), 'items': [ - {'name': 'organizations.url', 'label': 'Urls'}, - {'name': 'scanners.endpoint', 'label': 'Endpoints'}, - {'name': 'scanners.endpointgenericscan', 'label': 'Endpoint Scans'}, - ]}, - - {'label': _('📊 Report'), 'items': [ - {'name': 'reporting.urlreport', 'label': 'Url Reports'}, - {'name': 'internet_nl_dashboard.urllistreport', 'label': 'Full Reports'} - ]}, - - {'label': _('🕒 Periodic Tasks'), 'items': [ - {'name': 'django_celery_beat.periodictask'}, - {'name': 'django_celery_beat.crontabschedule'}, - ]}, - - {'label': _('📨 E-Mail'), 'items': [ - {'name': 'django_mail_admin.emailtemplate', 'label': 'Templates'}, - {'name': 'django_mail_admin.outgoingemail', 'label': 'Sent mail'}, - {'name': 'django_mail_admin.outbox', 'label': 'Outboxes'}, - {'name': 'django_mail_admin.log', 'label': 'Logs'}, - ]}, - - {'label': _('✨ Activity'), 'items': [ - {'name': 'actstream.action'}, - ]}, + { + "label": "", + "items": [ + {"name": "constance.config", "label": "🎛️ Dashboard Configuration"}, + {"name": "django_mail_admin.emailtemplate", "label": "📨 E-Mail Templates"}, + {"name": "django_mail_admin.outbox", "label": "📨 Outboxes"}, + {"name": "django_celery_beat.periodictask", "label": "⏰ Periodic Tasks"}, + {"name": "auth.user", "label": "👤 Users"}, + {"name": "internet_nl_dashboard.account", "label": "🏢 Accounts"}, + {"name": "otp_totp.totpdevice", "label": "📱 TOTP Devices"}, + ], + }, + { + "label": _("📘 Dashboard"), + "items": [ + {"name": "internet_nl_dashboard.urllist", "label": "Domain lists"}, + {"name": "internet_nl_dashboard.taggedurlinurllist", "label": "Tagged Url"}, + {"name": "internet_nl_dashboard.uploadlog", "label": "Uploads"}, + ], + }, + { + "label": _("🔬 Scan"), + "items": [ + {"name": "scanners.internetnlscaninspection", "label": "Scan Inspections"}, + {"name": "internet_nl_dashboard.accountinternetnlscan"}, + {"name": "internet_nl_dashboard.accountinternetnlscanlog"}, + {"name": "scanners.internetnlv2scan", "label": "Internet.nl Scans Tasks"}, + {"name": "scanners.internetnlv2statelog", "label": "Internet.nl Scans Log"}, + {"name": "internet_nl_dashboard.subdomaindiscoveryscan", "label": "Subdomain Discovery"}, + ], + }, + { + "label": _("💽 Data"), + "items": [ + {"name": "organizations.url", "label": "Urls"}, + {"name": "scanners.endpoint", "label": "Endpoints"}, + {"name": "scanners.endpointgenericscan", "label": "Endpoint Scans"}, + ], + }, + { + "label": _("📊 Report"), + "items": [ + {"name": "reporting.urlreport", "label": "Url Reports"}, + {"name": "internet_nl_dashboard.urllistreport", "label": "Full Reports"}, + ], + }, + { + "label": _("🕒 Periodic Tasks"), + "items": [ + {"name": "django_celery_beat.periodictask"}, + {"name": "django_celery_beat.crontabschedule"}, + ], + }, + { + "label": _("📨 E-Mail"), + "items": [ + {"name": "django_mail_admin.emailtemplate", "label": "Templates"}, + {"name": "django_mail_admin.outgoingemail", "label": "Sent mail"}, + {"name": "django_mail_admin.outbox", "label": "Outboxes"}, + {"name": "django_mail_admin.log", "label": "Logs"}, + ], + }, + { + "label": _("✨ Activity"), + "items": [ + {"name": "actstream.action"}, + ], + }, ] JET_SIDE_MENU_COMPACT = True diff --git a/dashboard/settings_util.py b/dashboard/settings_util.py index c1dac259..c7311f4a 100644 --- a/dashboard/settings_util.py +++ b/dashboard/settings_util.py @@ -4,7 +4,7 @@ from django.core.management.utils import get_random_secret_key # These secret keys are defaults and will not work as confirm_keys_are_changed will check if these are changed. -DEFAULT_SECRET_KEY = '_dzlo^9d#ox6!7c9rju@=u8+4^sprqocy3s*l*ejc2yr34@&98' # nosec +DEFAULT_SECRET_KEY = "_dzlo^9d#ox6!7c9rju@=u8+4^sprqocy3s*l*ejc2yr34@&98" # nosec DEFAULT_FIELD_ENCRYPTION_KEY = "JjvHNnFMfEaGd7Y0SAHBRNZYGGpNs7ydEp-ixmKSvkQ=" # nosec @@ -13,19 +13,19 @@ def get_secret_key_from_file_or_env() -> str: creating a keyfile is one is not present yet. Otherwise will read from SECRET_KEY env or fallback to default key.""" - if secret_key_file := os.environ.get('SECRET_KEY_FILE', None): + if secret_key_file := os.environ.get("SECRET_KEY_FILE", None): if not os.path.exists(secret_key_file): # It's keys all the way down secret_key = get_random_secret_key() # noqa py/clear-text-storage-sensitive-data - with open(secret_key_file, 'w+', encoding="UTF-8") as f: + with open(secret_key_file, "w+", encoding="UTF-8") as f: f.write(secret_key) - with open(secret_key_file, 'r', encoding="UTF-8") as f: + with open(secret_key_file, "r", encoding="UTF-8") as f: secret_key = f.readline().strip() else: # SECURITY WARNING: keep the secret key used in production secret! # The routine confirm_keys_are_changed is run in production and will prevent the default keys to be used. - secret_key: str = os.environ.get('SECRET_KEY', DEFAULT_SECRET_KEY) + secret_key: str = os.environ.get("SECRET_KEY", DEFAULT_SECRET_KEY) return secret_key @@ -35,14 +35,14 @@ def get_field_encryption_key_from_file_or_env() -> bytes: creating a keyfile is one is not present yet. Otherwise will read from FIELD_ENCRYPTION_KEY env or fallback to default key.""" - if field_encryption_key_file := os.environ.get('FIELD_ENCRYPTION_KEY_FILE', None): + if field_encryption_key_file := os.environ.get("FIELD_ENCRYPTION_KEY_FILE", None): if not os.path.exists(field_encryption_key_file): field_encryption_key: bytes = Fernet.generate_key() # Binary mode doesn't take an encoding argument - with open(field_encryption_key_file, 'wb+') as f: + with open(field_encryption_key_file, "wb+") as f: f.write(field_encryption_key) - with open(field_encryption_key_file, 'rb') as f: + with open(field_encryption_key_file, "rb") as f: field_encryption_key: bytes = f.readline().strip() else: # Note that this key is not stored in the database, that would be a security risk. @@ -56,6 +56,6 @@ def get_field_encryption_key_from_file_or_env() -> bytes: # Also note that on the production server a different key is required, otherwise the server will not start. # See dashboard_prdserver for more details. # The routine confirm_keys_are_changed is run in production and will prevent the default keys to be used. - field_encryption_key: bytes = os.environ.get('FIELD_ENCRYPTION_KEY', DEFAULT_FIELD_ENCRYPTION_KEY).encode() + field_encryption_key: bytes = os.environ.get("FIELD_ENCRYPTION_KEY", DEFAULT_FIELD_ENCRYPTION_KEY).encode() return field_encryption_key diff --git a/dashboard/urls.py b/dashboard/urls.py index 118fa311..ad9a332d 100644 --- a/dashboard/urls.py +++ b/dashboard/urls.py @@ -23,9 +23,9 @@ from . import __version__ -admin.site.site_header = f'Dashboard Admin {__version__}' +admin.site.site_header = f"Dashboard Admin {__version__}" # Don't show version in title, as that might be shared without auth -admin.site.site_title = 'Dashboard Admin' +admin.site.site_title = "Dashboard Admin" def trigger_error(request): @@ -34,13 +34,13 @@ def trigger_error(request): admin_urls = [ - path('sentry-debug/', trigger_error), - re_path(r'^admin/', admin.site.urls), - re_path(r'^admin/doc/', include('django.contrib.admindocs.urls')), - re_path(r'^admin/jet/', include('jet.urls', 'jet')), - re_path(r'^admin/jet/dashboard/', include('jet.dashboard.urls', 'jet-dashboard')), - re_path(r'^nested_admin/', include('nested_admin.urls')), - re_path(r'^activity/', include('actstream.urls')), + path("sentry-debug/", trigger_error), + re_path(r"^admin/", admin.site.urls), + re_path(r"^admin/doc/", include("django.contrib.admindocs.urls")), + re_path(r"^admin/jet/", include("jet.urls", "jet")), + re_path(r"^admin/jet/dashboard/", include("jet.dashboard.urls", "jet-dashboard")), + re_path(r"^nested_admin/", include("nested_admin.urls")), + re_path(r"^activity/", include("actstream.urls")), ] @@ -49,11 +49,10 @@ def not_supported(**args): frontend_urls = [ - path('', include('dashboard.internet_nl_dashboard.urls')), + path("", include("dashboard.internet_nl_dashboard.urls")), # Enabling the default auth logins can bypass the two factor authentication. Don't enable it. # path('', include('django.contrib.auth.urls')), - re_path(r'', include(tf_urls)), - + re_path(r"", include(tf_urls)), # allauth also uses 429.html template when the included rate limits are reached. You can try this # by submitting a bunch of password changes in a second or so. # this exposes a password reset and change form. @@ -62,9 +61,12 @@ def not_supported(**args): # with the accounts we currently have. So social login might be feasible, but the normal log # not yet. Also disable reauthenticate # -> this view is mandatory to register, but directing it to the wrong page on purpose - path("accounts/password/", not_supported, name="account_login", ), + path( + "accounts/password/", + not_supported, + name="account_login", + ), path("accounts/password/reset/", not_supported, name="account_reset_password"), - # https://github.com/pennersr/django-allauth/issues/468 path( "accounts/password/change/", @@ -75,7 +77,6 @@ def not_supported(**args): ), name="account_change_password", ), - # These have not yet been requested and not tested how the work and if they are useful. # path( # "accounts/password/reset/", password_reset, name="account_reset_password" diff --git a/dashboard/wsgi.py b/dashboard/wsgi.py index 07d87ee6..0c6dbad3 100644 --- a/dashboard/wsgi.py +++ b/dashboard/wsgi.py @@ -12,6 +12,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dashboard.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings") application = get_wsgi_application() diff --git a/docs/input/conf.py b/docs/input/conf.py index a4e4ce9f..a3a14382 100644 --- a/docs/input/conf.py +++ b/docs/input/conf.py @@ -17,9 +17,9 @@ # -- Project information ----------------------------------------------------- -project = 'Internet.nl Dashboard' -copyright = '2020-2024, ECP / Internet.nl' -author = 'internet.nl' +project = "Internet.nl Dashboard" +copyright = "2020-2024, ECP / Internet.nl" +author = "internet.nl" # -- General configuration --------------------------------------------------- @@ -27,15 +27,17 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'rst2pdf.pdfbuilder', + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "rst2pdf.pdfbuilder", ] -pdf_documents = [('index', u'dashboard', u'Internet.nl Dashboard Documentation', u'internet.nl'), ] +pdf_documents = [ + ("index", "dashboard", "Internet.nl Dashboard Documentation", "internet.nl"), +] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -47,32 +49,31 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] -autodoc_member_order = 'bysource' +autodoc_member_order = "bysource" -latex_engine = 'xelatex' +latex_engine = "xelatex" latex_elements = { - 'papersize': 'a4papaer', - - 'fontpkg': r''' + "papersize": "a4papaer", + "fontpkg": r""" \setmainfont{DejaVu Serif} \setsansfont{DejaVu Sans} \setmonofont{DejaVu Sans Mono} -''', - 'preamble': r''' +""", + "preamble": r""" \usepackage[titles]{tocloft} \cftsetpnumwidth {1.25cm}\cftsetrmarg{1.5cm} \setlength{\cftchapnumwidth}{0.75cm} \setlength{\cftsecindent}{\cftchapnumwidth} \setlength{\cftsecnumwidth}{1.25cm} -''', - 'fncychap': r'\usepackage[Bjornstrup]{fncychap}', - 'printindex': r'\footnotesize\raggedright\printindex', +""", + "fncychap": r"\usepackage[Bjornstrup]{fncychap}", + "printindex": r"\footnotesize\raggedright\printindex", } -latex_show_urls = 'footnote' +latex_show_urls = "footnote" diff --git a/requirements-dev.in b/requirements-dev.in index d7a9fd0b..00db44cd 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -48,6 +48,7 @@ vulture pylint pylint-django ruff +black types-redis types-freezegun diff --git a/requirements-dev.txt b/requirements-dev.txt index 67d0f03a..7f75e157 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -76,6 +76,8 @@ billiard==4.2.1 # -c requirements.txt # celery # websecmap +black==24.10.0 + # via -r requirements-dev.in bs4==0.0.2 # via # -c requirements.txt @@ -122,6 +124,7 @@ charset-normalizer==3.4.0 click==8.1.7 # via # -c requirements.txt + # black # celery # click-didyoumean # click-plugins @@ -455,7 +458,9 @@ multidict==6.1.0 mypy==1.13.0 # via -r requirements-dev.in mypy-extensions==1.0.0 - # via mypy + # via + # black + # mypy mysqlclient==2.2.5 # via # -c requirements.txt @@ -501,9 +506,12 @@ osm2geojson==0.2.5 packaging==24.2 # via # -r requirements-dev.in + # black # pytest # rst2pdf # sphinx +pathspec==0.12.1 + # via black pbr==6.1.0 # via stevedore phonenumberslite==8.13.49 @@ -516,7 +524,9 @@ pillow==11.0.0 # reportlab # websecmap platformdirs==4.3.6 - # via pylint + # via + # black + # pylint pluggy==1.5.0 # via pytest prometheus-client==0.21.0 @@ -836,6 +846,7 @@ tomli==2.1.0 # via # autoflake # autopep8 + # black # coverage # django-stubs # mypy @@ -878,6 +889,7 @@ typing-extensions==4.12.2 # anyio # asgiref # astroid + # black # django-countries # django-stubs # django-stubs-ext diff --git a/setup.py b/setup.py index 93c25a94..c068d249 100644 --- a/setup.py +++ b/setup.py @@ -13,37 +13,41 @@ def get_version(): # try to use git tag if building python package try: # append git sha to version - return '0.0.' + check_output("git rev-list --count HEAD",shell=True).decode('utf8') + return "0.0." + check_output("git rev-list --count HEAD", shell=True).decode("utf8") except Exception as e: print("Failed to acquire version info from git: {e}".format(e=e), file=sys.stderr) - return '0.0.0' + return "0.0.0" def requirements(extra=None): """Return list of required package names from requirements.txt.""" # strip trailing comments, and extract package name from git urls. if extra: - filename = 'requirements-' + extra + '.txt' + filename = "requirements-" + extra + ".txt" else: - filename = 'requirements.txt' - requirements = [r.strip().split(';',1)[0].split(' ', 1)[0].split('egg=', 1)[-1] - for r in open(filename) if r.strip() and not r.strip().startswith('#')] + filename = "requirements.txt" + requirements = [ + r.strip().split(";", 1)[0].split(" ", 1)[0].split("egg=", 1)[-1] + for r in open(filename) + if r.strip() and not r.strip().startswith("#") + ] return requirements -for x in requirements(extra='deploy'): + +for x in requirements(extra="deploy"): print(x) setup( - name='dashboard', + name="dashboard", version=get_version(), packages=find_packages(), install_requires=requirements(), extras_require={ - 'deploy': requirements(extra='deploy'), + "deploy": requirements(extra="deploy"), }, entry_points={ - 'console_scripts': [ - 'dashboard = dashboard.manage:main', + "console_scripts": [ + "dashboard = dashboard.manage:main", ], }, include_package_data=True,