diff --git a/CHANGELOG.md b/CHANGELOG.md index 306a4e19..786988db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## V4.2.0 - 17 April 2023 + +### Added +- Editing of domain lists via spreadsheets (uploading and downloading spreadsheet files) +- Tag/label support in spreadsheet uploads and downloads +- Be able to automatically share the latest report of a list +- Signup form for easier onboarding + +### Changed +- www subdomain discovery has been replaced with crt.sh subdomain discovery +- Support for Django 4.2, with psql12 +- Minimum python version is now 3.10 + ## V4.1.0 - 7 February 2023 diff --git a/Makefile b/Makefile index a665fd78..f61b7caa 100644 --- a/Makefile +++ b/Makefile @@ -262,7 +262,7 @@ pylint: ${app} DJANGO_SETTINGS_MODULE=${app_name}.settings ${bin}/pylint --load-plugins pylint_django dashboard .QA: qa -qa: fix pylint bandit mypy vulture check test ruff +qa: fix pylint bandit vulture check test ruff ## Utility diff --git a/dashboard/internet_nl_dashboard/logic/__init__.py b/dashboard/internet_nl_dashboard/logic/__init__.py index 6775b2a2..c972731e 100644 --- a/dashboard/internet_nl_dashboard/logic/__init__.py +++ b/dashboard/internet_nl_dashboard/logic/__init__.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 from datetime import datetime, timezone -from typing import Any, Dict +from typing import Any, Dict, Optional def operation_response( - error: bool = False, success: bool = False, message: str = "", data: Dict = None + error: bool = False, success: bool = False, message: str = "", data: Optional[Dict[Any, Any]] = None ) -> Dict[str, Any]: return {'error': error, 'success': success, diff --git a/dashboard/internet_nl_dashboard/logic/domains.py b/dashboard/internet_nl_dashboard/logic/domains.py index bf6e17de..472324a5 100644 --- a/dashboard/internet_nl_dashboard/logic/domains.py +++ b/dashboard/internet_nl_dashboard/logic/domains.py @@ -2,7 +2,7 @@ import logging import re from datetime import datetime, timezone -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, Union import pyexcel as p import tldextract @@ -330,7 +330,7 @@ def update_list_settings(account: Account, user_input: Dict) -> Dict[str, Any]: to_attr='last_report' ) - urllist = UrlList.objects.all().filter( + urllist: UrlList = UrlList.objects.all().filter( account=account, id=user_input['id'], is_deleted=False @@ -762,7 +762,7 @@ def _add_to_urls_to_urllist_nicer(account: Account, current_list: UrlList, urls: return counters -def clean_urls(urls: List[str]) -> Dict[str, List]: +def clean_urls(urls: List[str]) -> Dict[str, List[Union[str, int]]]: """ Incorrect urls are urls that are not following the uri scheme standard and don't have a recognizable suffix. They are returned for informational purposes and can contain utter garbage. The editor of the urls can then easily see @@ -772,7 +772,7 @@ def clean_urls(urls: List[str]) -> Dict[str, List]: :return: """ - result: Dict[str, List] = {'incorrect': [], 'correct': []} + result: Dict[str, List[Union[str, int]]] = {'incorrect': [], 'correct': []} for url in urls: # all urls in the system must be lowercase (if applicable to used character) diff --git a/dashboard/internet_nl_dashboard/logic/shared_report_lists.py b/dashboard/internet_nl_dashboard/logic/shared_report_lists.py index 5b1bcc2d..c32d37c6 100644 --- a/dashboard/internet_nl_dashboard/logic/shared_report_lists.py +++ b/dashboard/internet_nl_dashboard/logic/shared_report_lists.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from typing import Dict, List, Optional from django.db.models import Prefetch @@ -6,7 +6,7 @@ from dashboard.internet_nl_dashboard.models import UrlList, UrlListReport -def get_latest_report_id_from_list_and_type(urllist_id: int, report_type: str = '') -> Dict[str, int]: +def get_latest_report_id_from_list_and_type(urllist_id: int, report_type: str = '') -> Dict[str, str]: report = UrlListReport.objects.filter(urllist=urllist_id, is_publicly_shared=True) if report_type in {"web", "mail"}: @@ -17,7 +17,7 @@ def get_latest_report_id_from_list_and_type(urllist_id: int, report_type: str = return ( {'latest_report_public_report_code': found_report.public_report_code} if found_report - else {'latest_report_public_report_code': None} + else {'latest_report_public_report_code': ''} ) @@ -25,7 +25,7 @@ def get_publicly_shared_lists_per_account_and_list_id(account_id: int, urllist_i return get_publicly_shared_lists_per_account(account_id, urllist_id) -def get_publicly_shared_lists_per_account(account_id, urllist_id: int = None) -> List[dict]: +def get_publicly_shared_lists_per_account(account_id, urllist_id: Optional[int] = None) -> List[dict]: log.debug(f"get_publicly_shared_lists_per_account account_id: {account_id}") diff --git a/dashboard/internet_nl_dashboard/logic/spreadsheet.py b/dashboard/internet_nl_dashboard/logic/spreadsheet.py index 4dd81911..990dacf4 100644 --- a/dashboard/internet_nl_dashboard/logic/spreadsheet.py +++ b/dashboard/internet_nl_dashboard/logic/spreadsheet.py @@ -131,7 +131,7 @@ def get_data(file: str) -> Dict[str, Dict[str, Dict[str, set]]]: :return: """ - data: Dict[str] = {} + data: Dict[str, Any] = {} try: sheet = p.get_sheet(file_name=file, name_columns_by_row=0) diff --git a/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py b/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py index 3d5a5f9e..38105ace 100644 --- a/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py +++ b/dashboard/internet_nl_dashboard/logic/urllist_dashboard_report.py @@ -2,7 +2,7 @@ import logging from copy import deepcopy from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from celery import group from deepdiff import DeepDiff @@ -120,7 +120,8 @@ def rate_urllists_now(urllists: List[UrlList], prevent_duplicates: bool = True, @app.task(queue='storage') def rate_urllist_on_moment( - urllist: UrlList, when: datetime = None, prevent_duplicates: bool = True, scan_type: str = "web") -> int: + urllist: UrlList, when: Optional[datetime] = None, prevent_duplicates: bool = True, + scan_type: str = "web") -> int: """ :param urllist: :param when: A moment in time of which data should be aggregated diff --git a/dashboard/internet_nl_dashboard/views/download_spreadsheet.py b/dashboard/internet_nl_dashboard/views/download_spreadsheet.py index f4822c2c..927a29a5 100644 --- a/dashboard/internet_nl_dashboard/views/download_spreadsheet.py +++ b/dashboard/internet_nl_dashboard/views/download_spreadsheet.py @@ -45,11 +45,11 @@ def create_spreadsheet_download(file_name: str, spreadsheet_data: Any, file_type if file_type == "xlsx-openpyxl": with open(spreadsheet_data.name, 'rb') as file_handle: - output = HttpResponse(file_handle.read()) + output: HttpResponse = HttpResponse(file_handle.read()) file_type = "xlsx" else: # Simple xls files and such - output: HttpResponse = excel.make_response(spreadsheet_data, file_type) + output = excel.make_response(spreadsheet_data, file_type) output["Content-Disposition"] = f"attachment; filename={slugify(file_name)}.{file_type}" output["Content-type"] = content_types[file_type] diff --git a/dashboard/settings.py b/dashboard/settings.py index e7e869d8..3f8319ca 100644 --- a/dashboard/settings.py +++ b/dashboard/settings.py @@ -684,9 +684,10 @@ SENTRY_DSN = os.environ.get('SENTRY_DSN') if SENTRY_DSN: # new sentry_sdk implementation, with hopes to also get exceptions from workers. - sentry_sdk.init(dsn=SENTRY_DSN, # pylint: disable=abstract-class-instantiated # (following the documentation) - integrations=[CeleryIntegration(), DjangoIntegration(), RedisIntegration()], - release=__version__, send_default_pii=False) + sentry_sdk.init( # pylint: disable=abstract-class-instantiated # (following the documentation) # type: ignore + dsn=SENTRY_DSN, + integrations=[CeleryIntegration(), DjangoIntegration(), RedisIntegration()], + release=__version__, send_default_pii=False) SENTRY_ORGANIZATION = 'internet-cleanup-foundation' SENTRY_PROJECT = 'internet-nl-dashboard' diff --git a/setup.cfg b/setup.cfg index 8e80fe83..af85266e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ python_version = 3.10 warn_return_any = True warn_unused_configs = True plugins = mypy_django_plugin.main - +exclude = dashboard/internet_nl_dashboard/tests # We're early in trying mypy, therefore 99.99% of python projects do not have stub files. # their solution to add #type: ignore after each external import is a bit over the top. # https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports