Merge pull request #557 from internetstandards/50

clean up some installation details

CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## V5.0.0 - t.b.d.
+Subdomain suggestions
+
+### Added
+- Subdomain suggestions using the [CTLSSA tool](https://github.com/internetstandards/Internet.nl-ct-log-subdomain-suggestions-api/) (#434)
+- Extensive installation guide, [these quick instructions](https://github.com/internetstandards/Internet.nl-dashboard/blob/main/docs/render/markdown/1_installation.md) (#495)
+- Added German and French translations via DeepL + translations warnings (these will contain imperfections)
+
+### Changed
+- Major javascript front-end rework to remove vulnerabilities and being able to stay up to date
+- Various layout fixes to improve experience of the dashboard on mobile (#472)
+- Reworked the translations to support AI translations
+
+
 ## V4.4.0 - 22 july 2024
 Maintenance release to reduce the amount of disk space used.
 

README.md

@@ -1,5 +1,5 @@
-For quick installation: Follow [these quick instructions](https://github.com/internetstandards/Internet.nl-dashboard/blob/50/docs/render/markdown/1_installation.md) 
-and watch [this 6 minute video](https://github.com/internetstandards/Internet.nl-dashboard/tree/50/docs/input/internet.nl%20dashboard%20installation%20video%20small.mp4).
+For quick installation: Follow [these quick instructions](https://github.com/internetstandards/Internet.nl-dashboard/blob/main/docs/render/markdown/1_installation.md) 
+and watch [this 6 minute video](https://github.com/internetstandards/Internet.nl-dashboard/tree/main/docs/input/internet.nl%20dashboard%20installation%20video%20small.mp4).
 
 # Internet.nl Dashboard
 The internet.nl dashboard allows you to visualize batch scans from the internet.nl API. It allows:
@@ -18,8 +18,8 @@ The internet.nl dashboard allows you to visualize batch scans from the internet.
 
 ## Setup / installation
 
-For quick installation: Follow [these quick instructions](https://github.com/internetstandards/Internet.nl-dashboard/blob/50/docs/render/markdown/1_installation.md) 
-and watch [this 6 minute video](https://github.com/internetstandards/Internet.nl-dashboard/tree/50/docs/input/internet.nl%20dashboard%20installation%20video%20small.mp4).
+For quick installation: Follow [these quick instructions](https://github.com/internetstandards/Internet.nl-dashboard/blob/main/docs/render/markdown/1_installation.md) 
+and watch [this 6 minute video](https://github.com/internetstandards/Internet.nl-dashboard/tree/main/docs/input/internet.nl%20dashboard%20installation%20video%20small.mp4).
 
 
 ## Screenshots
@@ -244,6 +244,33 @@ The dependency on Web Security Map is version pinned by a Git SHA in the Websecm
 
 ## FAQ / Troubleshooting
 
+
+### Updating translations
+Works best on a clean commit.
+
+```python
+dashboard update_translations_from_internet_nl 
+```
+
+This will update two PO files and Three JS files, the following:
+```sh
+git status
+```
+
+```text
+Changes not staged for commit:
+  (use "git add <file>..." to update what will be committed)
+  (use "git restore <file>..." to discard changes in working directory)
+	modified:   dashboard/internet_nl_dashboard/locale/en/LC_MESSAGES/django.po
+	modified:   dashboard/internet_nl_dashboard/locale/nl/LC_MESSAGES/django.po
+	modified:   dashboard/internet_nl_dashboard/static/js/translations/internet_nl.en.js
+	modified:   dashboard/internet_nl_dashboard/static/js/translations/internet_nl.js
+	modified:   dashboard/internet_nl_dashboard/static/js/translations/internet_nl.nl.js
+```
+
+The .js files can be used for inspiration in the internet.nl dashboard.
+
+
 ### Missing xcode (mac users)
 During installation mac users might get the following error, due to not having xcode installed or updated.
 

dashboard/internet_nl_dashboard/logic/internet_nl_translations.py

@@ -1,26 +1,24 @@
 # SPDX-License-Identifier: Apache-2.0
+import json
 import logging
 import os
 import tempfile
+from functools import lru_cache
 from pathlib import Path
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List
 
 import markdown
 import polib
 import requests
 from django.utils.text import slugify
 
-# Todo: refactor this to languages from settings.py
 SUPPORTED_LOCALES: List[str] = ["nl", "en"]
 
 log = logging.getLogger(__package__)
 
-# .parent = logic; .parent.parent = internet_nl_dashboard;
 DASHBOARD_APP_DIRECTORY = Path(__file__).parent.parent
 VUE_I18N_OUTPUT_PATH = f"{DASHBOARD_APP_DIRECTORY}/static/js/translations/"
 DJANGO_I18N_OUTPUT_PATH = f"{DASHBOARD_APP_DIRECTORY}/locale/"
-# log.debug(f"VUE_I18N_OUTPUT_PATH: {VUE_I18N_OUTPUT_PATH}")
-# log.debug(f"DJANGO_I18N_OUTPUT_PATH: {DJANGO_I18N_OUTPUT_PATH}")
 
 
 def convert_internet_nl_content_to_vue():
@@ -51,27 +49,20 @@ def convert_internet_nl_content_to_vue():
 
     :return: None
     """
-
-    translated_locales: List[Dict[str, Union[str, List[Any]]]] = []
-    combined_vue_i18n_content = ""
-
     for locale in SUPPORTED_LOCALES:
         raw_content: bytes = get_locale_content(locale)
         store_as_django_locale(locale, raw_content)
-        structured_content = load_as_po_file(raw_content)
-        translated_locales.append({"locale": locale, "content": structured_content})
 
         # support a per-language kind of file, in case we're going to do dynamic loading of languages.
-        vue_i18n_content: str = convert_vue_i18n_format(locale, structured_content)
-        combined_vue_i18n_content += vue_i18n_content
-        store_vue_i18n_file(f"internet_nl.{locale}", vue_i18n_content)
-
-    # the locales are easiest stored together. This makes language switching a lot easier.
-    store_vue_i18n_file("internet_nl", combined_vue_i18n_content)
+        json_content = convert_json_format(load_as_po_file(raw_content))
+        store_vue_i18n_file(f"internet_nl.{locale}.json", json.dumps(json_content, indent=2, ensure_ascii=False))
 
 
+@lru_cache(maxsize=None)
 def get_locale_content(locale: str) -> bytes:
     """
+    Use LRU cache as this script is not run for long times and basically returns the same stuff per run.
+
     A simple download and return response function.
 
     Input files:
@@ -127,51 +118,51 @@ def load_as_po_file(raw_content: bytes) -> List[Any]:
         return polib.pofile(file.name)
 
 
-def convert_vue_i18n_format(locale: str, po_content: Any) -> str:
-    """
-    done: will markdown be parsed to html in this method? Or should we do that on the fly, everywhere...
-          It seems the logical place will be to parse it here. Otherwise the rest of the application becomes more
-          complex. Using markdown will have the benefit of the output being a single html string with proper
-          formatting.
-
-    todo: change parameters {{ param }} to hello: '%{msg} world'
-          see: http://kazupon.github.io/vue-i18n/guide/formatting.html#list-formatting
-          The change is very large we don't need to do that, as we don't need those sentences.
-
-    The content is added to the 'internet_nl' key, like this:
-
-    const internet_nl_messages = {
-        en: {
-            internet_nl: {
-                key: 'value',
-                key: 'value'
-
-            },
-        },
-    }
-
-    There is a slight challenge that translations in vue are based on javascript properties, meaning, no quotes.
-
-    :return:
-    """
-
-    content: str = _vue_format_start()
-
-    content += _vue_format_locale_start(locale)
+def convert_json_format(po_content: Any) -> dict:
+    content = {}
 
     for entry in po_content:
         # to save a boatload of data, we're not storing the 'content' from the pages of internet.nl
         # we'll just have to point to this content.
         if entry.msgid.endswith("content"):
             continue
 
-        content += f"            {_js_safe_msgid(entry.msgid)}: '{_js_safe_msgstr(entry.msgstr)}',\n"
-    content += _vue_format_locale_end()
-    content += _vue_format_end()
+        message_id = _js_safe_msgid(entry.msgid)
+        content[message_id] = _json_safe_msgstr(entry.msgstr)
+
+        # todo: split tech table key into multiple keys so translations can be performed
+        # tech table is pipe seperated implicit translated, like this:
+        # "detail_mail_ipv6_mx_aaaa_tech_table": "Mail server (MX)|IPv6 address|IPv4 address",
+        # so we will make a translation like this:
+        # "detail_mail_ipv6_mx_aaaa_tech_table_mail_server_mx": "Mail server (MX)",
+        # "detail_mail_ipv6_mx_aaaa_tech_table_ipv6_address": "IPv6 address",
+        # "detail_mail_ipv6_mx_aaaa_tech_table_ipv4_address": "IPv4 address"
+        # MAAR je moet de keys van de ENGELSE vertaling gebruiken :)
+        if message_id.endswith("_tech_table"):
+            submessage_titles = dirty_workaround_to_still_get_tech_table_titles_in_english(message_id)
+            submessages = entry.msgstr.split("|")
+            for submessage_title, submessage in zip(submessage_titles, submessages):
+                content[f"{message_id}_{_js_safe_msgid(submessage_title)}"] = _json_safe_msgstr(submessage)
 
     return content
 
 
+def dirty_workaround_to_still_get_tech_table_titles_in_english(message_id: str):
+    # this prevents some parameterized calls and even more spagetti. This is the 'lunch' version.
+    # given that all translations will be json in the future, this approach is somewhat 'ok'
+    # always has to be english.
+    raw_content: bytes = get_locale_content("en")
+    # with lru cache it's fast enough :)
+    structured_content = load_as_po_file(raw_content)
+
+    for entry in structured_content:
+        # print(entry.msgid)
+        if _js_safe_msgid(entry.msgid) == message_id:
+            return entry.msgstr.split("|")
+
+    return []
+
+
 def get_po_as_dictionary(file):
     structured_content = polib.pofile(file)
     po_file_as_dictionary = {}
@@ -198,34 +189,17 @@ def get_po_as_dictionary_v2(language="en"):
         ) from error
 
 
-def _vue_format_start() -> str:
-    return """const internet_nl_messages = {
-"""
-
-
-def _vue_format_locale_start(locale) -> str:
-    return f"""    {locale}: {{
-        internet_nl: {{
-"""
-
-
-def _vue_format_locale_end() -> str:
-    return """        },
-    },
-"""
-
-
-def _vue_format_end() -> str:
-    return """};"""
-
-
 def _js_safe_msgid(text):
     return slugify(text).replace("-", "_")
 
 
 def _js_safe_msgstr(msgstr):
     # a poor mans escape for single quotes.
     msgstr = msgstr.replace("'", "\\'")
+    return _json_safe_msgstr(msgstr)
+
+
+def _json_safe_msgstr(msgstr):
     html = markdown.markdown(msgstr)
     one_line_html = html.replace("\n", "")
 
@@ -258,7 +232,7 @@ def store_vue_i18n_file(filename: str, content: str) -> None:
     :param content:
     :return:
     """
-    with open(f"{VUE_I18N_OUTPUT_PATH}{filename}.js", "w", encoding="UTF-8") as file:
+    with open(f"{VUE_I18N_OUTPUT_PATH}{filename}", "w", encoding="UTF-8") as file:
         file.write(content)
 
 

dashboard/internet_nl_dashboard/management/commands/connect_superusers.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+import logging
+
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+
+from dashboard.internet_nl_dashboard.models import Account, DashboardUser
+
+log = logging.getLogger(__package__)
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        superusers = User.objects.all().filter(is_superuser=True)
+
+        for user in superusers:
+            if not DashboardUser.objects.filter(user=user).first():
+                DashboardUser.objects.create(
+                    user=user,
+                    account=Account.objects.all().first(),  # should always exist
+                    mail_preferred_language="en",
+                    mail_send_mail_after_scan_finished=False,
+                    mail_after_mail_unsubscribe_code="",
+                )
+                print(f"Added DashboardUser for superuser {user}")
+        print("Done")

dashboard/internet_nl_dashboard/scanners/scan_internet_nl_per_account.py

@@ -152,7 +152,7 @@ def check_running_dashboard_scans(**kwargs) -> Task:
                 .only("id")
             )
 
-        log.debug(f"Checking the state of scan {scans}.")
+        log.debug(f"Checking the state of scan {[scan.id for scan in scans]}.")
         tasks = [progress_running_scan(scan.id) for scan in scans]
 
         # All transactional state stuff is done now, so remove the lock