From e6763b2294500df8ae29800be5843d0930f92b04 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Mon, 5 Feb 2024 13:12:26 +0100 Subject: [PATCH 01/20] Fix invocation of certbot for certificate renewal --- docker/webserver/certbot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/webserver/certbot.sh b/docker/webserver/certbot.sh index 866c8dbda..d6f393996 100755 --- a/docker/webserver/certbot.sh +++ b/docker/webserver/certbot.sh @@ -105,4 +105,4 @@ configure_letsencrypt() { # check certificates for renewal twice a day, make sure the schedule is a moving window so we # don't accidentally fall in line with the busiest time (eg: 00:00) and get errors due to ACME # servers being overloaded at that moment -while sleep 11h; do certbot renew --post-hook "nginx -s reload"; done& +while sleep 11h; do /opt/certbot/bin/certbot renew --post-hook "nginx -s reload"; done& From 9a5c9057a0d490b8b9f2267da645c853d82bed71 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Thu, 8 Feb 2024 15:37:58 +0100 Subject: [PATCH 02/20] Remove old testing from github actions This periodically breaks and is entirely obsolete with the new docker builds working well. --- .github/workflows/pull_request.yml | 101 ----------------------------- 1 file changed, 101 deletions(-) delete mode 100644 .github/workflows/pull_request.yml diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml deleted file mode 100644 index de7452715..000000000 --- a/.github/workflows/pull_request.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Internet.nl - -on: - pull_request: - push: - branches: - - main - - release/* - -jobs: - build: - runs-on: ubuntu-20.04 - timeout-minutes: 15 - env: - GITHUB_ACTIONS: True - ENABLE_BATCH: True - SECRET_KEY: Github - DEBUG: True - DB_NAME: internetnl - DB_USER: internetnluser - DB_PASSWORD: internetnluser - strategy: - matrix: - # Support Matrix from python.org/downloads: - # 3.7, 2023-06-27 - # 3.8, 2024-10 - # 3.9, 2025-10 - # 3.10 2026-10 - python-version: [ "3.10" ] - continue-on-error: false - - services: - rabbitmq: - image: rabbitmq:3-management - ports: - - 15672:15672 - - 5672:5672 - postgres: - image: postgres:12.5 - env: - POSTGRES_DB: internetnl - POSTGRES_USER: internetnluser - POSTGRES_PASSWORD: internetnluser - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 2 - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Run Redis 6 - uses: supercharge/redis-github-action@1.4.0 - with: - redis-version: 6 - - name: Install needed packages for build and run - run: | - sudo apt update - sudo apt-get update - sudo apt-get -y install libevent-dev libssl-dev libffi-dev python-dev swig libhiredis-dev - - name: Check out cache (only if match on OS, Python version, requirements, Makefile and workflow file) - uses: actions/cache@v3 - id: cache-venv - with: - # Prevent re-building the venv when all requirements.txts stays the same. - path: | - ./.venv/ - ./unbound/ - ./_unbound/ - ./nassl_freebsd/ - key: ${{ runner.os }}-${{ matrix.python-version }}-v5-venv-and-deps-${{ hashFiles('**/requirements*.txt', 'Makefile', '.github/workflows/*') }} - # the venv and all (slow) custom dependencies is only built when there was no cache hit. - - name: Make venv (if not from cache) - run: make venv - if: ${{ steps.cache-venv.outputs.cache-hit != 'true' }} - - name: Make nassl (if not from cache) - run: make nassl - if: ${{ steps.cache-venv.outputs.cache-hit != 'true' }} - - name: Make unbound (if not from cache) - run: make unbound-${{ matrix.python-version }}-github - if: ${{ steps.cache-venv.outputs.cache-hit != 'true' }} - - name: Check linting - run: make check - env: - GITHUB_ACTIONS: True - - name: Run tests - run: make old-test - env: - GITHUB_ACTIONS: True - ENABLE_BATCH: True - SECRET_KEY: Github - DEBUG: True - DB_NAME: internetnl - DB_USER: internetnluser - DB_PASSWORD: internetnluser - - name: Verify if Build API documentation command works (needs /static/ dir to exist) - run: | - mkdir -p static - make manage api_generate_doc From 4654a40131e5c9de6f14ba11d5cb4171cfd5a5b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 22:51:57 +0000 Subject: [PATCH 03/20] Bump django from 3.2.23 to 3.2.24 Bumps [django](https://github.com/django/django) from 3.2.23 to 3.2.24. - [Commits](https://github.com/django/django/compare/3.2.23...3.2.24) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3f41f304e..071bdf121 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ colorlog==6.7.0 # via -r requirements.in cryptography==38.0.4 # via -r requirements.in -django==3.2.23 +django==3.2.24 # via # -r requirements.in # django-bleach From 118b8b87fa9d1b6f0898bf16de9027012d403347 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Thu, 8 Feb 2024 17:34:56 +0100 Subject: [PATCH 04/20] Fix #1267 - Allow 4xx and 5xx in IPv4/6 similarity with notice Adds a requirement for a label `detail web ipv6 web-ipv46 verdict notice-status-code` --- checks/categories.py | 4 ++++ checks/tasks/ipv6.py | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/checks/categories.py b/checks/categories.py index 917b10fea..d4a2829b0 100644 --- a/checks/categories.py +++ b/checks/categories.py @@ -538,6 +538,10 @@ def result_bad(self): self._status(STATUS_FAIL) self.verdict = "detail web ipv6 web-ipv46 verdict bad" + def result_notice_status_code(self): + self._status(STATUS_NOTICE) + self.verdict = "detail web ipv6 web-ipv46 verdict notice-status-code" + def result_notice(self): self._status(STATUS_NOTICE) self.verdict = "detail web ipv6 web-ipv46 verdict notice" diff --git a/checks/tasks/ipv6.py b/checks/tasks/ipv6.py index 75a90d1a8..95d3d5433 100644 --- a/checks/tasks/ipv6.py +++ b/checks/tasks/ipv6.py @@ -262,6 +262,7 @@ def callback(results, addr, parent, parent_name, category): elif testname == "web": parent.web_simhash_score = result.get("simhash_score") web_simhash_distance = result.get("simhash_distance") + simhash_status_code = result.get("simhash_status_code") parent.web_simhash_distance = web_simhash_distance parent.web_score = result.get("score") @@ -276,8 +277,11 @@ def callback(results, addr, parent, parent_name, category): category.subtests["web_reach"].result_good() if len(good_conn) > 0: - if web_simhash_distance <= settings.SIMHASH_MAX and web_simhash_distance >= 0: - category.subtests["web_ipv46"].result_good() + if settings.SIMHASH_MAX >= web_simhash_distance >= 0: + if 200 <= simhash_status_code <= 400: + category.subtests["web_ipv46"].result_good() + else: + category.subtests["web_ipv46"].result_notice_status_code() elif web_simhash_distance == SIMHASH_NOT_CALCULABLE: category.subtests["web_ipv46"].result_bad() elif web_simhash_distance >= 0: @@ -584,18 +588,23 @@ def strip_irrelevant_html(html): # Could not connect on given port, try another port. # If we managed to connect on IPv4 however, fail the test. if v4_response: - return simhash_score, distance + return simhash_score, distance, None - if not v4_response: + if v4_response is None: # FAIL: Could not establish a connection on both addresses. - return simhash_score, distance + return simhash_score, distance, v6_response.status_code + + # Regardless of content, status code must be identical (#1267) + if v4_response.status_code != v6_response.status_code: + # FAIL: Could not establish a connection on both addresses. + return scoring.WEB_IPV6_WS_SIMHASH_OK, distance, v6_response.status_code try: html_v4 = response_content_chunk(v4_response, SIMHASH_MAX_RESPONSE_SIZE) html_v6 = response_content_chunk(v6_response, SIMHASH_MAX_RESPONSE_SIZE) except (OSError, IOError) as exc: log.debug("simhash encountered exception while reading response: {exc}", exc_info=exc) - return simhash_score, distance + return simhash_score, distance, v6_response.status_code for html, response in (html_v4, v4_response), (html_v6, v6_response): content_length = response.headers.get("content-length", "") @@ -609,7 +618,7 @@ def strip_irrelevant_html(html): if distance <= settings.SIMHASH_MAX: simhash_score = scoring.WEB_IPV6_WS_SIMHASH_OK - return simhash_score, distance + return simhash_score, distance, v6_response.status_code def do_web(self, url, *args, **kwargs): @@ -618,6 +627,8 @@ def do_web(self, url, *args, **kwargs): domain = [] simhash_score = scoring.WEB_IPV6_WS_SIMHASH_FAIL simhash_distance = SIMHASH_NOT_CALCULABLE + simhash_status_code = None + score = scoring.WEB_IPV6_WS_CONN_FAIL domain = get_domain_results( @@ -641,7 +652,7 @@ def do_web(self, url, *args, **kwargs): simhash_score = scoring.WEB_IPV6_WS_SIMHASH_OK simhash_distance = -1 elif len(v6_good) > 0 and len(v4_good) > 0 and len(v6_conn_diff) == 0: - simhash_score, simhash_distance = simhash(url, task=self) + simhash_score, simhash_distance, simhash_status_code = simhash(url, task=self) except SoftTimeLimitExceeded: log.debug("Soft time limit exceeded.") @@ -656,4 +667,13 @@ def do_web(self, url, *args, **kwargs): score=scoring.WEB_IPV6_WS_CONN_FAIL, ) - return ("web", dict(domains=[domain], simhash_score=simhash_score, simhash_distance=simhash_distance, score=score)) + return ( + "web", + dict( + domains=[domain], + simhash_score=simhash_score, + simhash_distance=simhash_distance, + simhash_status_code=simhash_status_code, + score=score, + ), + ) From 03751a8e611355f4adccbddbbb4dc2532802bc70 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Thu, 8 Feb 2024 14:34:05 +0100 Subject: [PATCH 05/20] Update unbound to 1.19.0+internetnl https://github.com/internetstandards/unbound/pull/3 https://github.com/internetstandards/unbound/pull/4 --- docker/Dockerfile | 5 ++++- vendor/unbound | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 549d83ac7..934848178 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -28,7 +28,10 @@ RUN apt update && \ # TODO: swig3.0 hard requirements? get from oldstable swig \ # nassl dependencies - python3-setuptools + python3-setuptools \ + # unbound dependencies + flex \ + bison FROM build-deps as build-unbound ARG PYTHON_VERSION diff --git a/vendor/unbound b/vendor/unbound index 4a15da644..09b62cc17 160000 --- a/vendor/unbound +++ b/vendor/unbound @@ -1 +1 @@ -Subproject commit 4a15da644965778e548b52c6eea3568fe0fa0c31 +Subproject commit 09b62cc170f64a4cdda25b8850b69a6e98a20840 From f06996db13ee2b55f7ee5bebd2c40d2ce88756fd Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Tue, 13 Feb 2024 15:09:48 +0100 Subject: [PATCH 06/20] Update unbound to 1.19.1-internetnl Fixes CVE-2023-50387 and CVE-2023-50868 --- vendor/unbound | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/unbound b/vendor/unbound index 09b62cc17..b78aef780 160000 --- a/vendor/unbound +++ b/vendor/unbound @@ -1 +1 @@ -Subproject commit 09b62cc170f64a4cdda25b8850b69a6e98a20840 +Subproject commit b78aef78049721196f46e6c34ff63c15d436338c From 57f69f73010df552bfd344515ed87f21a25be2ff Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Tue, 13 Feb 2024 15:38:00 +0100 Subject: [PATCH 07/20] Add changelog for 1.8.4 --- Changelog.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Changelog.md b/Changelog.md index 88532816b..e1d65cf07 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,14 @@ # Change Log +## 1.8.4 + +Release 1.8.4: + +- Updates unbound to 1.19.1-internetnl to fix CVE-2023-50387 and CVE-2023-50868. +- Restricts HTTPS redirects to the same domain, no longer allowing directions to a subdomain first (#1208). +- Updates a number of other dependencies. +- Fixes an issue where certbot renewals were not correctly run. +- ## 1.8.3 Release 1.8.3 fixes an issue where HSTS and CSP headers were missing from he www-subdomain of the main domain (#1210, #1211). From fa9e29f391844283fe7229e2f621315147eb79c3 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Mon, 12 Feb 2024 16:51:01 +0100 Subject: [PATCH 08/20] Ref #1267 - Remove metadata and manual management for BatchUser --- Makefile | 7 - checks/migrations/0015_auto_20240212_1616.py | 24 ++++ checks/models.py | 10 +- integration_tests/integration/test_batch.py | 10 -- interface/apps.py | 20 --- interface/batch/util.py | 28 +--- interface/management/commands/api_users.py | 141 ------------------- interface/test/batch/test_batch.py | 2 +- 8 files changed, 33 insertions(+), 209 deletions(-) create mode 100644 checks/migrations/0015_auto_20240212_1616.py delete mode 100644 interface/management/commands/api_users.py diff --git a/Makefile b/Makefile index f856cd534..a0c481793 100644 --- a/Makefile +++ b/Makefile @@ -651,12 +651,5 @@ documentation-images: ${images} documentation/images/%.png: documentation/images/%.py | ${nwdiag} docker run -it --rm -v "$${PWD}/$(@D)/":/$(@D) -w /$(@D) gtramontina/diagrams:0.23.1 $( Tuple[Optional[Exception], Dict]: data = {} try: diff --git a/interface/management/commands/api_users.py b/interface/management/commands/api_users.py deleted file mode 100644 index ced02844a..000000000 --- a/interface/management/commands/api_users.py +++ /dev/null @@ -1,141 +0,0 @@ -import json -from enum import Enum, auto - -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction - -from checks.models import BatchUser -from interface.batch.util import create_batch_user - - -class FormatType(Enum): - cli = auto() - csv = auto() - json = auto() - - -class Command(BaseCommand): - help = "Creates an API user in the database." - - def info(self, text): - if self.v_level: - self.stdout.write(f"{text}") - - def debug(self, text): - if self.v_level > 1: - self.stdout.write(f"{text}") - - def output_cli(self, headers, users): - res = [headers] + users - max_length = [0 for i in range(len(res[0]))] - for i in range(len(max_length)): - max_length[i] = max(map(lambda x: len(x[i]), res)) - for s in res: - lines = ["{:<{}}".format(x, max_length[i]) for i, x in enumerate(s)] - self.info(" ".join(lines)) - - def output_csv(self, headers, users): - res = [headers] + users - for s in res: - self.info(",".join(s)) - - def output_json(self, headers, users): - res = [] - for user in users: - res.append({headers[i].lower(): x for i, x in enumerate(user)}) - self.info(json.dumps(res)) - - def list_users(self): - users = [ - (f"{u.id}", u.username, u.name, u.organization, u.email) for u in BatchUser.objects.order_by("id").all() - ] - if not users: - raise CommandError("No registered API users.") - headers = ("ID", "Username", "Name", "Organization", "Email") - if self.format == FormatType.cli.name: - self.output_cli(headers, users) - elif self.format == FormatType.csv.name: - self.output_csv(headers, users) - elif self.format == FormatType.json.name: - self.output_json(headers, users) - - def register_user(self): - try: - BatchUser.objects.get(username=self.username) - raise CommandError(f"The user '{self.username}' already exists. Try " f"`api_users update` instead.") - except BatchUser.DoesNotExist: - create_batch_user(self.username, self.name, self.organization, self.email) - - def update_user(self): - try: - user = BatchUser.objects.get(username=self.username) - user.name = self.name - user.organization = self.organization - user.email = self.email - user.save() - except BatchUser.DoesNotExist: - raise CommandError(f"The user '{self.username}' does not exist. Try " f"`api_users register` instead.") - - @transaction.atomic - def remove_user(self): - try: - user = BatchUser.objects.get(username=self.username) - if not self.yes: - cont = False - res = input( - f"This will delete user {self.username} along with all " - f"the submitted batch requests, batch results and " - f"generated reports.\n" - f"Do you want to continue? [y/N]\n" - ) - if res.lower() == "y": - cont = True - if not cont: - return - user.delete_related_data(delete_self=True) - - except BatchUser.DoesNotExist: - raise CommandError(f"The user '{self.username}' does not exist.") - - def add_arguments(self, parser): - subparsers = parser.add_subparsers( - title="sub-commands", description="Available sub-commands", dest="sub-command", required=True - ) - - list_parser = subparsers.add_parser("list", help="List the registered API users") - list_parser.add_argument( - "-f", - "--format", - default=FormatType.cli.name, - choices=[FormatType.cli.name, FormatType.csv.name, FormatType.json.name], - ) - list_parser.set_defaults(func=self.list_users) - - register_parser = subparsers.add_parser("register", help="Register a new API user") - register_parser.set_defaults(func=self.register_user) - register_parser.add_argument("-u", "--username", required=True) - register_parser.add_argument("-n", "--name", required=True) - register_parser.add_argument("-o", "--organization", required=True) - register_parser.add_argument("-e", "--email", required=True) - - update_parser = subparsers.add_parser("update", help="Update an existing API user") - update_parser.set_defaults(func=self.update_user) - update_parser.add_argument("-u", "--username", required=True) - update_parser.add_argument("-n", "--name", required=True) - update_parser.add_argument("-o", "--organization", required=True) - update_parser.add_argument("-e", "--email", required=True) - - remove_parser = subparsers.add_parser("remove", help="Remove an existing API user") - remove_parser.set_defaults(func=self.remove_user) - remove_parser.add_argument("-u", "--username", required=True) - remove_parser.add_argument("-y", "--yes", action="store_true", help="Assume yes to removal prompt") - - def handle(self, *args, **options): - self.v_level = options["verbosity"] - self.format = options.get("format") - self.username = options.get("username") - self.name = options.get("name") - self.organization = options.get("organization") - self.email = options.get("email") - self.yes = options.get("yes") - options["func"]() diff --git a/interface/test/batch/test_batch.py b/interface/test/batch/test_batch.py index a0b51d163..9da68eeb3 100644 --- a/interface/test/batch/test_batch.py +++ b/interface/test/batch/test_batch.py @@ -22,7 +22,7 @@ def test_register_batch_request(db): # ah, and now this nonsense where there are two users on the same database at the same time because we have # a test-worker. - test_user = BatchUser.objects.create(name="test_user") + test_user = BatchUser.objects.create(username="test_user") random_name = str(uuid.uuid4()) From 63541c37df5285dd5aecedd575544e07f87d34b5 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Mon, 12 Feb 2024 22:36:44 +0100 Subject: [PATCH 09/20] Ref #1267 - Move batch auth to separate volume --- docker/defaults.env | 8 +------- docker/develop.env | 3 --- docker/docker-compose.yml | 7 ++----- docker/test.env | 7 +------ docker/webserver/generate_htpasswd.sh | 2 -- docker/webserver/nginx_templates/app.conf.template | 4 ++-- integration_tests/integration/test_batch.py | 2 +- 7 files changed, 7 insertions(+), 26 deletions(-) diff --git a/docker/defaults.env b/docker/defaults.env index fb050370e..bdd8b4f64 100644 --- a/docker/defaults.env +++ b/docker/defaults.env @@ -57,11 +57,6 @@ HOSTERS_HOF_URL= # manual HoF pages to include MANUAL_HOF_PAGES= -# comma separated user:password pairs for Batch API authentication -# eg: BATCH_AUTH=user1:welkom01,user2:hunter2 -BATCH_AUTH= -BATCH_USER_DEFAULT_ORGANISATION=n/a -BATCH_USER_DEFAULT_EMAIL_DOMAIN=example.com # comma separated user:password pairs for /grafana and /prometheus metrics endpoints MONITORING_AUTH= # comma separated user:password pairs for side wide authentication @@ -72,8 +67,7 @@ ALLOW_LIST= # comma separated user:htpasswd_encrypted pairs, same AUTH above, except password must already be encrypted # please not that the value needs to be enclosed by single quotes to prevent interpolation of the dollar signs -# eg: BATCH_AUTH='test1:$apr1$wGM8gxBe$DxGwifTGWZJ7nftK7LzFt/,user2:$apr1$BoZzsbb/$2NgfYCfF9lxmGrfSqsZKc/' -BATCH_AUTH_RAW= +# eg: BASIC_AUTH_RAW='test1:$apr1$wGM8gxBe$DxGwifTGWZJ7nftK7LzFt/,user2:$apr1$BoZzsbb/$2NgfYCfF9lxmGrfSqsZKc/' MONITORING_AUTH_RAW= BASIC_AUTH_RAW= diff --git a/docker/develop.env b/docker/develop.env index f410cc98f..2870bb78e 100644 --- a/docker/develop.env +++ b/docker/develop.env @@ -12,9 +12,6 @@ COMPOSE_PROJECT_NAME=internetnl-develop ENABLE_BATCH=True # use easy user/passwords for authenticated endpoints -BATCH_AUTH=test:test -BATCH_USER_DEFAULT_ORGANISATION=n/a -BATCH_USER_DEFAULT_EMAIL_DOMAIN=example.com MONITORING_AUTH=test:test LETSENCRYPT_STAGING=1 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ae7945b00..492d6a520 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,10 +26,8 @@ services: environment: - INTERNETNL_DOMAINNAME - IPV6_TEST_ADDR - - BATCH_AUTH - MONITORING_AUTH - BASIC_AUTH - - BATCH_AUTH_RAW - MONITORING_AUTH_RAW - BASIC_AUTH_RAW - ALLOW_LIST @@ -55,6 +53,7 @@ services: volumes: # persist certbot configuration between restarts - certbot-config:/etc/letsencrypt + - htpasswd-files:/etc/nginx/htpasswd/external healthcheck: test: ["CMD", "service", "nginx", "status"] @@ -143,9 +142,6 @@ services: - SMTP_EHLO_DOMAIN - IPV6_TEST_ADDR - CSP_DEFAULT_SRC - - BATCH_AUTH - - BATCH_USER_DEFAULT_ORGANISATION - - BATCH_USER_DEFAULT_EMAIL_DOMAIN - IPV4_IP_RESOLVER_INTERNAL_VALIDATING - IPV4_IP_RESOLVER_INTERNAL_PERMISSIVE - SENTRY_DSN @@ -825,6 +821,7 @@ volumes: routinator: {} batch_results: {} certbot-config: {} + htpasswd-files: {} unbound-zones: {} # permanent storage for Prometheus metrics prometheus-data: {} diff --git a/docker/test.env b/docker/test.env index 86018d894..76c6a24fc 100644 --- a/docker/test.env +++ b/docker/test.env @@ -14,7 +14,7 @@ ENABLE_BATCH=True # disable as it messes with batch jobs ENABLE_HOF=True -# enable manual hof entry +# enable manual hof entryB HOSTERS_HOF_URL=http://static.test/static/hosters.yaml MANUAL_HOF_PAGES=hosters @@ -54,12 +54,7 @@ IPV6_IP_TEST_TARGET_PUBLIC=fd00:43:1::51 IPV6_IP_TEST_TARGET_MAIL_PUBLIC=fd00:43:1::52 # use easy user/passwords for authenticated endpoints -BATCH_AUTH=test:test -BATCH_USER_DEFAULT_ORGANISATION=n/a -BATCH_USER_DEFAULT_EMAIL_DOMAIN=example.com MONITORING_AUTH=test:test - -BATCH_AUTH_RAW='test_raw:$apr1$6YuDyduL$706z.FPTe5c09R767N3W90' MONITORING_AUTH_RAW='test_raw:$apr1$6YuDyduL$706z.FPTe5c09R767N3W90' # use lower scheduler interval to speed up batch API tests diff --git a/docker/webserver/generate_htpasswd.sh b/docker/webserver/generate_htpasswd.sh index 3a1c6482d..413867e41 100755 --- a/docker/webserver/generate_htpasswd.sh +++ b/docker/webserver/generate_htpasswd.sh @@ -1,9 +1,7 @@ # generate htaccess files from environment variables. Add a user:password for every comma separated pair -echo $BATCH_AUTH|tr ',' '\n'|tr ':' ' '| xargs --max-args=2 --no-run-if-empty htpasswd -b /etc/nginx/htpasswd/batch_api.htpasswd echo $MONITORING_AUTH|tr ',' '\n'|tr ':' ' '| xargs --max-args=2 --no-run-if-empty htpasswd -b /etc/nginx/htpasswd/monitoring.htpasswd echo $BASIC_AUTH|tr ',' '\n'|tr ':' ' '| xargs --max-args=2 --no-run-if-empty htpasswd -b /etc/nginx/htpasswd/basic_auth.htpasswd # append raw entries to htpasswd file -echo $BATCH_AUTH_RAW|tr ',' '\n' >> /etc/nginx/htpasswd/batch_api.htpasswd echo $MONITORING_AUTH_RAW|tr ',' '\n' >> /etc/nginx/htpasswd/monitoring.htpasswd echo $BASIC_AUTH_RAW|tr ',' '\n' >> /etc/nginx/htpasswd/basic_auth.htpasswd diff --git a/docker/webserver/nginx_templates/app.conf.template b/docker/webserver/nginx_templates/app.conf.template index 45ce66244..ce8c23ce8 100644 --- a/docker/webserver/nginx_templates/app.conf.template +++ b/docker/webserver/nginx_templates/app.conf.template @@ -247,8 +247,8 @@ server { # batch API, requires authentication and passes basic auth user to Django App via headers location /api/batch/v2 { - auth_basic "Please enter your username and password"; - auth_basic_user_file /etc/nginx/htpasswd/batch_api.htpasswd; + auth_basic "Please enter your batch username and password"; + auth_basic_user_file /etc/nginx/htpasswd/external/batch_api.htpasswd; # pass logged in user to Django proxy_set_header REMOTE-USER $remote_user; diff --git a/integration_tests/integration/test_batch.py b/integration_tests/integration/test_batch.py index bd5efcf20..909d5b6d4 100644 --- a/integration_tests/integration/test_batch.py +++ b/integration_tests/integration/test_batch.py @@ -44,7 +44,7 @@ def register_test_user(unique_id): # create test used in Apache2 password file command = ( 'docker compose --ansi=never --project-name "internetnl-test"' - f" exec webserver htpasswd -b /etc/nginx/htpasswd/batch_api.htpasswd {username} {username}" + f" exec webserver htpasswd -c -b /etc/nginx/htpasswd/external/batch_api.htpasswd {username} {username}" ) subprocess.check_call(command, shell=True, universal_newlines=True) From 5b61b0920066ee175d43e387f95badbfee8cad95 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Wed, 14 Feb 2024 16:35:57 +0100 Subject: [PATCH 10/20] Ref #1267 - Add htpasswd management scripts for batch API --- docker/batch_user.sh | 4 ++++ docker/webserver.Dockerfile | 2 ++ docker/webserver/batch_user_inner.sh | 20 ++++++++++++++++++++ documentation/Docker-deployment-batch.md | 13 ++++++++++--- 4 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 docker/batch_user.sh create mode 100755 docker/webserver/batch_user_inner.sh diff --git a/docker/batch_user.sh b/docker/batch_user.sh new file mode 100644 index 000000000..67cf504fb --- /dev/null +++ b/docker/batch_user.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +# Small wrapper around batch user mgmt script shipped in webserver image +# For both convenience, and to have a suitable command to put in sudo +/usr/bin/docker compose --env-file=/opt/Internet.nl/docker/defaults.env --env-file=/opt/Internet.nl/docker/host.env --env-file=/opt/Internet.nl/docker/local.env exec -ti webserver /batch_user_inner.sh "$1" "$2" diff --git a/docker/webserver.Dockerfile b/docker/webserver.Dockerfile index 6669f50d4..6b801b973 100644 --- a/docker/webserver.Dockerfile +++ b/docker/webserver.Dockerfile @@ -27,6 +27,8 @@ COPY docker/webserver/generate_htpasswd.sh /docker-entrypoint.d/generate_htpassw COPY docker/webserver/tls_init.sh /docker-entrypoint.d/tls_init.sh COPY docker/webserver/authentication.sh /docker-entrypoint.d/authentication.sh +COPY docker/webserver/batch_user_inner.sh /batch_user_inner.sh + RUN mkdir -p /var/www/internet.nl/ COPY robots.txt /var/www/internet.nl/ diff --git a/docker/webserver/batch_user_inner.sh b/docker/webserver/batch_user_inner.sh new file mode 100755 index 000000000..0b2d3f2f5 --- /dev/null +++ b/docker/webserver/batch_user_inner.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env sh + +HTPASSWD_FILE="/etc/nginx/htpasswd/external/batch_api.htpasswd" + +if [ ! -f "$HTPASSWD_FILE" ]; then + touch "$HTPASSWD_FILE" +fi + +if [ "$1" = "add_update" ]; then + /usr/bin/htpasswd -B "$HTPASSWD_FILE" "$2" + /usr/sbin/nginx -s reload +elif [ "$1" = "remove" ]; then + /usr/bin/htpasswd -D "$HTPASSWD_FILE" "$2" + /usr/sbin/nginx -s reload +elif [ "$1" = "verify" ]; then + /usr/bin/htpasswd -v "$HTPASSWD_FILE" "$2" +else + echo "Usage: $0 " + exit 1 +fi diff --git a/documentation/Docker-deployment-batch.md b/documentation/Docker-deployment-batch.md index 4992b3eda..5f4af25f8 100644 --- a/documentation/Docker-deployment-batch.md +++ b/documentation/Docker-deployment-batch.md @@ -71,6 +71,8 @@ Run the following commands to install the files in the expected location: curl -sSfO --output-dir docker https://raw.githubusercontent.com/internetstandards/Internet.nl/${RELEASE}/docker/defaults.env && \ curl -sSfO --output-dir docker https://raw.githubusercontent.com/internetstandards/Internet.nl/${RELEASE}/docker/host-dist.env && \ curl -sSfO --output-dir docker https://raw.githubusercontent.com/internetstandards/Internet.nl/${RELEASE}/docker/docker-compose.yml && \ + curl -sSfO https://raw.githubusercontent.com/internetstandards/Internet.nl/${RELEASE}/docker/batch_user.sh && \ + chmod 755 batch_user.sh && \ touch docker/local.env To create the `docker/host.env` configuration file, the following input is required: @@ -98,7 +100,6 @@ Batch installations require the following settings: - `ENABLE_BATCH`: Must be set to `True`, to enable batch API - `ENABLE_HOF`: Must be set to `False`, to disable Hall of Fame processing -- `BATCH_AUTH`: Must be a comma separated list of `user:password` pairs which are allowed to access the Batch API. And optionally: @@ -110,8 +111,6 @@ For example: cat >> docker/local.env < Date: Wed, 31 Jan 2024 16:29:18 +0100 Subject: [PATCH 11/20] Changed to common crawler User-Agent Fixes https://github.com/internetstandards/Internet.nl/issues/1224 --- checks/http_client.py | 3 ++- checks/tasks/tls_connection.py | 2 +- docker/defaults.env | 4 ++++ docker/docker-compose.yml | 5 +++++ internetnl/settings.py | 9 +++++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/checks/http_client.py b/checks/http_client.py index a921a2d3e..edb1b959b 100644 --- a/checks/http_client.py +++ b/checks/http_client.py @@ -12,6 +12,7 @@ from checks.tasks import SetupUnboundContext from checks.tasks.tls_connection import DEFAULT_TIMEOUT from checks.tasks.tls_connection_exceptions import NoIpError +from django.conf import settings from interface.views.shared import ub_resolve_with_timeout from internetnl import log @@ -59,7 +60,7 @@ def http_get( if not headers: headers = {} - headers["User-Agent"] = "internetnl/1.0" + headers["User-Agent"] = settings.USER_AGENT if not session: session = requests.session() diff --git a/checks/tasks/tls_connection.py b/checks/tasks/tls_connection.py index 20c6c9301..7a562cf2d 100644 --- a/checks/tasks/tls_connection.py +++ b/checks/tasks/tls_connection.py @@ -660,7 +660,7 @@ def http_fetch( ) conn.putrequest(http_method, path, skip_accept_encoding=True) # Must specify User-Agent. Some webservers return error otherwise. - conn.putheader("User-Agent", "internetnl/1.0") + conn.putheader("User-Agent", settings.USER_AGENT) # Set headers (eg for HTTP-compression test) for k, v in put_headers: conn.putheader(k, v) diff --git a/docker/defaults.env b/docker/defaults.env index bdd8b4f64..9dd37d739 100644 --- a/docker/defaults.env +++ b/docker/defaults.env @@ -21,6 +21,10 @@ CONN_TEST_DOMAIN= # EHLO domain used in the "Test your email" test SMTP_EHLO_DOMAIN=internet.nl +# used in User-Agent as contact hint for website administrators +# if not set it defaults to https://{INTERNETNL_DOMAINNAME}/ +# USER_AGENT_URL= + # use letsencrypt staging server, set to 0 for production environments LETSENCRYPT_STAGING=0 # email address to use for letsencrypt contact diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 492d6a520..d5b5f4aee 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -153,6 +153,8 @@ services: - MANUAL_HOF_PAGES - CLIENT_RATE_LIMIT - INTERNETNL_BRANDING + - INTERNETNL_DOMAINNAME + - USER_AGENT_URL healthcheck: test: ["CMD", "nc", "-z", "127.0.0.1", "8080"] interval: $HEALTHCHECK_INTERVAL @@ -285,6 +287,9 @@ services: - ROUTINATOR_URL - CONN_TEST_DOMAIN - SMTP_EHLO_DOMAIN + - INTERNETNL_BRANDING + - INTERNETNL_DOMAINNAME + - USER_AGENT_URL - IPV4_IP_RESOLVER_INTERNAL_VALIDATING - IPV4_IP_RESOLVER_INTERNAL_PERMISSIVE - SENTRY_DSN diff --git a/internetnl/settings.py b/internetnl/settings.py index b8d80667b..1fe190029 100644 --- a/internetnl/settings.py +++ b/internetnl/settings.py @@ -664,3 +664,12 @@ IPV4_IP_RESOLVER_INTERNAL_VALIDATING = getenv("IPV4_IP_RESOLVER_INTERNAL_VALIDATING") INTERNETNL_BRANDING = get_boolean_env("INTERNETNL_BRANDING", False) + +# 'common crawler format' +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent#crawler_and_bot_ua_strings) +USER_AGENT_URL = getenv( + "USER_AGENT_URL", + "https://internet.nl/about/" if INTERNETNL_BRANDING else f"https://{getenv('INTERNETNL_DOMAINNAME')}/", +) + +USER_AGENT = f"Mozilla/5.0 (compatible; internetnl/{VERSION}; +{USER_AGENT_URL})" From f077e64e93ced678c8a31779727a3a93cb07d32d Mon Sep 17 00:00:00 2001 From: "Benjamin W. Broersma" Date: Mon, 12 Feb 2024 20:45:56 +0100 Subject: [PATCH 12/20] Fix certbot subdomains (fixes #1275) --- docker/docker-compose.yml | 1 + docker/webserver/certbot.sh | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d5b5f4aee..dafc3c137 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -36,6 +36,7 @@ services: - IPV4_IP_APP_INTERNAL - IPV4_IP_GRAFANA_INTERNAL - IPV4_IP_PROMETHEUS_INTERNAL + - ENABLE_BATCH - LETSENCRYPT_STAGING - LETSENCRYPT_EMAIL - CERTBOT_SERVER diff --git a/docker/webserver/certbot.sh b/docker/webserver/certbot.sh index d6f393996..4dc56aa62 100755 --- a/docker/webserver/certbot.sh +++ b/docker/webserver/certbot.sh @@ -28,7 +28,10 @@ if [ ! -z $CERTBOT_EAB_HMAC_KEY ]; then fi domain=$INTERNETNL_DOMAINNAME -subdomains="nl.$domain,en.$domain,www.$domain,ipv6.$domain,conn.$domain,en.conn.$domain,nl.conn.$domain,www.conn.$domain" +subdomains="nl.$domain,en.$domain,ipv6.$domain,nl.ipv6.$domain,en.ipv6.$domain" +if [ "$ENABLE_BATCH" != True ]; then + subdomains="$subdomains,www.$domain,conn.$domain,en.conn.$domain,nl.conn.$domain,www.conn.$domain" +fi if [ ! -z $REDIRECT_DOMAINS ];then subdomains="$subdomains,$REDIRECT_DOMAINS" fi From c4a9dae7946e10a744eaad8273ffd540af781a36 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Wed, 15 Nov 2023 20:15:29 +0100 Subject: [PATCH 13/20] Fix nginx config warnings --- .../nginx_templates/app.conf.template | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/docker/webserver/nginx_templates/app.conf.template b/docker/webserver/nginx_templates/app.conf.template index ce8c23ce8..ac607212e 100644 --- a/docker/webserver/nginx_templates/app.conf.template +++ b/docker/webserver/nginx_templates/app.conf.template @@ -28,8 +28,10 @@ ssl_stapling_verify on; # default server for http, primary used for ACME and https redirect server { - listen 80 http2; - listen [::]:80 http2; + listen 80; + listen [::]:80; + + http2 on; server_name ${INTERNETNL_DOMAINNAME} ~(nl|en|www|ipv6)\.${INTERNETNL_DOMAINNAME} ${REDIRECT_DOMAINS_LIST}; @@ -49,8 +51,10 @@ server { # http server for connection test, does not redirect to https server { - listen 80 http2; - listen [::]:80 http2; + listen 80; + listen [::]:80; + + http2 on; server_name ~(conn|(?en\.|nl\.|www\.)conn).${INTERNETNL_DOMAINNAME}; @@ -87,8 +91,10 @@ server { } # http server for connection test XHR requests server { - listen 80 http2; - listen [::]:80 http2; + listen 80; + listen [::]:80; + + http2 on; server_name *.test-ns-signed.${INTERNETNL_DOMAINNAME} @@ -114,8 +120,10 @@ server { # No-www.org Class B compliance, see https://www.no-www.org/faq.php server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; server_name www.${INTERNETNL_DOMAINNAME} ~(nl|en|conn)\.www.${INTERNETNL_DOMAINNAME} ${REDIRECT_DOMAINS_LIST}; @@ -149,8 +157,10 @@ server { # default https server server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; server_name ${INTERNETNL_DOMAINNAME} ~(?nl\.|en\.|www\.|ipv6\.)${INTERNETNL_DOMAINNAME}; @@ -275,8 +285,10 @@ server { # Temporary (1 year) exception for conn. subdomain to disable HSTS and redirect back to HTTP for # clients that accessed the HTTPS version in the past and got a HSTS set of 1 year. server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; server_name conn.${INTERNETNL_DOMAINNAME}; @@ -309,6 +321,8 @@ server { listen 443 ssl default_server; listen [::]:443 ssl default_server; + http2 on; + ssl_reject_handshake on; location / { @@ -321,6 +335,8 @@ server { listen 80 default_server; listen [::]:80 default_server; + http2 on; + server_name _; location / { From 58545297ae87eed4ca9921b50e4ec9cbe1ccd76c Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Tue, 27 Feb 2024 20:17:58 +0100 Subject: [PATCH 14/20] Fix #1204 - Upgrade Django to 4.2.10 --- checks/probes.py | 2 +- interface/batch/util.py | 2 +- interface/conntesturls/ipv4_urls.py | 4 +- interface/conntesturls/ipv6_urls.py | 4 +- interface/conntesturls/resolver_urls.py | 4 +- interface/templates/base-results.html | 4 +- interface/templates/base.html | 10 +- interface/templatetags/translate.py | 2 +- interface/urls.py | 128 ++++++++++++------------ interface/views/__init__.py | 2 +- interface/views/connection.py | 2 +- interface/views/domain.py | 2 +- interface/views/mail.py | 2 +- interface/views/shared.py | 4 +- internetnl/custom_middlewares.py | 2 +- internetnl/urls.py | 6 +- requirements.in | 5 +- requirements.txt | 6 +- 18 files changed, 94 insertions(+), 97 deletions(-) diff --git a/checks/probes.py b/checks/probes.py index bf79111f8..9f64b1856 100644 --- a/checks/probes.py +++ b/checks/probes.py @@ -1,7 +1,7 @@ # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors # SPDX-License-Identifier: Apache-2.0 from django.conf import settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from checks import categories from checks.categories import MailTlsStarttlsExists, WebTlsHttpsExists diff --git a/interface/batch/util.py b/interface/batch/util.py index cc61b7e29..dbc1a2b1a 100644 --- a/interface/batch/util.py +++ b/interface/batch/util.py @@ -195,7 +195,7 @@ def get_user_from_request(request): DB) return the relevant user object from the DB. """ - username = request.META.get("REMOTE_USER") or request.META.get("HTTP_REMOTE_USER") + username = request.META.get("REMOTE_USER") or request.headers.get("remote-user") if not username: username = getattr(settings, "BATCH_TEST_USER", None) user, created = BatchUser.objects.get_or_create(username=username) diff --git a/interface/conntesturls/ipv4_urls.py b/interface/conntesturls/ipv4_urls.py index e4975720e..2a0fe0d53 100644 --- a/interface/conntesturls/ipv4_urls.py +++ b/interface/conntesturls/ipv4_urls.py @@ -1,9 +1,9 @@ # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors # SPDX-License-Identifier: Apache-2.0 -from django.conf.urls import url +from django.urls import path from interface.views.connection import network_ipv4 urlpatterns = [ - url(r"^$", network_ipv4), + path("", network_ipv4), ] diff --git a/interface/conntesturls/ipv6_urls.py b/interface/conntesturls/ipv6_urls.py index 533d20242..2c7cd6b94 100644 --- a/interface/conntesturls/ipv6_urls.py +++ b/interface/conntesturls/ipv6_urls.py @@ -1,9 +1,9 @@ # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors # SPDX-License-Identifier: Apache-2.0 -from django.conf.urls import url +from django.urls import path from interface.views.connection import aaaa_ipv6 urlpatterns = [ - url(r"^$", aaaa_ipv6), + path("", aaaa_ipv6), ] diff --git a/interface/conntesturls/resolver_urls.py b/interface/conntesturls/resolver_urls.py index 5054b0433..8a55484e4 100644 --- a/interface/conntesturls/resolver_urls.py +++ b/interface/conntesturls/resolver_urls.py @@ -1,9 +1,9 @@ # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors # SPDX-License-Identifier: Apache-2.0 -from django.conf.urls import url +from django.urls import path from interface.views.connection import network_resolver urlpatterns = [ - url(r"^$", network_resolver), + path("", network_resolver), ] diff --git a/interface/templates/base-results.html b/interface/templates/base-results.html index 4b89c1d8b..c8c7d022c 100644 --- a/interface/templates/base-results.html +++ b/interface/templates/base-results.html @@ -30,11 +30,11 @@ {% include "string.html" with name="results score-description"%} {% block perfectscore %} - {% ifequal score 100 %} + {% if score == 100 %}
{% include "string.html" with name="results perfect-score" %}
- {% endifequal %} + {% endif %} {% endblock %}
diff --git a/interface/templates/base.html b/interface/templates/base.html index 5de8f53c7..f2940069d 100644 --- a/interface/templates/base.html +++ b/interface/templates/base.html @@ -71,14 +71,14 @@ diff --git a/interface/templatetags/translate.py b/interface/templatetags/translate.py index 6480e2219..b4f8abfdc 100644 --- a/interface/templatetags/translate.py +++ b/interface/templatetags/translate.py @@ -6,7 +6,7 @@ from django import template from django.conf import settings from django.template import Template -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from checks.scoring import ( STATUS_ERROR, diff --git a/interface/urls.py b/interface/urls.py index e7c6f6316..622b91212 100644 --- a/interface/urls.py +++ b/interface/urls.py @@ -1,7 +1,7 @@ # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors # SPDX-License-Identifier: Apache-2.0 from django.conf import settings -from django.conf.urls import url +from django.urls import path, re_path from django.conf.urls.static import static from interface import views @@ -17,65 +17,65 @@ ) urlpatterns = [ - url(r"^$", views.indexpage), - url(r"^statistics/(?P[0-9]{8})/(?P[0-9]{8})/$", stats.statistics), - url(r"^copyright/$", views.copyrightpage), - url(r"^faqs/$", views.faqindexpage), - url(r"^faqs/report/$", views.faqreport, name="faqs_report"), - url(r"^faqs/badges/$", views.faqbadges, name="faqs_badges"), - url(r"^faqs/(?P[a-zA-Z0-9\-]{1,40})/$", views.faqarticlepage), - url(r"^usage/$", views.indexpage), - url(r"^widget-site/$", views.widgetsitepage), - url(r"^widget-mail/$", views.widgetmailpage), - url(r"^halloffame/$", views.hofchampionspage), - url(r"^halloffame/web/$", views.hofwebpage), - url(r"^halloffame/mail/$", views.hofmailpage), - url(r"^test-connection/$", views.testconnectionpage), - url(r"^connection/$", connection.index), - url(r"^(connection|conn)/gettestid/$", connection.gettestid), - url(rf"^(connection|conn)/finished/{regex_testid}$", connection.finished), - url(rf"^(connection|conn)/addr-test/{regex_testid}/$", connection.addr_ipv6), - url(rf"^(connection|conn)/{regex_testid}/results$", connection.results), - url(r"^test-site/$", views.testsitepage), - url(r"^(domain|site)/$", domain.index), - url(rf"^(domain|site)/{regex_dname}/$", domain.siteprocess), - url(rf"^(domain|site)/probes/{regex_dname}/$", domain.siteprobesstatus), - url(rf"^(domain|site)/(?P(ipv6|tls|dnssec|appsecpriv))/{regex_dname}/$", domain.siteprobeview), - url(rf"^(domain|site)/{regex_dname}/results$", domain.resultscurrent), - url(r"^(domain|site)/(?P.*)/(?P[0-9]+)/$", domain.resultsstored, name="webtest_results"), + path("", views.indexpage), + re_path(r"^statistics/(?P[0-9]{8})/(?P[0-9]{8})/$", stats.statistics), + path("copyright/", views.copyrightpage), + path("faqs/", views.faqindexpage), + path("faqs/report/", views.faqreport, name="faqs_report"), + path("faqs/badges/", views.faqbadges, name="faqs_badges"), + re_path(r"^faqs/(?P[a-zA-Z0-9\-]{1,40})/$", views.faqarticlepage), + path("usage/", views.indexpage), + path("widget-site/", views.widgetsitepage), + path("widget-mail/", views.widgetmailpage), + path("halloffame/", views.hofchampionspage), + path("halloffame/web/", views.hofwebpage), + path("halloffame/mail/", views.hofmailpage), + path("test-connection/", views.testconnectionpage), + path("connection/", connection.index), + re_path(r"^(connection|conn)/gettestid/$", connection.gettestid), + re_path(rf"^(connection|conn)/finished/{regex_testid}$", connection.finished), + re_path(rf"^(connection|conn)/addr-test/{regex_testid}/$", connection.addr_ipv6), + re_path(rf"^(connection|conn)/{regex_testid}/results$", connection.results), + path("test-site/", views.testsitepage), + re_path(r"^(domain|site)/$", domain.index), + re_path(rf"^(domain|site)/{regex_dname}/$", domain.siteprocess), + re_path(rf"^(domain|site)/probes/{regex_dname}/$", domain.siteprobesstatus), + re_path(rf"^(domain|site)/(?P(ipv6|tls|dnssec|appsecpriv))/{regex_dname}/$", domain.siteprobeview), + re_path(rf"^(domain|site)/{regex_dname}/results$", domain.resultscurrent), + re_path(r"^(domain|site)/(?P.*)/(?P[0-9]+)/$", domain.resultsstored, name="webtest_results"), # Non valid domain, convert to punycode and try again # these url()s should always be the last in the ^domain/ group - url(r"^(domain|site)/(?P.*)/$", domain.validate_domain), - url(r"^(domain|site)/(?P.*)/results$", domain.validate_domain), - url(r"^test-mail/$", views.testmailpage), - url(r"^mail/$", mail.index), - url(rf"^mail/{regex_mailaddr}/$", mail.mailprocess), - url(rf"^mail/probes/{regex_dname}/$", mail.siteprobesstatus), - url(rf"^mail/(?P(ipv6|auth|dnssec|tls))/{regex_mailaddr}/$", mail.mailprobeview), - url(rf"^mail/{regex_mailaddr}/results$", mail.resultscurrent), - url(r"^mail/(?P.*)/(?P[0-9]+)/$", mail.resultsstored, name="mailtest_results"), + re_path(r"^(domain|site)/(?P.*)/$", domain.validate_domain), + re_path(r"^(domain|site)/(?P.*)/results$", domain.validate_domain), + path("test-mail/", views.testmailpage), + path("mail/", mail.index), + re_path(rf"^mail/{regex_mailaddr}/$", mail.mailprocess), + re_path(rf"^mail/probes/{regex_dname}/$", mail.siteprobesstatus), + re_path(rf"^mail/(?P(ipv6|auth|dnssec|tls))/{regex_mailaddr}/$", mail.mailprobeview), + re_path(rf"^mail/{regex_mailaddr}/results$", mail.resultscurrent), + re_path(r"^mail/(?P.*)/(?P[0-9]+)/$", mail.resultsstored, name="mailtest_results"), # Non valid mail, convert to punycode and try again # these url()s should always be the last in the ^mail/ group - url(r"^mail/(?P.*)/$", mail.validate_domain), - url(r"^mail/(?P.*)/results$", mail.validate_domain), - url(rf"^clear/{regex_dname}/$", views.clear), - url(r"^change_language/$", views.change_language, name="change_language"), + re_path(r"^mail/(?P.*)/$", mail.validate_domain), + re_path(r"^mail/(?P.*)/results$", mail.validate_domain), + re_path(rf"^clear/{regex_dname}/$", views.clear), + path("change_language/", views.change_language, name="change_language"), ] if settings.INTERNETNL_BRANDING: urlpatterns += [ - url(r"^contact/$", views.indexpage), - url(r"^blogs/$", views.blogindexpage), - url(r"^blogs/(?P[a-zA-Z0-9\-]{1,40})/$", views.blogarticlepage), - url(r"^blogs/(?P[a-zA-Z0-9\-]{1,40})/(?P
[a-zA-Z0-9\-]{1,80})/$", views.blogarticlepage), - url(r"^news/$", views.newsindexpage), - url(r"^news/(?P
[a-zA-Z0-9\-]{1,80})/$", views.newsarticlepage), - url(r"^articles/$", views.articleindexpage), - url(r"^article/$", views.articlespage), - url(r"^article/(?P
[a-zA-Z0-9\.\-]{1,80})/$", views.articlepage), - url(r"^about/$", views.aboutpage), - url(r"^disclosure/$", views.disclosurepage), - url(r"^privacy/$", views.privacypage), + path("contact/", views.indexpage), + path("blogs/", views.blogindexpage), + re_path(r"^blogs/(?P[a-zA-Z0-9\-]{1,40})/$", views.blogarticlepage), + re_path(r"^blogs/(?P[a-zA-Z0-9\-]{1,40})/(?P
[a-zA-Z0-9\-]{1,80})/$", views.blogarticlepage), + path("news/", views.newsindexpage), + re_path(r"^news/(?P
[a-zA-Z0-9\-]{1,80})/$", views.newsarticlepage), + path("articles/", views.articleindexpage), + path("article/", views.articlespage), + re_path(r"^article/(?P
[a-zA-Z0-9\.\-]{1,80})/$", views.articlepage), + path("about/", views.aboutpage), + path("disclosure/", views.disclosurepage), + path("privacy/", views.privacypage), ] # Host-urls that are accessible by host-only, which should be approachable by developers as well during @@ -83,52 +83,52 @@ # This is not enabled by default because it returns the ip address (pii) of the requester. if settings.DEBUG: urlpatterns += [ - url(r"^network_ipv4/(?P[0-9abcdef]+)/$", views.connection.network_ipv4), - url(r"^network_ipv6/(?P[0-9abcdef]+)/$", views.connection.network_ipv6), - url(r"^network_resolver/(?P[0-9abcdef]+)/$", views.connection.network_resolver), + re_path(r"^network_ipv4/(?P[0-9abcdef]+)/$", views.connection.network_ipv4), + re_path(r"^network_ipv6/(?P[0-9abcdef]+)/$", views.connection.network_ipv6), + re_path(r"^network_resolver/(?P[0-9abcdef]+)/$", views.connection.network_resolver), ] if hasattr(settings, "MANUAL_HOF") and settings.MANUAL_HOF: for key in settings.MANUAL_HOF: urlpatterns += [ - url(rf"^halloffame/(?P{key})/$", views.hofmanualpage), + re_path(rf"^halloffame/(?P{key})/$", views.hofmanualpage), ] if hasattr(settings, "HAS_ACCESSIBILITY_PAGE") and settings.HAS_ACCESSIBILITY_PAGE: urlpatterns += [ - url(r"^accessibility/$", views.accessibility), + path("accessibility/", views.accessibility), ] if settings.ENABLE_BATCH is True: urlpatterns += [ - url( + re_path( rf"^api/batch/v{BATCH_API_MAJOR_VERSION}/requests$", batch.endpoint_requests, name="batch_endpoint_requests", ), - url( + re_path( rf"^api/batch/v{BATCH_API_MAJOR_VERSION}/requests/{regex_testid}$", batch.endpoint_request, name="batch_endpoint_request", ), - url( + re_path( rf"^api/batch/v{BATCH_API_MAJOR_VERSION}/requests/{regex_testid}/results$", batch.endpoint_results, name="batch_endpoint_results", ), - url( + re_path( rf"^api/batch/v{BATCH_API_MAJOR_VERSION}/requests/{regex_testid}/results_technical$", batch.endpoint_results_technical, name="batch_endpoint_results_technical", ), - url( + re_path( rf"^api/batch/v{BATCH_API_MAJOR_VERSION}/metadata/report$", batch.endpoint_metadata_report, name="batch_endpoint_metadata_report", ), - url(r"^api/batch/openapi.yaml$", batch.documentation, name="batch_documentation"), + re_path(r"^api/batch/openapi.yaml$", batch.documentation, name="batch_documentation"), # The following should always be the last to catch now-invalid urls. - url(r"^api/batch/", batch.old_url, name="batch_old"), + re_path(r"^api/batch/", batch.old_url, name="batch_old"), ] # Serve static files for development, for production `whitenoise` app is used and the webserver is diff --git a/interface/views/__init__.py b/interface/views/__init__.py index df447b644..cc74c77ac 100644 --- a/interface/views/__init__.py +++ b/interface/views/__init__.py @@ -7,7 +7,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import redirect, render from django.utils import translation -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from interface import redis_id, simple_cache_page from interface.views.shared import ( diff --git a/interface/views/connection.py b/interface/views/connection.py index a22af39db..52c115eaa 100644 --- a/interface/views/connection.py +++ b/interface/views/connection.py @@ -10,7 +10,7 @@ from django.core.cache import cache from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django_redis import get_redis_connection import unbound diff --git a/interface/views/domain.py b/interface/views/domain.py index e944c4630..a2373ec9c 100644 --- a/interface/views/domain.py +++ b/interface/views/domain.py @@ -7,7 +7,7 @@ from django.core.cache import cache from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from checks.models import ( AutoConfOption, diff --git a/interface/views/mail.py b/interface/views/mail.py index 252fd1417..93eb571ff 100644 --- a/interface/views/mail.py +++ b/interface/views/mail.py @@ -7,7 +7,7 @@ from django.core.cache import cache from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from checks.models import ( AutoConfOption, diff --git a/interface/views/shared.py b/interface/views/shared.py index 3c2d970eb..d6e26bb76 100644 --- a/interface/views/shared.py +++ b/interface/views/shared.py @@ -18,7 +18,7 @@ from django.shortcuts import render from django.utils import timezone from django.utils.http import url_has_allowed_host_and_scheme -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ import unbound @@ -146,7 +146,7 @@ def get_client_ip(request): """ if settings.DJANGO_IS_PROXIED: - ip = request.META.get("HTTP_X_FORWARDED_FOR", None) + ip = request.headers.get("x-forwarded-for", None) else: ip = request.META.get("REMOTE_ADDR") return ip diff --git a/internetnl/custom_middlewares.py b/internetnl/custom_middlewares.py index ca579af56..a611f1826 100644 --- a/internetnl/custom_middlewares.py +++ b/internetnl/custom_middlewares.py @@ -24,7 +24,7 @@ def process_request(self, request): if translation.check_for_language(current_language): request.current_language_code = current_language else: - request.current_language_code = self.get_preferred_language(request.META.get("HTTP_ACCEPT_LANGUAGE", "")) + request.current_language_code = self.get_preferred_language(request.headers.get("accept-language", "")) translation.activate(request.current_language_code) def get_preferred_language(self, http_accept_language): diff --git a/internetnl/urls.py b/internetnl/urls.py index f603b2e9e..5f048a9e8 100644 --- a/internetnl/urls.py +++ b/internetnl/urls.py @@ -1,15 +1,15 @@ # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors # SPDX-License-Identifier: Apache-2.0 from django.conf import settings -from django.conf.urls import include, url +from django.urls import include, path, re_path from django.contrib import admin urlpatterns = [ - url(r"^", include("interface.urls")), + path("", include("interface.urls")), ] handler404 = "interface.views.page404" if settings.DEBUG is True: urlpatterns += [ - url(r"^admin/", admin.site.urls), + re_path(r"^admin/", admin.site.urls), ] diff --git a/requirements.in b/requirements.in index ecdc44803..67c45264e 100644 --- a/requirements.in +++ b/requirements.in @@ -2,9 +2,8 @@ # virtual environments wheel -# The current LTS of django is 3.2, see: https://www.djangoproject.com/download/#supported-versions -# We'll probably wait until 4.2 LTS to be available for the next upgrade. -Django<4 +# The current LTS of django is 4.2, see: https://www.djangoproject.com/download/#supported-versions +Django<5 django-redis<5 diff --git a/requirements.txt b/requirements.txt index 071bdf121..1af1ad0f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --resolver=backtracking requirements.in +# pip-compile requirements.in # amqp==5.1.1 # via kombu @@ -56,7 +56,7 @@ colorlog==6.7.0 # via -r requirements.in cryptography==38.0.4 # via -r requirements.in -django==3.2.24 +django==4.2.10 # via # -r requirements.in # django-bleach @@ -150,8 +150,6 @@ python-dateutil==2.8.2 # sectxt pythonwhois-internet-nl @ https://github.com/internetstandards/python-whois/releases/download/v1.0.0/pythonwhois-internet.nl-1.0.0.tar.gz # via -r requirements.in -pytz==2023.3 - # via django pyyaml==6.0.1 # via -r requirements.in redis==5.0.0 From c22b15cb37bbd278ab4ece4ef6752f2a985ac54e Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Wed, 28 Feb 2024 14:40:32 +0100 Subject: [PATCH 15/20] Fix #1291 - Compare HTTPS redirect hostname, not netloc (incl port) When the redirect URL includes an explicit port, as done by some AWS, the netloc includes this port and comparison would fail. --- checks/tasks/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/tasks/tls.py b/checks/tasks/tls.py index 93ec45add..b904b0b99 100644 --- a/checks/tasks/tls.py +++ b/checks/tasks/tls.py @@ -2998,7 +2998,7 @@ def forced_http_check(af_ip_pair, url, task): parsed_url = urlparse(response.url) # Requirement: in case of redirecting, a domain should firstly upgrade itself by # redirecting to its HTTPS version before it may redirect to another domain (#1208) - if parsed_url.scheme == "https" and url == parsed_url.netloc: + if parsed_url.scheme == "https" and url == parsed_url.hostname: forced_https = ForcedHttpsStatus.good forced_https_score = scoring.WEB_TLS_FORCED_HTTPS_GOOD break From 0cb0b3e34fff6edc3b0dcfb7589a2126f805ca27 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Wed, 28 Feb 2024 11:06:20 +0100 Subject: [PATCH 16/20] Set cache-max-negative-ttl in DNS queries (in addition to cache-max-ttl) --- checks/tasks/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/checks/tasks/__init__.py b/checks/tasks/__init__.py index ad401164b..5a8e0db2d 100644 --- a/checks/tasks/__init__.py +++ b/checks/tasks/__init__.py @@ -38,10 +38,11 @@ def ub_ctx(self): self._ub_ctx.zone_add("test.", "transparent") self._ub_ctx.set_option("cache-max-ttl:", str(settings.CACHE_TTL * 0.9)) - # Some (unknown) tests probably depend on consistent ordering in unbound responses + self._ub_ctx.set_option("cache-max-negative-ttl:", str(settings.CACHE_TTL * 0.9)) + # Some may depend on consistent ordering in unbound responses # https://github.com/internetstandards/Internet.nl/pull/613#discussion_r892196819 + # https://github.com/internetstandards/Internet.nl/pull/1292#discussion_r1505778673 self._ub_ctx.set_option("rrset-roundrobin:", "no") - self._ub_ctx.set_option("cache-max-ttl:", str(settings.CACHE_TTL * 0.9)) # XXX: Remove for now; inconsistency with applying settings on celery. # YYY: Removal caused infinite waiting on pipe to unbound. Added again. self._ub_ctx.set_async(True) From 098d6b76e1ea1338688d51a341fb140309b1986f Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Wed, 28 Feb 2024 15:15:26 +0100 Subject: [PATCH 17/20] Limit cache-ttl to 1 minute in central unbound --- docker/resolver/resolver-permissive.conf.template | 3 +++ docker/resolver/resolver-validating.conf.template | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docker/resolver/resolver-permissive.conf.template b/docker/resolver/resolver-permissive.conf.template index 58706106d..c5672236f 100644 --- a/docker/resolver/resolver-permissive.conf.template +++ b/docker/resolver/resolver-permissive.conf.template @@ -12,6 +12,9 @@ server: module-config: "iterator" chroot: "" + cache-max-ttl: 60 + cache-max-negative-ttl: 60 + logfile: /dev/stdout ${DEBUG_LOG_UNBOUND_STATEMENTS} diff --git a/docker/resolver/resolver-validating.conf.template b/docker/resolver/resolver-validating.conf.template index eefd7bbb8..cb268807f 100644 --- a/docker/resolver/resolver-validating.conf.template +++ b/docker/resolver/resolver-validating.conf.template @@ -12,6 +12,9 @@ server: module-config: "validator iterator" chroot: "" + cache-max-ttl: 60 + cache-max-negative-ttl: 60 + logfile: /dev/stdout ${DEBUG_LOG_UNBOUND_STATEMENTS} From 9cf83dbad92d75046f91d3dcae251d3068dcfa65 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Wed, 28 Feb 2024 16:28:40 +0100 Subject: [PATCH 18/20] Update makefile for improved translation flow --- Makefile | 38 +++++++++++--------------------------- bin/update_translations.sh | 5 +++++ 2 files changed, 16 insertions(+), 27 deletions(-) create mode 100755 bin/update_translations.sh diff --git a/Makefile b/Makefile index a0c481793..9a75be72f 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,6 @@ PY?=python TAR?=0 BINDIR=bin -POFILESEXEC=$(BINDIR)/pofiles.py FRONTENDEXEC=$(BINDIR)/frontend.py REMOTEDATADIR=remote_data @@ -26,13 +25,6 @@ mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path)))) ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -ifeq ($(TAR), 0) - POFILES_TAR_ARGS=to_tar -else - POFILES_TAR_ARGS=from_tar - POFILES_TAR_ARGS+=$(TAR) -endif - pysrcdirs = internetnl tests interface checks integration_tests pysrc = $(shell find ${pysrcdirs} -name \*.py) @@ -45,9 +37,8 @@ help: @echo 'Makefile for internet.nl' @echo '' @echo 'Usage:' - @echo ' make translations combine the translation files to Django PO files' - @echo ' make translations_tar create a tar from the translations' - @echo ' make translations_tar TAR= read the tar and update the translations' + @echo ' make update_content update the translation files from content repo.' + @echo ' Optional branch=x to use a specific content repo branch.' @echo ' make frontend (re)generate CSS and Javascript' @echo ' make update_padded_macs update padded MAC information' @echo ' make update_cert_fingerprints update certificate fingerpint information' @@ -69,25 +60,18 @@ frontend: . .venv/bin/activate && ${_env} python3 manage.py collectstatic --no-input . .venv/bin/activate && ${_env} python3 manage.py api_generate_doc + ${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/lint.sh ${pysrcdirs} -translate_content_to_main: - # Note: you may need to run this a few times to get rid of the access denied errors... - # This retrieves the content from the content repository and merges it with the .po files of this repo. - # The procedure is detailed at: https://github.com/internetstandards/Internet.nl_content/blob/master/.README.md +branch ?= main +update_content: + # This retrieves the content from the content repository and merges it with the .po files of this repo. + # The procedure is detailed at: https://github.com/internetstandards/Internet.nl_content/blob/master/.README.md rm -rf tmp/locale_files/ rm -f tmp/content_repo.tar.gz - git clone git@github.com:internetstandards/Internet.nl_content/ tmp/locale_files/ - - # If you need a specific branch people are working on: - # git clone -b news-item_PLIS-meeting_on_IPv6 https://github.com/internetstandards/Internet.nl_content/ tmp/locale_files/ - - # change dir to tmp to prevent the /tmp dir being mentioned in the resulting tar file. - cd tmp && tar zcvf content_repo.tar.gz locale_files/* - ${MAKE} translations_tar TAR=tmp/content_repo.tar.gz - ${MAKE} translations - . .venv/bin/activate && ${_env} python3 manage.py compilemessages --ignore=.venv - # Purposefully _not_ deleting things in the tmp dir so it allows inspection after execution. - + mkdir -p tmp/locale_files/ + git clone -b $(branch) git@github.com:internetstandards/Internet.nl_content/ tmp/locale_files/ + ${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/update_translations.sh + rm -rf tmp/locale_files update_padded_macs: chmod +x $(MACSDIR)/update-macs.sh diff --git a/bin/update_translations.sh b/bin/update_translations.sh new file mode 100755 index 000000000..5cf33bb5f --- /dev/null +++ b/bin/update_translations.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +tar --strip-components=1 -cf tmp/content_repo.tar.gz locale_files/* +python3 bin/pofiles.py from_tar tmp/content_repo.tar.gz + +# to_django is performed in Dockerfile \ No newline at end of file From 5f1dc41f9fe1f1f59c492b5269cd34a13814d4b9 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Thu, 29 Feb 2024 12:10:38 +0100 Subject: [PATCH 19/20] Update sectxt dependency to 0.8.3+hotfix --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1af1ad0f9..7ab33f5ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -163,7 +163,7 @@ requests==2.31.0 # sectxt rjsmin==1.2.1 # via -r requirements.in -sectxt==0.8.3 +sectxt @ https://github.com/mxsasha/sectxt/archive/refs/tags/0.8.3_leap.tar.gz # via -r requirements.in selenium==3.141.0 # via -r requirements.in From dd4ded3db4d8046db4e97eedbdecfe605cb497ff Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Thu, 29 Feb 2024 12:25:42 +0100 Subject: [PATCH 20/20] Add 1.8.5 to change log. Signed-off-by: Sasha Romijn --- Changelog.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index e1d65cf07..33943a18e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # Change Log +## 1.8.5 + +Release 1.8.5 contains a hotfix for the [sectxt library failing on leap days](https://github.com/DigitalTrustCenter/sectxt/issues/66). + ## 1.8.4 Release 1.8.4: @@ -8,7 +12,7 @@ Release 1.8.4: - Restricts HTTPS redirects to the same domain, no longer allowing directions to a subdomain first (#1208). - Updates a number of other dependencies. - Fixes an issue where certbot renewals were not correctly run. -- + ## 1.8.3 Release 1.8.3 fixes an issue where HSTS and CSP headers were missing from he www-subdomain of the main domain (#1210, #1211).