[feature] Wire persistent-mass-upgrade Beat tasks into docker stack

.env

@@ -61,6 +61,8 @@ OPENWISP_CELERY_MONITORING_COMMAND_FLAGS=--concurrency=1
 OPENWISP_CELERY_MONITORING_CHECKS_COMMAND_FLAGS=--concurrency=1
 USE_OPENWISP_CELERY_FIRMWARE=True
 OPENWISP_CELERY_FIRMWARE_COMMAND_FLAGS=--concurrency=1
+OPENWISP_FIRMWARE_CHECK_PENDING_PERIOD_MINUTES=10
+OPENWISP_FIRMWARE_REMINDER_SCAN_PERIOD_DAYS=7
 CELERY_SERVICE_NETWORK_MODE=service:openvpn
 # Metric collection
 METRIC_COLLECTION=True

docs/user/settings.rst

@@ -370,6 +370,25 @@ framework.
 - **Valid Values:** STRING
 - **Default:** ``--concurrency=1``
 
+``OPENWISP_FIRMWARE_CHECK_PENDING_PERIOD_MINUTES``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- **Explanation:** Period in minutes between scans of pending persistent
+  firmware upgrades whose ``next_retry_at`` has elapsed.
+- **Valid Values:** INTEGER.
+- **Default:** ``10``
+
+``OPENWISP_FIRMWARE_REMINDER_SCAN_PERIOD_DAYS``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- **Explanation:** Period in days between scans for long-running
+  persistent batches that need a pending-upgrade reminder notification.
+  The per-batch reminder cadence itself is controlled upstream by
+  ``OPENWISP_FIRMWARE_UPGRADER_PERSISTENT_REMINDER_PERIOD`` (default 60
+  days).
+- **Valid Values:** INTEGER.
+- **Default:** ``7``
+
 ``USE_OPENWISP_CELERY_MONITORING``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

images/common/openwisp/celery.py

@@ -13,6 +13,7 @@
     {},
     {},
 )
+firmware_schedule = {}
 task_routes = {}
 
 if env_bool(os.environ.get("USE_OPENWISP_CELERY_NETWORK")):
@@ -40,6 +41,22 @@
     task_routes["openwisp_firmware_upgrader.tasks.batch_upgrade_operation"] = {
         "queue": "firmware_upgrader"
     }
+    check_pending_minutes = int(
+        os.environ.get("OPENWISP_FIRMWARE_CHECK_PENDING_PERIOD_MINUTES", "10")
+    )
+    reminder_scan_days = int(
+        os.environ.get("OPENWISP_FIRMWARE_REMINDER_SCAN_PERIOD_DAYS", "7")
+    )
+    firmware_schedule = {
+        "check_pending_upgrades": {
+            "task": "openwisp_firmware_upgrader.tasks.check_pending_upgrades",
+            "schedule": timedelta(minutes=check_pending_minutes),
+        },
+        "send_pending_upgrade_reminders": {
+            "task": "openwisp_firmware_upgrader.tasks.send_pending_upgrade_reminders",
+            "schedule": timedelta(days=reminder_scan_days),
+        },
+    }
 
 if env_bool(os.environ.get("USE_OPENWISP_RADIUS")):
     radius_schedule = {
@@ -93,6 +110,7 @@
         **notification_schedule,
         **monitoring_schedule,
         **metric_collection_schedule,
+        **firmware_schedule,
     },
 )
 app.config_from_object("django.conf:settings", namespace="CELERY")

images/openwisp_base/Dockerfile

@@ -30,13 +30,13 @@ ENV PATH="${PATH}:/home/openwisp/.local/bin"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade pip setuptools wheel
-ARG OPENWISP_MONITORING_SOURCE="openwisp-monitoring~=1.2.0"
+ARG OPENWISP_MONITORING_SOURCE="https://github.com/openwisp/openwisp-monitoring/archive/584b29db6b8ee2657cdb14722c3b129afe9a417f.tar.gz"
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_MONITORING_SOURCE}
-ARG OPENWISP_FIRMWARE_SOURCE="openwisp-firmware-upgrader~=1.2.0"
+ARG OPENWISP_FIRMWARE_SOURCE="https://github.com/openwisp/openwisp-firmware-upgrader/tarball/issues/417-persistence-schema-fields"
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_FIRMWARE_SOURCE}
-ARG OPENWISP_TOPOLOGY_SOURCE="openwisp-network-topology~=1.2.0"
+ARG OPENWISP_TOPOLOGY_SOURCE="https://github.com/openwisp/openwisp-network-topology/archive/2f23a6c8a16af19643f1abba67d436da349be3e6.tar.gz"
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_TOPOLOGY_SOURCE}
 ARG OPENWISP_RADIUS_SOURCE="openwisp-radius~=1.2.0"
@@ -46,27 +46,27 @@ RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_RADIUS_SOURCE}
 # here we try to install custom versions of the modules only if the
 # supplied argument does not equal the default value, because
 # otherwise these modules will have already been installed above
-ARG OPENWISP_IPAM_SOURCE=default
+ARG OPENWISP_IPAM_SOURCE="https://github.com/openwisp/openwisp-ipam/archive/refs/heads/1.3.tar.gz"
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_IPAM_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade ${OPENWISP_IPAM_SOURCE}; \
     fi
-ARG OPENWISP_CONTROLLER_SOURCE=default
+ARG OPENWISP_CONTROLLER_SOURCE="https://github.com/openwisp/openwisp-controller/archive/e0ee749e22c7fa1445d8ce9ad1d289c08ce74feb.tar.gz"
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_CONTROLLER_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade ${OPENWISP_CONTROLLER_SOURCE}; \
     fi
-ARG OPENWISP_NOTIFICATION_SOURCE=default
+ARG OPENWISP_NOTIFICATION_SOURCE="https://github.com/openwisp/openwisp-notifications/archive/refs/heads/1.3.tar.gz"
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_NOTIFICATION_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade ${OPENWISP_NOTIFICATION_SOURCE}; \
     fi
-ARG OPENWISP_USERS_SOURCE=default
+ARG OPENWISP_USERS_SOURCE="https://github.com/openwisp/openwisp-users/archive/refs/heads/1.3.tar.gz"
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_USERS_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade --force-reinstall ${OPENWISP_USERS_SOURCE}; \
     fi
-ARG OPENWISP_UTILS_SOURCE=default
+ARG OPENWISP_UTILS_SOURCE="https://github.com/openwisp/openwisp-utils/archive/764ea4917c2b52cddbf2e45ff9c67ac6e4c87261.tar.gz"
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_UTILS_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade --force-reinstall "${OPENWISP_UTILS_SOURCE}"; \
@@ -175,4 +175,6 @@ ENV DASHBOARD_APP_SERVICE=dashboard \
     CRON_DELETE_OLD_POSTAUTH=365 \
     CRON_CLEANUP_STALE_RADACCT=365 \
     CRON_DELETE_OLD_RADIUSBATCH_USERS=365 \
+    OPENWISP_FIRMWARE_CHECK_PENDING_PERIOD_MINUTES=10 \
+    OPENWISP_FIRMWARE_REMINDER_SCAN_PERIOD_DAYS=7 \
     METRIC_COLLECTION=True

images/openwisp_dashboard/load_init_data.py

@@ -167,12 +167,8 @@ def create_default_credentials():
 
 
 def create_ssh_key_template():
-    if Template.objects.filter(
-        default=True, config__contains="/etc/dropbear/authorized_keys"
-    ).exists():
-        return Template.objects.filter(
-            default=True, config__contains="/etc/dropbear/authorized_keys"
-        ).first()
+    if Template.objects.filter(name="SSH Keys").exists():
+        return Template.objects.filter(name="SSH Keys").first()
     public_key_filepath = os.environ["SSH_PUBLIC_KEY_PATH"]
     try:
         with open(public_key_filepath, "r") as file:

images/openwisp_dashboard/urls.py

@@ -42,6 +42,12 @@
             include((fw_private_storage_urls, "firmware"), namespace="firmware"),
         )
     ]
+    # register the REST API URLs so admin templates can reverse the
+    # "upgrader" namespace (e.g. the upgrade-operation cancel button);
+    # nginx routes the actual requests to the API container.
+    urlpatterns += [
+        path("api/v1/", include("openwisp_firmware_upgrader.api.urls")),
+    ]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 urlpatterns += staticfiles_urlpatterns()

tests/runtests.py

@@ -402,8 +402,11 @@ def test_celery(self):
             "openwisp_controller.subnet_division.tasks.update_subnet_division_index",
             "openwisp_controller.subnet_division.tasks.update_subnet_name_description",
             "openwisp_firmware_upgrader.tasks.batch_upgrade_operation",
+            "openwisp_firmware_upgrader.tasks.check_pending_upgrades",
             "openwisp_firmware_upgrader.tasks.create_all_device_firmwares",
             "openwisp_firmware_upgrader.tasks.create_device_firmware",
+            "openwisp_firmware_upgrader.tasks.retry_pending_upgrade",
+            "openwisp_firmware_upgrader.tasks.send_pending_upgrade_reminders",
             "openwisp_firmware_upgrader.tasks.upgrade_firmware",
             "openwisp_monitoring.check.tasks.auto_create_check",
             "openwisp_monitoring.check.tasks.perform_check",