Add task backup import/export

Dockerfile

@@ -15,15 +15,15 @@ WORKDIR /webodm
 RUN printf "deb http://old-releases.ubuntu.com/ubuntu/ hirsute main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-backports main restricted universe multiverse" > /etc/apt/sources.list
 
 # Install Node.js using new Node install method
-RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl && \
-    apt-get install -y ca-certificates gnupg && \
+RUN apt-get -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends wget curl && \
+    apt-get -o Acquire::Retries=3 install -y ca-certificates gnupg && \
     mkdir -p /etc/apt/keyrings && \
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
     NODE_MAJOR=20 && \
     echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    apt-get -qq update && apt-get -qq install -y nodejs && \
+    apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs && \
     # Install Python3, GDAL, PDAL, nginx, letsencrypt, psql
-    apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \
+    apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \
     update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \
     # Install pip reqs
     pip install -U pip && pip install -r requirements.txt "boto3==1.14.14" && \

app/api/tasks.py

@@ -402,6 +402,26 @@ def get(self, request, pk=None, project_pk=None, unsafe_asset_path=""):
 
         return download_file_response(request, asset_path, 'inline')
 
+"""
+Task backup endpoint
+"""
+class TaskBackup(TaskNestedView):
+    def get(self, request, pk=None, project_pk=None):
+        """
+        Downloads a task's backup
+        """
+        task = self.get_and_check_task(request, pk)
+
+        # Check and download
+        try:
+            asset_fs = task.get_task_backup_stream()
+        except FileNotFoundError:
+            raise exceptions.NotFound(_("Asset does not exist"))
+
+        download_filename = request.GET.get('filename', get_asset_download_filename(task, "backup.zip"))
+
+        return download_file_stream(request, asset_fs, 'attachment', download_filename=download_filename)
+
 """
 Task assets import
 """

app/api/urls.py

@@ -3,7 +3,7 @@
 from app.api.presets import PresetViewSet
 from app.plugins.views import api_view_handler
 from .projects import ProjectViewSet
-from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport
+from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskBackup, TaskAssetsImport
 from .imageuploads import Thumbnail, ImageDownload
 from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView
 from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet
@@ -46,6 +46,7 @@
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/import$', TaskAssetsImport.as_view()),
+    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/backup$', TaskBackup.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/images/thumbnail/(?P<image_filename>.+)$', Thumbnail.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/images/download/(?P<image_filename>.+)$', ImageDownload.as_view()),
 

app/models/task.py

@@ -3,6 +3,7 @@
 import shutil
 import time
 import struct
+from datetime import datetime
 import uuid as uuid_module
 from app.vendor import zipfly
 
@@ -453,6 +454,48 @@ def duplicate(self, set_new_name=True):
             logger.warning("Cannot duplicate task: {}".format(str(e)))
 
         return False
+
+    def write_backup_file(self):
+        """Dump this tasks's fields to a backup file"""
+        with open(self.data_path("backup.json"), "w") as f:
+            f.write(json.dumps({
+                'name': self.name,
+                'processing_time': self.processing_time,
+                'options': self.options,
+                'created_at': self.created_at.astimezone(timezone.utc).timestamp(),
+                'public': self.public,
+                'resize_to': self.resize_to,
+                'potree_scene': self.potree_scene,
+                'tags': self.tags
+            }))
+
+    def read_backup_file(self):
+        """Set this tasks fields based on the backup file (but don't save)"""
+        backup_file = self.data_path("backup.json")
+        if os.path.isfile(backup_file):
+            try:
+                with open(backup_file, "r") as f:
+                    backup = json.loads(f.read())
+
+                    self.name = backup.get('name', self.name)
+                    self.processing_time = backup.get('processing_time', self.processing_time)
+                    self.options = backup.get('options', self.options)
+                    self.created_at = datetime.fromtimestamp(backup.get('created_at', self.created_at.astimezone(timezone.utc).timestamp()), tz=timezone.utc)
+                    self.public = backup.get('public', self.public)
+                    self.resize_to = backup.get('resize_to', self.resize_to)
+                    self.potree_scene = backup.get('potree_scene', self.potree_scene)
+                    self.tags = backup.get('tags', self.tags)
+
+            except Exception as e:
+                logger.warning("Cannot read backup file: %s" % str(e))
+
+    def get_task_backup_stream(self):
+        self.write_backup_file()
+        zip_dir = self.task_path("")
+        paths = [{'n': os.path.relpath(os.path.join(dp, f), zip_dir), 'fs': os.path.join(dp, f)} for dp, dn, filenames in os.walk(zip_dir) for f in filenames]
+        if len(paths) == 0:
+            raise FileNotFoundError("No files available for export")
+        return zipfly.ZipStream(paths)
 
     def get_asset_file_or_stream(self, asset):
         """
@@ -568,7 +611,6 @@ def handle_import(self):
                 pass
 
         self.pending_action = None
-        self.processing_time = 0
         self.save()
 
     def process(self):
@@ -859,10 +901,24 @@ def extract_assets_and_complete(self):
             zip_h.extractall(assets_dir)
 
         logger.info("Extracted all.zip for {}".format(self))
-
-        # Remove zip
+
         os.remove(zip_path)
 
+        # Check if this looks like a backup file, in which case we need to move the files
+        # a directory level higher
+        is_backup = os.path.isfile(self.assets_path("data", "backup.json")) and os.path.isdir(self.assets_path("assets"))
+        if is_backup:
+            logger.info("Restoring from backup")
+            try:
+                tmp_dir = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{self.id}.backup")
+
+                shutil.move(assets_dir, tmp_dir)
+                shutil.rmtree(self.task_path(""))
+                shutil.move(tmp_dir, self.task_path(""))
+            except shutil.Error as e:
+                logger.warning("Cannot restore from backup: %s" % str(e))
+                raise NodeServerError("Cannot restore from backup")
+
         # Populate *_extent fields
         extent_fields = [
             (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
@@ -904,8 +960,13 @@ def extract_assets_and_complete(self):
         self.update_size()
         self.potree_scene = {}
         self.running_progress = 1.0
-        self.console += gettext("Done!") + "\n"
         self.status = status_codes.COMPLETED
+
+        if is_backup:
+            self.read_backup_file()
+        else:
+            self.console += gettext("Done!") + "\n"
+
         self.save()
 
         from app.plugins import signals as plugin_signals

app/static/app/js/components/AssetDownloadButtons.jsx

@@ -82,6 +82,9 @@ class AssetDownloadButtons extends React.Component {
                         </li>);
                 }
             })}
+            <li>
+                <a href={`/api/projects/${this.props.task.project}/tasks/${this.props.task.id}/backup`}><i className="fa fa-file-download fa-fw"></i> {_("Backup")}</a>
+            </li>
           </ul>
         </div>);
     }

app/static/app/js/components/ImportTaskPanel.jsx

@@ -158,8 +158,8 @@ class ImportTaskPanel extends React.Component {
           <ErrorMessage bind={[this, 'error']} />
 
           <button type="button" className="close theme-color-primary" title="Close" onClick={this.cancel}><span aria-hidden="true">&times;</span></button>
-          <h4>{_("Import Existing Assets")}</h4>
-          <p><Trans params={{arrow: '<i class="glyphicon glyphicon-arrow-right"></i>'}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets.")}</Trans></p>
+          <h4>{_("Import Assets or Backups")}</h4>
+          <p><Trans params={{arrow: '<i class="glyphicon glyphicon-arrow-right"></i>'}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets | Backup.")}</Trans></p>
 
           <button disabled={this.state.uploading}
                   type="button"