From 1add2fefe5980511ed08900d91bdfc877cf49457 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 31 May 2024 13:09:54 -0400 Subject: [PATCH 1/3] Add task backup import/export --- app/api/tasks.py | 20 +++ app/api/urls.py | 3 +- app/models/task.py | 66 +++++++- .../js/components/AssetDownloadButtons.jsx | 3 + .../app/js/components/ImportTaskPanel.jsx | 4 +- .../app/js/translations/odm_autogenerated.js | 144 +++++++++--------- locale | 2 +- 7 files changed, 163 insertions(+), 79 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index ed02314ed..c65e94253 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -402,6 +402,26 @@ def get(self, request, pk=None, project_pk=None, unsafe_asset_path=""): return download_file_response(request, asset_path, 'inline') +""" +Task backup endpoint +""" +class TaskBackup(TaskNestedView): + def get(self, request, pk=None, project_pk=None): + """ + Downloads a task's backup + """ + task = self.get_and_check_task(request, pk) + + # Check and download + try: + asset_fs = task.get_task_backup_stream() + except FileNotFoundError: + raise exceptions.NotFound(_("Asset does not exist")) + + download_filename = request.GET.get('filename', get_asset_download_filename(task, "backup.zip")) + + return download_file_stream(request, asset_fs, 'attachment', download_filename=download_filename) + """ Task assets import """ diff --git a/app/api/urls.py b/app/api/urls.py index 3de135901..7cf862d45 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -3,7 +3,7 @@ from app.api.presets import PresetViewSet from app.plugins.views import api_view_handler from .projects import ProjectViewSet -from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport +from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskBackup, TaskAssetsImport from .imageuploads import Thumbnail, ImageDownload from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet @@ -46,6 +46,7 @@ url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P.+)$', TaskDownloads.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/assets/(?P.+)$', TaskAssets.as_view()), url(r'projects/(?P[^/.]+)/tasks/import$', TaskAssetsImport.as_view()), + url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/backup$', TaskBackup.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/thumbnail/(?P.+)$', Thumbnail.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/download/(?P.+)$', ImageDownload.as_view()), diff --git a/app/models/task.py b/app/models/task.py index 66dd5dda9..ab156dd5c 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -3,6 +3,7 @@ import shutil import time import struct +from datetime import datetime import uuid as uuid_module from app.vendor import zipfly @@ -453,6 +454,48 @@ def duplicate(self, set_new_name=True): logger.warning("Cannot duplicate task: {}".format(str(e))) return False + + def write_backup_file(self): + """Dump this tasks's fields to a backup file""" + with open(self.data_path("backup.json"), "w") as f: + f.write(json.dumps({ + 'name': self.name, + 'processing_time': self.processing_time, + 'options': self.options, + 'created_at': self.created_at.timestamp(), + 'public': self.public, + 'resize_to': self.resize_to, + 'potree_scene': self.potree_scene, + 'tags': self.tags + })) + + def read_backup_file(self): + """Set this tasks fields based on the backup file (but don't save)""" + backup_file = self.data_path("backup.json") + if os.path.isfile(backup_file): + try: + with open(backup_file, "r") as f: + backup = json.loads(f.read()) + + self.name = backup.get('name', self.name) + self.processing_time = backup.get('processing_time', self.processing_time) + self.options = backup.get('options', self.options) + self.created_at = datetime.fromtimestamp(backup.get('created_at', self.created_at.timestamp())) + self.public = backup.get('public', self.public) + self.resize_to = backup.get('resize_to', self.resize_to) + self.potree_scene = backup.get('potree_scene', self.potree_scene) + self.tags = backup.get('tags', self.tags) + + except Exception as e: + logger.warning("Cannot read backup file: %s" % str(e)) + + def get_task_backup_stream(self): + self.write_backup_file() + zip_dir = self.task_path("") + paths = [{'n': os.path.relpath(os.path.join(dp, f), zip_dir), 'fs': os.path.join(dp, f)} for dp, dn, filenames in os.walk(zip_dir) for f in filenames] + if len(paths) == 0: + raise FileNotFoundError("No files available for export") + return zipfly.ZipStream(paths) def get_asset_file_or_stream(self, asset): """ @@ -568,7 +611,6 @@ def handle_import(self): pass self.pending_action = None - self.processing_time = 0 self.save() def process(self): @@ -859,10 +901,24 @@ def extract_assets_and_complete(self): zip_h.extractall(assets_dir) logger.info("Extracted all.zip for {}".format(self)) - - # Remove zip + os.remove(zip_path) + # Check if this looks like a backup file, in which case we need to move the files + # a directory level higher + is_backup = os.path.isfile(self.assets_path("data", "backup.json")) and os.path.isdir(self.assets_path("assets")) + if is_backup: + logger.info("Restoring from backup") + try: + tmp_dir = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{self.id}.backup") + + shutil.move(assets_dir, tmp_dir) + shutil.rmtree(self.task_path("")) + shutil.move(tmp_dir, self.task_path("")) + except shutil.Error as e: + logger.warning("Cannot restore from backup: %s" % str(e)) + raise NodeServerError("Cannot restore from backup") + # Populate *_extent fields extent_fields = [ (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), @@ -906,6 +962,10 @@ def extract_assets_and_complete(self): self.running_progress = 1.0 self.console += gettext("Done!") + "\n" self.status = status_codes.COMPLETED + + if is_backup: + self.read_backup_file() + self.save() from app.plugins import signals as plugin_signals diff --git a/app/static/app/js/components/AssetDownloadButtons.jsx b/app/static/app/js/components/AssetDownloadButtons.jsx index 29fd6d593..445d62d6f 100644 --- a/app/static/app/js/components/AssetDownloadButtons.jsx +++ b/app/static/app/js/components/AssetDownloadButtons.jsx @@ -82,6 +82,9 @@ class AssetDownloadButtons extends React.Component { ); } })} +
  • + {_("Backup")} +
    • ); } diff --git a/app/static/app/js/components/ImportTaskPanel.jsx b/app/static/app/js/components/ImportTaskPanel.jsx index b3ed11dcb..c4aaab5a2 100644 --- a/app/static/app/js/components/ImportTaskPanel.jsx +++ b/app/static/app/js/components/ImportTaskPanel.jsx @@ -158,8 +158,8 @@ class ImportTaskPanel extends React.Component { -

    {_("Import Existing Assets")}

    -

    '}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets.")}

    +

    {_("Import Assets or Backups")}

    +

    '}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets | Backup.")}