diff --git a/Dockerfile b/Dockerfile index d6f6ea784..027a3207a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,15 +15,15 @@ WORKDIR /webodm RUN printf "deb http://old-releases.ubuntu.com/ubuntu/ hirsute main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-backports main restricted universe multiverse" > /etc/apt/sources.list # Install Node.js using new Node install method -RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl && \ - apt-get install -y ca-certificates gnupg && \ +RUN apt-get -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends wget curl && \ + apt-get -o Acquire::Retries=3 install -y ca-certificates gnupg && \ mkdir -p /etc/apt/keyrings && \ curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ NODE_MAJOR=20 && \ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get -qq update && apt-get -qq install -y nodejs && \ + apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs && \ # Install Python3, GDAL, PDAL, nginx, letsencrypt, psql - apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \ + apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \ update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \ # Install pip reqs pip install -U pip && pip install -r requirements.txt "boto3==1.14.14" && \ diff --git a/app/api/tasks.py b/app/api/tasks.py index ed02314ed..c65e94253 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -402,6 +402,26 @@ def get(self, request, pk=None, project_pk=None, unsafe_asset_path=""): return download_file_response(request, asset_path, 'inline') +""" +Task backup endpoint +""" +class TaskBackup(TaskNestedView): + def get(self, request, pk=None, project_pk=None): + """ + Downloads a task's backup + """ + task = self.get_and_check_task(request, pk) + + # Check and download + try: + asset_fs = task.get_task_backup_stream() + except FileNotFoundError: + raise exceptions.NotFound(_("Asset does not exist")) + + download_filename = request.GET.get('filename', get_asset_download_filename(task, "backup.zip")) + + return download_file_stream(request, asset_fs, 'attachment', download_filename=download_filename) + """ Task assets import """ diff --git a/app/api/urls.py b/app/api/urls.py index 3de135901..7cf862d45 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -3,7 +3,7 @@ from app.api.presets import PresetViewSet from app.plugins.views import api_view_handler from .projects import ProjectViewSet -from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport +from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskBackup, TaskAssetsImport from .imageuploads import Thumbnail, ImageDownload from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet @@ -46,6 +46,7 @@ url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P.+)$', TaskDownloads.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/assets/(?P.+)$', TaskAssets.as_view()), url(r'projects/(?P[^/.]+)/tasks/import$', TaskAssetsImport.as_view()), + url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/backup$', TaskBackup.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/thumbnail/(?P.+)$', Thumbnail.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/images/download/(?P.+)$', ImageDownload.as_view()), diff --git a/app/models/task.py b/app/models/task.py index 66dd5dda9..44c449eaf 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -3,6 +3,7 @@ import shutil import time import struct +from datetime import datetime import uuid as uuid_module from app.vendor import zipfly @@ -453,6 +454,48 @@ def duplicate(self, set_new_name=True): logger.warning("Cannot duplicate task: {}".format(str(e))) return False + + def write_backup_file(self): + """Dump this tasks's fields to a backup file""" + with open(self.data_path("backup.json"), "w") as f: + f.write(json.dumps({ + 'name': self.name, + 'processing_time': self.processing_time, + 'options': self.options, + 'created_at': self.created_at.astimezone(timezone.utc).timestamp(), + 'public': self.public, + 'resize_to': self.resize_to, + 'potree_scene': self.potree_scene, + 'tags': self.tags + })) + + def read_backup_file(self): + """Set this tasks fields based on the backup file (but don't save)""" + backup_file = self.data_path("backup.json") + if os.path.isfile(backup_file): + try: + with open(backup_file, "r") as f: + backup = json.loads(f.read()) + + self.name = backup.get('name', self.name) + self.processing_time = backup.get('processing_time', self.processing_time) + self.options = backup.get('options', self.options) + self.created_at = datetime.fromtimestamp(backup.get('created_at', self.created_at.astimezone(timezone.utc).timestamp()), tz=timezone.utc) + self.public = backup.get('public', self.public) + self.resize_to = backup.get('resize_to', self.resize_to) + self.potree_scene = backup.get('potree_scene', self.potree_scene) + self.tags = backup.get('tags', self.tags) + + except Exception as e: + logger.warning("Cannot read backup file: %s" % str(e)) + + def get_task_backup_stream(self): + self.write_backup_file() + zip_dir = self.task_path("") + paths = [{'n': os.path.relpath(os.path.join(dp, f), zip_dir), 'fs': os.path.join(dp, f)} for dp, dn, filenames in os.walk(zip_dir) for f in filenames] + if len(paths) == 0: + raise FileNotFoundError("No files available for export") + return zipfly.ZipStream(paths) def get_asset_file_or_stream(self, asset): """ @@ -568,7 +611,6 @@ def handle_import(self): pass self.pending_action = None - self.processing_time = 0 self.save() def process(self): @@ -859,10 +901,24 @@ def extract_assets_and_complete(self): zip_h.extractall(assets_dir) logger.info("Extracted all.zip for {}".format(self)) - - # Remove zip + os.remove(zip_path) + # Check if this looks like a backup file, in which case we need to move the files + # a directory level higher + is_backup = os.path.isfile(self.assets_path("data", "backup.json")) and os.path.isdir(self.assets_path("assets")) + if is_backup: + logger.info("Restoring from backup") + try: + tmp_dir = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{self.id}.backup") + + shutil.move(assets_dir, tmp_dir) + shutil.rmtree(self.task_path("")) + shutil.move(tmp_dir, self.task_path("")) + except shutil.Error as e: + logger.warning("Cannot restore from backup: %s" % str(e)) + raise NodeServerError("Cannot restore from backup") + # Populate *_extent fields extent_fields = [ (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), @@ -904,8 +960,13 @@ def extract_assets_and_complete(self): self.update_size() self.potree_scene = {} self.running_progress = 1.0 - self.console += gettext("Done!") + "\n" self.status = status_codes.COMPLETED + + if is_backup: + self.read_backup_file() + else: + self.console += gettext("Done!") + "\n" + self.save() from app.plugins import signals as plugin_signals diff --git a/app/static/app/js/components/AssetDownloadButtons.jsx b/app/static/app/js/components/AssetDownloadButtons.jsx index 29fd6d593..445d62d6f 100644 --- a/app/static/app/js/components/AssetDownloadButtons.jsx +++ b/app/static/app/js/components/AssetDownloadButtons.jsx @@ -82,6 +82,9 @@ class AssetDownloadButtons extends React.Component { ); } })} +
  • + {_("Backup")} +
    • ); } diff --git a/app/static/app/js/components/ImportTaskPanel.jsx b/app/static/app/js/components/ImportTaskPanel.jsx index b3ed11dcb..c4aaab5a2 100644 --- a/app/static/app/js/components/ImportTaskPanel.jsx +++ b/app/static/app/js/components/ImportTaskPanel.jsx @@ -158,8 +158,8 @@ class ImportTaskPanel extends React.Component { -

    {_("Import Existing Assets")}

    -

    '}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets.")}

    +

    {_("Import Assets or Backups")}

    +

    '}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets | Backup.")}