Merge pull request #1386 from pierotofy/console

Move task.console_output

app/api/tasks.py

@@ -74,7 +74,7 @@ def get_can_rerun_from(self, obj):
 
     class Meta:
         model = models.Task
-        exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', )
+        exclude = ('orthophoto_extent', 'dsm_extent', 'dtm_extent', )
         read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', )
 
 class TaskViewSet(viewsets.ViewSet):
@@ -83,7 +83,7 @@ class TaskViewSet(viewsets.ViewSet):
     A task represents a set of images and other input to be sent to a processing node.
     Once a processing node completes processing, results are stored in the task.
     """
-    queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', 'console_output', )
+    queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', )
 
     parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser, )
     ordering_fields = '__all__'
@@ -145,8 +145,7 @@ def output(self, request, pk=None, project_pk=None):
             raise exceptions.NotFound()
 
         line_num = max(0, int(request.query_params.get('line', 0)))
-        output = task.console_output or ""
-        return Response('\n'.join(output.rstrip().split('\n')[line_num:]))
+        return Response('\n'.join(task.console.output().rstrip().split('\n')[line_num:]))
 
     def list(self, request, project_pk=None):
         get_and_check_project(request, project_pk)
@@ -296,7 +295,7 @@ def partial_update(self, request, *args, **kwargs):
 
 
 class TaskNestedView(APIView):
-    queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', 'console_output', )
+    queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', )
     permission_classes = (AllowAny, )
 
     def get_and_check_task(self, request, pk, annotate={}):

app/classes/console.py

@@ -0,0 +1,48 @@
+import os
+import logging
+logger = logging.getLogger('app.logger')
+
+class Console:
+    def __init__(self, file):
+        self.file = file
+        self.base_dir = os.path.dirname(self.file)
+        self.parent_dir = os.path.dirname(self.base_dir)
+
+    def __repr__(self):
+        return "<Console output: %s>" % self.file
+
+    def __str__(self):
+        if not os.path.isfile(self.file):
+            return ""
+
+        try:
+            with open(self.file, 'r') as f:
+                return f.read()
+        except IOError:
+            logger.warn("Cannot read console file: %s" % self.file)
+            return ""
+
+    def __add__(self, other):
+        self.append(other)
+        return self
+
+    def output(self):
+        return str(self)
+
+    def append(self, text):
+        if os.path.isdir(self.parent_dir):
+            # Write
+            if not os.path.isdir(self.base_dir):
+                os.makedirs(self.base_dir, exist_ok=True)
+
+            with open(self.file, "a") as f:
+                f.write(text)
+
+    def reset(self, text = ""):
+        if os.path.isdir(self.parent_dir):
+            if not os.path.isdir(self.base_dir):
+                os.makedirs(self.base_dir, exist_ok=True)
+
+            with open(self.file, "w") as f:
+                f.write(text)
+

app/migrations/0038_remove_task_console_output.py

@@ -0,0 +1,42 @@
+# Generated by Django 2.2.27 on 2023-09-11 19:11
+import os
+from django.db import migrations
+from webodm import settings
+
+def data_path(project_id, task_id, *args):
+    return os.path.join(settings.MEDIA_ROOT,
+                        "project",
+                        str(project_id),
+                        "task",
+                        str(task_id),
+                        "data",
+                        *args)
+
+def dump_console_outputs(apps, schema_editor):
+    Task = apps.get_model('app', 'Task')
+
+    for t in Task.objects.all():
+        if t.console_output is not None and len(t.console_output) > 0:
+            dp = data_path(t.project.id, t.id)
+            os.makedirs(dp, exist_ok=True)
+            outfile = os.path.join(dp, "console_output.txt")
+
+            with open(outfile, "w") as f:
+                f.write(t.console_output)
+            print("Wrote console output for %s to %s" % (t, outfile))
+        else:
+            print("No task output for %s" % t)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0037_profile'),
+    ]
+
+    operations = [
+        migrations.RunPython(dump_console_outputs),
+        migrations.RemoveField(
+            model_name='task',
+            name='console_output',
+        ),
+    ]

app/models/task.py

@@ -46,6 +46,7 @@
 
 from functools import partial
 import subprocess
+from app.classes.console import Console
 
 logger = logging.getLogger('app.logger')
 
@@ -247,7 +248,6 @@ class Task(models.Model):
     last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error"))
     options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options"))
     available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets"))
-    console_output = models.TextField(null=False, default="", blank=True, help_text=_("Console output of the processing node"), verbose_name=_("Console Output"))
 
     orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent"))
     dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM", verbose_name=_("DSM Extent"))
@@ -290,6 +290,8 @@ def __init__(self, *args, **kwargs):
 
         # To help keep track of changes to the project id
         self.__original_project_id = self.project.id
+
+        self.console = Console(self.data_path("console_output.txt"))
 
     def __str__(self):
         name = self.name if self.name is not None else gettext("unnamed")
@@ -354,6 +356,12 @@ def assets_path(self, *args):
         """
         return self.task_path("assets", *args)
 
+    def data_path(self, *args):
+        """
+        Path to task data that does not fit in database fields (e.g. console output)
+        """
+        return self.task_path("data", *args)
+
     def task_path(self, *args):
         """
         Get path relative to the root task directory
@@ -490,7 +498,7 @@ def get_asset_download_path(self, asset):
             raise FileNotFoundError("{} is not a valid asset".format(asset))
 
     def handle_import(self):
-        self.console_output += gettext("Importing assets...") + "\n"
+        self.console += gettext("Importing assets...") + "\n"
         self.save()
 
         zip_path = self.assets_path("all.zip")
@@ -709,7 +717,7 @@ def callback(progress):
                             self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options))
                             self.upload_progress = 0
 
-                        self.console_output = ""
+                        self.console.reset()
                         self.processing_time = -1
                         self.status = None
                         self.last_error = None
@@ -740,18 +748,18 @@ def callback(progress):
                 # Need to update status (first time, queued or running?)
                 if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
                     # Update task info from processing node
-                    if not self.console_output:
+                    if not self.console.output():
                         current_lines_count = 0
                     else:
-                        current_lines_count = len(self.console_output.split("\n"))
+                        current_lines_count = len(self.console.output().split("\n"))
 
                     info = self.processing_node.get_task_info(self.uuid, current_lines_count)
 
                     self.processing_time = info.processing_time
                     self.status = info.status.value
 
                     if len(info.output) > 0:
-                        self.console_output += "\n".join(info.output) + '\n'
+                        self.console += "\n".join(info.output) + '\n'
 
                     # Update running progress
                     self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE
@@ -891,7 +899,7 @@ def extract_assets_and_complete(self):
         self.update_size()
         self.potree_scene = {}
         self.running_progress = 1.0
-        self.console_output += gettext("Done!") + "\n"
+        self.console += gettext("Done!") + "\n"
         self.status = status_codes.COMPLETED
         self.save()
 

app/tests/test_api.py

@@ -1,4 +1,5 @@
 import datetime
+import os
 
 from django.contrib.auth.models import User
 from guardian.shortcuts import assign_perm, get_objects_for_user
@@ -140,22 +141,26 @@ def test_projects_and_tasks(self):
         self.assertEqual(res.status_code, status.HTTP_200_OK)
         self.assertTrue(res.data == "")
 
-        task.console_output = "line1\nline2\nline3"
+        data_path = task.data_path()
+        if not os.path.exists(data_path):
+            os.makedirs(data_path, exist_ok=True)
+
+        task.console.reset("line1\nline2\nline3")
         task.save()
 
         res = client.get('/api/projects/{}/tasks/{}/output/'.format(project.id, task.id))
         self.assertEqual(res.status_code, status.HTTP_200_OK)
-        self.assertTrue(res.data == task.console_output)
+        self.assertEqual(res.data, task.console.output())
 
         # Console output with line num
         res = client.get('/api/projects/{}/tasks/{}/output/?line=2'.format(project.id, task.id))
-        self.assertTrue(res.data == "line3")
+        self.assertEqual(res.data, "line3")
 
         # Console output with line num out of bounds
         res = client.get('/api/projects/{}/tasks/{}/output/?line=3'.format(project.id, task.id))
-        self.assertTrue(res.data == "")
+        self.assertEqual(res.data, "")
         res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id))
-        self.assertTrue(res.data == task.console_output)
+        self.assertEqual(res.data, task.console.output())
 
         # Cannot list task details for a task belonging to a project we don't have access to
         res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id))

coreplugins/cloudimport/api_views.py

@@ -41,7 +41,7 @@ def post(self, request, project_pk=None, pk=None):
         files = platform.import_from_folder(folder_url)
 
         # Update the task with the new information
-        task.console_output += "Importing {} images...\n".format(len(files))
+        task.console += "Importing {} images...\n".format(len(files))
         task.images_count = len(files)
         task.pending_action = pending_actions.IMPORT
         task.save()

coreplugins/dronedb/api_views.py

@@ -181,7 +181,7 @@ def post(self, request, project_pk=None, pk=None):
             return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST)
 
         # Update the task with the new information
-        task.console_output += "Importing {} images...\n".format(len(files))
+        task.console += "Importing {} images...\n".format(len(files))
         task.images_count = len(files)
         task.pending_action = pending_actions.IMPORT
         task.save()

coreplugins/tasknotification/signals.py

@@ -22,7 +22,7 @@ def handle_task_completed(sender, task_id, **kwargs):
         setting = Setting.objects.first()
         notification_app_name = config_data['notification_app_name'] or settings.app_name
 
-        console_output = reverse_output(task.console_output)
+        console_output = reverse_output(task.console.output())
         notification.send(
             f"{notification_app_name} - {task.project.name} Task Completed", 
             f"{task.project.name}\n{task.name} Completed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
@@ -41,7 +41,7 @@ def handle_task_removed(sender, task_id, **kwargs):
         task = Task.objects.get(id=task_id)
         setting = Setting.objects.first()
         notification_app_name = config_data['notification_app_name'] or settings.app_name
-        console_output = reverse_output(task.console_output)
+        console_output = reverse_output(task.console.output())
         notification.send(
             f"{notification_app_name} - {task.project.name} Task removed", 
             f"{task.project.name}\n{task.name} was removed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
@@ -60,7 +60,7 @@ def handle_task_failed(sender, task_id, **kwargs):
         task = Task.objects.get(id=task_id)
         setting = Setting.objects.first()
         notification_app_name = config_data['notification_app_name'] or settings.app_name
-        console_output = reverse_output(task.console_output)
+        console_output = reverse_output(task.console.output())
         notification.send(
             f"{notification_app_name} - {task.project.name} Task Failed", 
             f"{task.project.name}\n{task.name} Failed with error: {task.last_error}\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "WebODM",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "User-friendly, extendable application and API for processing aerial imagery.",
   "main": "index.js",
   "scripts": {