Cancel upload improvements, automatic task cleanup, configurable doc links

app/admin.py

@@ -40,9 +40,9 @@ class TaskAdmin(admin.ModelAdmin):
     def has_add_permission(self, request):
         return False
 
-    list_display = ('id', 'project', 'processing_node', 'created_at', 'status', 'last_error')
+    list_display = ('id', 'name', 'project', 'processing_node', 'created_at', 'status', 'last_error')
     list_filter = ('status', 'project',)
-    search_fields = ('id', 'project__name')
+    search_fields = ('id', 'name', 'project__name')
 
 
 admin.site.register(Task, TaskAdmin)

app/static/app/js/components/ProcessingNodeOption.jsx

@@ -106,6 +106,13 @@ class ProcessingNodeOption extends React.Component {
     }
   }
 
+  handleHelp = e => {
+    e.preventDefault();
+    if (window.__taskOptionsDocsLink){
+      window.open(window.__taskOptionsDocsLink + "#" + encodeURIComponent(this.props.name), "task-options")
+    }
+  }
+
   render() {
     let inputControl = "";
     let warningMsg = "";
@@ -168,7 +175,7 @@ class ProcessingNodeOption extends React.Component {
 
     return (
       <div className="processing-node-option form-inline form-group form-horizontal" ref={this.setTooltips}>
-        <label>{this.props.name} {(!this.isEnumType() && this.props.domain ? `(${this.props.domain})` : "")} <i data-toggle="tooltip" data-placement="bottom" title={this.props.help} onClick={e => e.preventDefault()} className="fa fa-info-circle info-button"></i></label><br/>
+        <label>{this.props.name} {(!this.isEnumType() && this.props.domain ? `(${this.props.domain})` : "")} <i data-toggle="tooltip" data-placement="bottom" title={this.props.help} onClick={this.handleHelp} className="fa fa-info-circle info-button help-button"></i></label><br/>
         {inputControl}
         {loadFileControl}
 

app/static/app/js/components/ProjectListItem.jsx

@@ -60,6 +60,7 @@ class ProjectListItem extends React.Component {
     this.toggleTaskList = this.toggleTaskList.bind(this);
     this.closeUploadError = this.closeUploadError.bind(this);
     this.cancelUpload = this.cancelUpload.bind(this);
+    this.handleCancel = this.handleCancel.bind(this);
     this.handleTaskSaved = this.handleTaskSaved.bind(this);
     this.viewMap = this.viewMap.bind(this);
     this.handleDelete = this.handleDelete.bind(this);
@@ -248,13 +249,19 @@ class ProjectListItem extends React.Component {
                     }
                 }
             }catch(e){
-                this.setUploadState({error: `${e.message}`, uploading: false});
-                this.dz.cancelUpload();
+                if (this.manuallyCanceled){
+                  // Manually canceled, ignore error
+                  this.setUploadState({uploading: false});
+                }else{
+                  this.setUploadState({error: `${e.message}`, uploading: false});
+                }
+
+                if (this.dz.files.length) this.dz.cancelUpload();
             }
         })
         .on("queuecomplete", () => {
             const remainingFilesCount = this.state.upload.totalCount - this.state.upload.uploadedCount;
-            if (remainingFilesCount === 0){
+            if (remainingFilesCount === 0 && this.state.upload.uploadedCount > 0){
                 // All files have uploaded!
                 this.setUploadState({uploading: false});
 
@@ -332,10 +339,26 @@ class ProjectListItem extends React.Component {
     this.setUploadState({error: ""});
   }
 
-  cancelUpload(e){
+  cancelUpload(){
     this.dz.removeAllFiles(true);
   }
 
+  handleCancel(){
+    this.manuallyCanceled = true;
+    this.cancelUpload();
+    if (this.dz._taskInfo && this.dz._taskInfo.id !== undefined){
+      $.ajax({
+        url: `/api/projects/${this.state.data.id}/tasks/${this.dz._taskInfo.id}/remove/`,
+        contentType: 'application/json',
+        dataType: 'json',
+        type: 'POST'
+      });
+    }
+    setTimeout(() => {
+      this.manuallyCanceled = false;
+    }, 500);
+  }
+
   taskDeleted(){
     this.refresh();
   }
@@ -628,7 +651,7 @@ class ProjectListItem extends React.Component {
             <button disabled={this.state.upload.error !== ""} 
                     type="button"
                     className={"btn btn-danger btn-sm " + (!this.state.upload.uploading ? "hide" : "")} 
-                    onClick={this.cancelUpload}>
+                    onClick={this.handleCancel}>
               <i className="glyphicon glyphicon-remove-circle"></i>
               Cancel Upload
             </button> 

app/static/app/js/components/TaskListItem.jsx

@@ -605,14 +605,14 @@ class TaskListItem extends React.Component {
               <div className="task-warning"><i className="fa fa-warning"></i> <span>{_("An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.")}</span></div> : ""}
 
               {showMemoryErrorWarning ?
-              <div className="task-warning"><i className="fa fa-support"></i> <Trans params={{ memlink: `<a href="${memoryErrorLink}" target='_blank'>${_("enough RAM allocated")}</a>`, cloudlink: `<a href='https://www.opendronemap.org/webodm/lightning/' target='_blank'>${_("cloud processing node")}</a>` }}>{_("It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has %(memlink)s. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's options. You can also try to use a %(cloudlink)s.")}</Trans></div> : ""}
+              <div className="task-warning"><i className="fa fa-support"></i> <Trans params={{ memlink: `<a href="${memoryErrorLink}" target='_blank'>${_("enough RAM allocated")}</a>`, cloudlink: `<a href='https://webodm.net' target='_blank'>${_("cloud processing node")}</a>` }}>{_("It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has %(memlink)s. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's options. You can also try to use a %(cloudlink)s.")}</Trans></div> : ""}
 
               {showTaskWarning ?
               <div className="task-warning"><i className="fa fa-support"></i> <span dangerouslySetInnerHTML={{__html: this.state.friendlyTaskError}} /></div> : ""}
 
               {showExitedWithCodeOneHints ?
               <div className="task-warning"><i className="fa fa-info-circle"></i> <div className="inline">
-                  <Trans params={{link: `<a href="https://docs.opendronemap.org" target="_blank">docs.opendronemap.org</a>` }}>{_("\"Process exited with code 1\" means that part of the processing failed. Sometimes it's a problem with the dataset, sometimes it can be solved by tweaking the Task Options. Check the documentation at %(link)")}</Trans>
+                  <Trans params={{link: `<a href="${window.__taskOptionsDocsLink}" target="_blank">${window.__taskOptionsDocsLink.replace("https://", "")}</a>` }}>{_("\"Process exited with code 1\" means that part of the processing failed. Sometimes it's a problem with the dataset, sometimes it can be solved by tweaking the Task Options. Check the documentation at %(link)s")}</Trans>
                 </div>
               </div>
               : ""}

app/static/app/js/css/ProcessingNodeOption.scss

@@ -29,4 +29,8 @@
         padding: 2px 4px 2px 4px;
         margin-top: 12px;
     }
+
+    .help-button:hover{
+        cursor: pointer;
+    }
 }

app/static/app/js/vendor/dropzone.js

@@ -2199,6 +2199,8 @@ var Dropzone = function (_Emitter) {
   }, {
     key: "cancelUpload",
     value: function cancelUpload(file) {
+      if (file === undefined) return;
+
       if (file.status === Dropzone.UPLOADING) {
         var groupedFiles = this._getFilesWithXhr(file.xhr);
         for (var _iterator19 = groupedFiles, _isArray19 = true, _i20 = 0, _iterator19 = _isArray19 ? _iterator19 : _iterator19[Symbol.iterator]();;) {

app/templates/app/base.html

@@ -115,6 +115,9 @@
 </body>
 <script src="{% static 'app/js/vendor/metisMenu.min.js' %}"></script>
 <script>
+{% task_options_docs_link as to_link %}
+window.__taskOptionsDocsLink = "{{ to_link|safe }}";
+
 $(function(){
     $('#side-menu').metisMenu();
 

app/templates/app/dashboard.html

@@ -38,7 +38,7 @@ <h3>{% trans 'Welcome!' %} ☺</h3>
 					<li>{% trans 'You need at least 5 images, but 16-32 is typically the minimum.' %}</li>
 					<li>{% trans 'Images must overlap by 65% or more. Aim for 70-72%' %}</li>
 					<li>{% trans 'For great 3D, images must overlap by 83%' %}</li>
-					<li>{% blocktrans with link_start='<a href="https://github.com/OpenDroneMap/OpenDroneMap/wiki/Running-OpenDroneMap#running-odm-with-ground-control" target="_blank">' link_end='</a>' %}A {{link_start}}GCP File{{link_end}} is optional, but can increase georeferencing accuracy{% endblocktrans %}</li>
+					<li>{% gcp_docs_link as gcp_link %}{% blocktrans with link_start=gcp_link|safe link_end='</a>' %}A {{link_start}}GCP File{{link_end}} is optional, but can increase georeferencing accuracy{% endblocktrans %}</li>
 				</ul>
 			</p>
 		{% endif %}

app/templatetags/settings.py

@@ -9,6 +9,14 @@
 register = template.Library()
 logger = logging.getLogger('app.logger')
 
+@register.simple_tag
+def task_options_docs_link():
+    return settings.TASK_OPTIONS_DOCS_LINK
+
+@register.simple_tag
+def gcp_docs_link():
+    return '<a href="%s" target="_blank">' % settings.GCP_DOCS_LINK
+
 @register.simple_tag
 def reset_password_link():
     return settings.RESET_PASSWORD_LINK

app/views/app.py

@@ -39,7 +39,7 @@ def dashboard(request):
 
     no_tasks = Task.objects.filter(project__owner=request.user).count() == 0
     no_projects = Project.objects.filter(owner=request.user).count() == 0
-    
+
     # Create first project automatically
     if no_projects and request.user.has_perm('app.add_project'):
         Project.objects.create(owner=request.user, name=_("First Project"))

package.json

@@ -1,6 +1,6 @@
 {
   "name": "WebODM",
-  "version": "2.2.0",
+  "version": "2.2.1",
   "description": "User-friendly, extendable application and API for processing aerial imagery.",
   "main": "index.js",
   "scripts": {

webodm/settings.py

@@ -411,6 +411,19 @@ def scalebyiv(color, n):
 # Maximum number of processing nodes to show in "Processing Nodes" menus/dropdowns
 UI_MAX_PROCESSING_NODES = None
 
+# Number of hours before partial tasks
+# are removed (or None to disable)
+CLEANUP_PARTIAL_TASKS = 72
+
+# Link to GCP docs
+GCP_DOCS_LINK = "https://docs.opendronemap.org/gcp/#gcp-file-format"
+
+# Link to general docs
+DOCS_LINK = "https://docs.opendronemap.org"
+
+# Link to task options docs
+TASK_OPTIONS_DOCS_LINK = "https://docs.opendronemap.org/arguments/"
+
 if TESTING or FLUSHING:
     CELERY_TASK_ALWAYS_EAGER = True
     EXTERNAL_AUTH_ENDPOINT = 'http://0.0.0.0:5555/auth'

worker/celery.py

@@ -28,6 +28,14 @@
         	'retry': False
         }
     },
+    'cleanup-tasks': {
+        'task': 'worker.tasks.cleanup_tasks',
+        'schedule': 3600,
+        'options': {
+            'expires': 1799,
+            'retry': False
+        }
+    },
     'cleanup-tmp-directory': {
         'task': 'worker.tasks.cleanup_tmp_directory',
         'schedule': 3600,

worker/tasks.py

@@ -23,6 +23,8 @@
 from .celery import app
 from app.raster_utils import export_raster as export_raster_sync, extension_for_export_format
 from app.pointcloud_utils import export_pointcloud as export_pointcloud_sync
+from django.utils import timezone
+from datetime import timedelta
 import redis
 
 logger = get_task_logger("app.logger")
@@ -67,6 +69,16 @@ def cleanup_projects():
     if total > 0 and 'app.Project' in count_dict:
         logger.info("Deleted {} projects".format(count_dict['app.Project']))
 
+@app.task(ignore_result=True)
+def cleanup_tasks():
+    # Delete tasks that are older than 
+    if settings.CLEANUP_PARTIAL_TASKS is None:
+        return
+
+    tasks_to_delete = Task.objects.filter(partial=True, created_at__lte=timezone.now() - timedelta(hours=settings.CLEANUP_PARTIAL_TASKS))
+    for t in tasks_to_delete:
+        logger.info("Cleaning up partial task {}".format(t))
+        t.delete()
 
 @app.task(ignore_result=True)
 def cleanup_tmp_directory():