Merge pull request #1398 from pierotofy/chunkedimport

Chunked import uploads

app/api/tasks.py

@@ -1,9 +1,11 @@
 import os
+import re
+import shutil
 from wsgiref.util import FileWrapper
 
 import mimetypes
 
-from shutil import copyfileobj
+from shutil import copyfileobj, move
 from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError
 from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.db import transaction
@@ -23,7 +25,7 @@
 from .tags import TagsField
 from app.security import path_traversal_check
 from django.utils.translation import gettext_lazy as _
-
+from webodm import settings
 
 def flatten_files(request_files):
     # MultiValueDict in, flat array of files out
@@ -420,25 +422,62 @@ def post(self, request, project_pk=None):
         if import_url and len(files) > 0:
             raise exceptions.ValidationError(detail=_("Cannot create task, either specify a URL or upload 1 file."))
 
+        chunk_index = request.data.get('dzchunkindex')
+        uuid = request.data.get('dzuuid') 
+        total_chunk_count = request.data.get('dztotalchunkcount', None)
+
+        # Chunked upload?
+        tmp_upload_file = None
+        if len(files) > 0 and chunk_index is not None and uuid is not None and total_chunk_count is not None:
+            byte_offset = request.data.get('dzchunkbyteoffset', 0) 
+
+            try:
+                chunk_index = int(chunk_index)
+                byte_offset = int(byte_offset)
+                total_chunk_count = int(total_chunk_count)
+            except ValueError:
+                raise exceptions.ValidationError(detail="Some parameters are not integers")
+            uuid = re.sub('[^0-9a-zA-Z-]+', "", uuid)
+
+            tmp_upload_file = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{uuid}.upload")
+            if os.path.isfile(tmp_upload_file) and chunk_index == 0:
+                os.unlink(tmp_upload_file)
+
+            with open(tmp_upload_file, 'ab') as fd:
+                fd.seek(byte_offset)
+                if isinstance(files[0], InMemoryUploadedFile):
+                    for chunk in files[0].chunks():
+                        fd.write(chunk)
+                else:
+                    with open(files[0].temporary_file_path(), 'rb') as file:
+                        fd.write(file.read())
+
+            if chunk_index + 1 < total_chunk_count:
+                return Response({'uploaded': True}, status=status.HTTP_200_OK)
+
+        # Ready to import
         with transaction.atomic():
             task = models.Task.objects.create(project=project,
-                                              auto_processing_node=False,
-                                              name=task_name,
-                                              import_url=import_url if import_url else "file://all.zip",
-                                              status=status_codes.RUNNING,
-                                              pending_action=pending_actions.IMPORT)
+                                            auto_processing_node=False,
+                                            name=task_name,
+                                            import_url=import_url if import_url else "file://all.zip",
+                                            status=status_codes.RUNNING,
+                                            pending_action=pending_actions.IMPORT)
             task.create_task_directories()
+            destination_file = task.assets_path("all.zip")
 
-            if len(files) > 0:
-                destination_file = task.assets_path("all.zip")
-
+            # Non-chunked file import
+            if tmp_upload_file is None and len(files) > 0:
                 with open(destination_file, 'wb+') as fd:
                     if isinstance(files[0], InMemoryUploadedFile):
                         for chunk in files[0].chunks():
                             fd.write(chunk)
                     else:
                         with open(files[0].temporary_file_path(), 'rb') as file:
                             copyfileobj(file, fd)
+            elif tmp_upload_file is not None:
+                # Move
+                shutil.move(tmp_upload_file, destination_file)
 
             worker_tasks.process_task.delay(task.id)
 

app/api/tiler.py

@@ -3,6 +3,7 @@
 from rasterio.enums import ColorInterp
 from rasterio.crs import CRS
 from rasterio.features import bounds as featureBounds
+from rasterio.errors import NotGeoreferencedWarning
 import urllib
 import os
 from .common import get_asset_download_filename
@@ -16,7 +17,7 @@
 from rio_tiler.profiles import img_profiles
 from rio_tiler.colormap import cmap as colormap, apply_cmap
 from rio_tiler.io import COGReader
-from rio_tiler.errors import InvalidColorMapName
+from rio_tiler.errors import InvalidColorMapName, AlphaBandWarning
 import numpy as np
 from .custom_colormaps_helper import custom_colormaps
 from app.raster_utils import extension_for_export_format, ZOOM_EXTRA_LEVELS
@@ -28,7 +29,13 @@
 from rest_framework.response import Response
 from worker.tasks import export_raster, export_pointcloud
 from django.utils.translation import gettext as _
+import warnings
 
+# Disable: NotGeoreferencedWarning: Dataset has no geotransform, gcps, or rpcs. The identity matrix be returned.
+warnings.filterwarnings("ignore", category=NotGeoreferencedWarning)
+
+# Disable: Alpha band was removed from the output data array
+warnings.filterwarnings("ignore", category=AlphaBandWarning)
 
 for custom_colormap in custom_colormaps:
     colormap = colormap.register(custom_colormap)

app/static/app/js/components/ImportTaskPanel.jsx

@@ -53,7 +53,8 @@ class ImportTaskPanel extends React.Component {
           clickable: this.uploadButton,
           chunkSize: 2147483647,
           timeout: 2147483647,
-
+          chunking: true,
+          chunkSize: 16000000, // 16MB
           headers: {
             [csrf.header]: csrf.token
           }
@@ -69,6 +70,7 @@ class ImportTaskPanel extends React.Component {
           this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0});
         })
         .on("uploadprogress", (file, progress, bytesSent) => {
+            if (progress == 100) return; // Workaround for chunked upload progress bar jumping around
             this.setState({
               progress,
               totalBytes: file.size,

app/static/app/js/components/Map.jsx

@@ -482,7 +482,11 @@ _('Example:'),
     });
     new AddOverlayCtrl().addTo(this.map);
 
-    this.map.fitWorld();
+    this.map.fitBounds([
+     [13.772919746115805,
+     45.664640939831735],
+     [13.772825784981254,
+     45.664591558975154]]);
     this.map.attributionControl.setPrefix("");
 
     this.setState({showLoading: true});

app/tests/test_api_task_import.py

@@ -166,3 +166,54 @@ def test_task(self):
             self.assertEqual(corrupted_task.status, status_codes.FAILED)
             self.assertTrue("Invalid" in corrupted_task.last_error)
 
+            # Test chunked upload import
+            assets_file = open(assets_path, 'rb')
+            assets_size = os.path.getsize(assets_path)
+            chunk_1_size = assets_size // 2
+            chunk_1_path = os.path.join(os.path.dirname(assets_path), "1.zip")
+            chunk_2_path = os.path.join(os.path.dirname(assets_path), "2.zip")
+            with open(chunk_1_path, 'wb') as f:
+                assets_file.seek(0)
+                f.write(assets_file.read(chunk_1_size))
+            with open(chunk_2_path, 'wb') as f:
+                f.write(assets_file.read())
+
+            chunk_1 = open(chunk_1_path, 'rb')
+            chunk_2 = open(chunk_2_path, 'rb')
+            assets_file.close()
+
+            res = client.post("/api/projects/{}/tasks/import".format(project.id), {
+                'file': [chunk_1],
+                'dzuuid': 'abc-test',
+                'dzchunkindex': 0,
+                'dztotalchunkcount': 2,
+                'dzchunkbyteoffset': 0
+            }, format="multipart")
+            self.assertEqual(res.status_code, status.HTTP_200_OK)
+            self.assertTrue(res.data['uploaded'])
+            chunk_1.close()
+
+            res = client.post("/api/projects/{}/tasks/import".format(project.id), {
+                'file': [chunk_2],
+                'dzuuid': 'abc-test',
+                'dzchunkindex': 1,
+                'dztotalchunkcount': 2,
+                'dzchunkbyteoffset': chunk_1_size
+            }, format="multipart")
+            self.assertEqual(res.status_code, status.HTTP_201_CREATED)
+            chunk_2.close()
+
+            file_import_task = Task.objects.get(id=res.data['id'])
+            # Wait for completion
+            c = 0
+            while c < 10:
+                worker.tasks.process_pending_tasks()
+                file_import_task.refresh_from_db()
+                if file_import_task.status == status_codes.COMPLETED:
+                    break
+                c += 1
+                time.sleep(1)
+
+            self.assertEqual(file_import_task.import_url, "file://all.zip")
+            self.assertEqual(file_import_task.images_count, 1)
+

webpack.config.js

@@ -4,10 +4,13 @@ let BundleTracker = require('webpack-bundle-tracker');
 let ExtractTextPlugin = require('extract-text-webpack-plugin');
 let LiveReloadPlugin = require('webpack-livereload-plugin');
 
+const mode = process.argv.indexOf("production") !== -1 ? "production" : "development";
+console.log(`Webpack mode: ${mode}`);
+
 module.exports = {
-  mode: 'development',
+  mode,
   context: __dirname,
-
+  
   entry: {
     main: ['./app/static/app/js/main.jsx'],
     Console: ['./app/static/app/js/Console.jsx'],