[wip] improve feedback of spreadsheet uploading

dashboard/internet_nl_dashboard/logic/spreadsheet.py

@@ -38,6 +38,7 @@
                                                            retrieve_possible_urls_from_unfiltered_input,
                                                            save_urllist_content_by_name)
 from dashboard.internet_nl_dashboard.models import Account, DashboardUser, UploadLog, UrlList
+from websecmap.celery import Task, app
 
 log = logging.getLogger(__package__)
 
@@ -323,11 +324,33 @@ def get_data_from_spreadsheet(
     return domain_lists, number_of_urls
 
 
-def complete_import(user: DashboardUser, file: str) -> Dict[str, Any]:
+def import_step_1(user: DashboardUser, file: str) -> Dict[str, Any]:
     domain_lists, number_of_urls = get_data_from_spreadsheet(user, file)
     if number_of_urls == "error":
         return domain_lists
 
+    return {
+        'error': False,
+        'success': True,
+        'message': "Spreadsheet will be uploaded asynchronously",
+        'details': {},
+        'status': 'success'
+    }
+
+
+@app.task(queue='storage')
+def import_step_2(user: int, file: str) -> Dict[str, Any]:
+
+    user = DashboardUser.objects.all().filter(id=user).first()
+    if not user:
+        return upload_error("User does not exist", user, file)
+
+    domain_lists, number_of_urls = get_data_from_spreadsheet(user, file)
+    if number_of_urls == "error":
+        return domain_lists
+
+    log_spreadsheet_upload(user=user, file=file, status='pending', message="Uploading and processing spreadsheet...")
+
     # File system full, database full.
     details = save_data(user.account, domain_lists)
 
@@ -344,7 +367,8 @@ def complete_import(user: DashboardUser, file: str) -> Dict[str, Any]:
 
 
 def upload_domain_spreadsheet_to_list(account: Account, user: DashboardUser, urllist_id: int, file: str):
-
+    # todo: perform this in two steps, and step two asynchronously as it may take a long time to add a long list of
+    # domains...
     file = save_file(file)
 
     domain_lists, number_of_urls = get_data_from_spreadsheet(user, file)

dashboard/internet_nl_dashboard/views/spreadsheet.py

@@ -2,14 +2,15 @@
 import logging
 
 from constance import config
+from celery import group
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
 from django.views.decorators.http import require_http_methods
 from websecmap.app.common import JSEncoder
 
-from dashboard.internet_nl_dashboard.logic.spreadsheet import (complete_import, get_upload_history, save_file,
-                                                               upload_domain_spreadsheet_to_list)
+from dashboard.internet_nl_dashboard.logic.spreadsheet import (import_step_2, get_upload_history, save_file,
+                                                               upload_domain_spreadsheet_to_list, import_step_1)
 from dashboard.internet_nl_dashboard.views import LOGIN_URL, get_account, get_dashboarduser
 
 log = logging.getLogger(__package__)
@@ -42,14 +43,17 @@ def upload_spreadsheet(request) -> HttpResponse:
 
     if request.method == 'POST' and request.FILES['file']:
         file = save_file(request.FILES['file'])
-        status = complete_import(user=get_dashboarduser(request), file=file)
+        status = import_step_1(user=get_dashboarduser(request), file=file)
+        print(status)
 
         if status['error']:
             # The GUI wants the error to contain some text: As that text is sensitive to xss(?) (probably only
             # if you see the output directly, not via javascript or json).
             status['error'] = status['message']
             return response
 
+        group([import_step_2.si(get_dashboarduser(request).id, file)]).apply_async()
+
         response.status_code = 200
         return response