Merge pull request #1395 from pierotofy/pnodeimp

Processing node handling improvements

.dockerignore

@@ -1 +1,2 @@
 **/.git
+.secret_key

.gitignore

@@ -102,4 +102,5 @@ package-lock.json
 
 # Debian builds
 dpkg/build
-dpkg/deb
+dpkg/deb
+.secret_key

app/api/processingnodes.py

@@ -6,7 +6,7 @@
 from rest_framework.views import APIView
 
 from nodeodm.models import ProcessingNode
-
+from webodm import settings
 
 class ProcessingNodeSerializer(serializers.ModelSerializer):
     online = serializers.SerializerMethodField()
@@ -49,6 +49,18 @@ class ProcessingNodeViewSet(viewsets.ModelViewSet):
     serializer_class = ProcessingNodeSerializer
     queryset = ProcessingNode.objects.all()
 
+    def list(self, request, *args, **kwargs):
+        queryset = self.filter_queryset(self.get_queryset())
+
+        if settings.UI_MAX_PROCESSING_NODES is not None:
+            queryset = queryset[:settings.UI_MAX_PROCESSING_NODES]
+
+        if settings.NODE_OPTIMISTIC_MODE:
+            for pn in queryset:
+                pn.update_node_info()
+
+        serializer = self.get_serializer(queryset, many=True)
+        return Response(serializer.data)
 
 class ProcessingNodeOptionsView(APIView):
     """

app/static/app/js/components/EditTaskForm.jsx

@@ -86,7 +86,7 @@ class EditTaskForm extends React.Component {
   }
 
   checkFilesCount(filesCount){
-    if (!this.state.selectedNode) return false;
+    if (!this.state.selectedNode) return true;
     if (filesCount === 0) return true;
     if (this.state.selectedNode.max_images === null) return true;
     return this.state.selectedNode.max_images >= filesCount;

app/templatetags/processingnode_extras.py

@@ -1,5 +1,6 @@
 from django import template
 from guardian.shortcuts import get_objects_for_user
+from webodm import settings
 
 from nodeodm.models import ProcessingNode
 
@@ -8,7 +9,11 @@
 
 @register.simple_tag(takes_context=True)
 def get_visible_processing_nodes(context):
-    return get_objects_for_user(context['request'].user, "nodeodm.view_processingnode", ProcessingNode, accept_global_perms=False)
+    queryset = get_objects_for_user(context['request'].user, "nodeodm.view_processingnode", ProcessingNode, accept_global_perms=False)
+    if settings.UI_MAX_PROCESSING_NODES is not None:
+        return queryset[:settings.UI_MAX_PROCESSING_NODES]
+    else:
+        return queryset
 
 
 @register.simple_tag(takes_context=True)

app/tests/test_api.py

@@ -460,6 +460,18 @@ def test_processingnodes(self):
         self.assertTrue(len(res.data) == 1)
         self.assertTrue(res.data[0]['name'] == 'a')
 
+        # Test optimistic mode
+        self.assertFalse(p4.is_online())
+
+        settings.NODE_OPTIMISTIC_MODE = True
+
+        self.assertTrue(p4.is_online())
+        res = client.get('/api/processingnodes/')
+        self.assertEqual(len(res.data), 3)
+        for nodes in res.data:
+            self.assertTrue(nodes['online'])
+
+        settings.NODE_OPTIMISTIC_MODE = False
 
     def test_token_auth(self):
         client = APIClient()

app/tests/test_app.py

@@ -1,6 +1,8 @@
 from django.contrib.auth.models import User, Group
 from django.test import Client
 from rest_framework import status
+from guardian.shortcuts import assign_perm
+from nodeodm.models import ProcessingNode
 
 from app.models import Project, Task
 from app.models import Setting
@@ -24,6 +26,11 @@ def setUp(self):
         # Add user to test Group
         User.objects.get(pk=1).groups.add(my_group)
 
+        # Add view permissions
+        user = User.objects.get(username=self.credentials['username'])
+        pns = ProcessingNode.objects.all()
+        for pn in pns:
+            assign_perm('view_processingnode', user, pn)
 
     def test_user_login(self):
         c = Client()
@@ -89,6 +96,27 @@ def test_views(self):
         self.assertEqual(message.tags, 'warning')
         self.assertTrue("offline" in message.message)
 
+        # The menu should have 3 processing nodes
+        res = c.get('/dashboard/', follow=True)
+        self.assertEqual(res.content.decode("utf-8").count('href="/processingnode/'), 3)
+        self.assertTemplateUsed(res, 'app/dashboard.html')
+
+        # The API should return 3 nodes
+        res = c.get('/api/processingnodes/')
+        self.assertEqual(len(res.data), 3)
+
+        # We can change that with a setting
+        settings.UI_MAX_PROCESSING_NODES = 1
+
+        res = c.get('/dashboard/', follow=True)
+        self.assertEqual(res.content.decode("utf-8").count('href="/processingnode/'), 1)
+        self.assertTemplateUsed(res, 'app/dashboard.html')
+
+        res = c.get('/api/processingnodes/')
+        self.assertEqual(len(res.data), 1)
+
+        settings.UI_MAX_PROCESSING_NODES = None
+
         res = c.get('/processingnode/9999/')
         self.assertTrue(res.status_code == 404)
 

app/views/app.py

@@ -109,7 +109,7 @@ def about(request):
 def processing_node(request, processing_node_id):
     pn = get_object_or_404(ProcessingNode, pk=processing_node_id)
     if not pn.update_node_info():
-        messages.add_message(request, messages.constants.WARNING, '{} seems to be offline.'.format(pn))
+        messages.add_message(request, messages.constants.WARNING, _('%(node)s seems to be offline.') % {'node': pn})
 
     return render(request, 'app/processing_node.html', 
             {

docker-compose.yml

@@ -33,6 +33,7 @@ services:
       - WO_BROKER
       - WO_DEV
       - WO_DEV_WATCH_PLUGINS
+      - WO_SECRET_KEY
     restart: unless-stopped
     oom_score_adj: 0
   broker:
@@ -52,5 +53,6 @@ services:
     environment:
       - WO_BROKER
       - WO_DEBUG
+      - WO_SECRET_KEY
     restart: unless-stopped
     oom_score_adj: 250

nodeodm/models.py

@@ -52,6 +52,9 @@ def find_best_available_node():
                                      .order_by('queue_count').first()
 
     def is_online(self):
+        if settings.NODE_OPTIMISTIC_MODE:
+            return True
+
         return self.last_refreshed is not None and \
                self.last_refreshed >= timezone.now() - timedelta(minutes=settings.NODE_OFFLINE_MINUTES)
 

webodm.sh

@@ -335,13 +335,27 @@ run(){
 	eval "$1"
 }
 
+get_secret(){
+	if [ ! -e ./.secret_key ] && [ -e /dev/random ]; then
+		echo "Generating secret in ./.secret_key"
+		export WO_SECRET_KEY=$(head -c50 < /dev/random | base64)
+		echo $WO_SECRET_KEY > ./.secret_key
+	elif [ -e ./.secret_key ]; then
+		export WO_SECRET_KEY=$(cat ./.secret_key)
+	else
+		export WO_SECRET_KEY=""
+	fi
+}
+
 start(){
-    if [[ $dev_mode = true ]]; then
-	    echo "Starting WebODM in development mode..."
-        down
-    else
-        echo "Starting WebODM..."
-    fi
+	get_secret
+
+	if [[ $dev_mode = true ]]; then
+		echo "Starting WebODM in development mode..."
+		down
+	else
+		echo "Starting WebODM..."
+	fi
 	echo ""
 	echo "Using the following environment:"
 	echo "================================"

webodm/settings.py

@@ -27,18 +27,21 @@
 try:
     from .secret_key import SECRET_KEY
 except ImportError:
-    # This will be executed the first time Django runs
-    # It generates a secret_key.py file that contains the SECRET_KEY
-    from django.utils.crypto import get_random_string
-
-    current_dir = os.path.abspath(os.path.dirname(__file__))
-    chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
-    secret = get_random_string(50, chars)
-    with open(os.path.join(current_dir, 'secret_key.py'), 'w') as f:
-        f.write("SECRET_KEY='{}'".format(secret))
-    SECRET_KEY=secret
-
-    print("Generated secret key")
+    if os.environ.get("WO_SECRET_KEY", "") != "":
+        SECRET_KEY = os.environ.get("WO_SECRET_KEY")
+    else:
+        # This will be executed the first time Django runs
+        # It generates a secret_key.py file that contains the SECRET_KEY
+        from django.utils.crypto import get_random_string
+
+        current_dir = os.path.abspath(os.path.dirname(__file__))
+        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
+        secret = get_random_string(50, chars)
+        with open(os.path.join(current_dir, 'secret_key.py'), 'w') as f:
+            f.write("SECRET_KEY='{}'".format(secret))
+        SECRET_KEY=secret
+
+        print("Generated secret key")
 
 with open(os.path.join(BASE_DIR, 'package.json')) as package_file:
     data = json.load(package_file)
@@ -389,15 +392,24 @@ def scalebyiv(color, n):
 
 # Number of minutes a processing node hasn't been seen 
 # before it should be considered offline
-NODE_OFFLINE_MINUTES = 5 
+NODE_OFFLINE_MINUTES = 5
 
+# When turned on, updates nodes information only when necessary
+# and assumes that all nodes are always online, avoiding polling
+NODE_OPTIMISTIC_MODE = False
+
+# URL to external auth endpoint
 EXTERNAL_AUTH_ENDPOINT = ''
+
+# URL to a page where a user can reset the password
 RESET_PASSWORD_LINK = ''
 
 # Number of hours before tasks are automatically deleted
 # from an account that is exceeding a disk quota
 QUOTA_EXCEEDED_GRACE_PERIOD = 8
 
+# Maximum number of processing nodes to show in "Processing Nodes" menus/dropdowns
+UI_MAX_PROCESSING_NODES = None
 
 if TESTING or FLUSHING:
     CELERY_TASK_ALWAYS_EAGER = True

worker/tasks.py

@@ -33,6 +33,9 @@
 
 @app.task
 def update_nodes_info():
+    if settings.NODE_OPTIMISTIC_MODE:
+        return
+
     processing_nodes = ProcessingNode.objects.all()
     for processing_node in processing_nodes:
         processing_node.update_node_info()