New UI reload

This patch changes the ui from reloading itself to using Vue.js and only
reloading the specific data.

It completely removed flask templating and slightly changes and expand the JSON
API.
The service API is now splitted in `upcoming` and `recorded` and there are new
endpoints for getting the name of the capture agent and a list of preview
images.

It also adds NPM and [Parcel.js](https://parceljs.org) for managing
the JavaScript dependencies and building a static version of the UI for
production.

There are also some slightly changes in the features.
The refresh is now a url parameter and is received from the backend on redirect
from `/` to the `index.html`.

.gitignore

@@ -3,6 +3,10 @@ venv
 *.pyo
 *.swp
 
+# JavaScript stuff
+node_modules/
+/pyca/ui/static/
+
 /recordings/*
 
 .coverage

.travis.yml

@@ -15,6 +15,7 @@ addons:
 
 before_install:
    - pip install flake8 python-coveralls mock coverage
+   - npm ci
 
 script:
    - make test

MANIFEST.in

@@ -1,2 +1 @@
 include pyca/ui/static/*
-include pyca/ui/templates/*

Makefile

@@ -4,4 +4,5 @@ lint:
 	@flake8 $$(find . -name '*.py')
 
 test:
+	@npm run build
 	@coverage run --source=pyca --omit='*.html' -m unittest discover -s tests

package.json

@@ -0,0 +1,20 @@
+{
+  "name": "pyCA",
+  "private": true,
+  "scripts": {
+    "server": "ui/server.js",
+    "build": "parcel build --public-url=/static/ --out-dir=pyca/ui/static/ ui/index.html"
+  },
+  "alias": {
+    "vue": "./node_modules/vue/dist/vue.common.js"
+  },
+  "dependencies": {
+    "axios": "^0.19.2",
+    "vue": "^2.6.11"
+  },
+  "devDependencies": {
+    "express": "^4.17.1",
+    "http-proxy-middleware": "^1.0.4",
+    "parcel-bundler": "^1.12.4"
+  }
+}

pyca/ui/__init__.py

@@ -2,12 +2,9 @@
 '''
 Simple UI telling about the current state of the capture agent.
 '''
-from flask import Flask, request, send_from_directory, render_template
+from flask import Flask, send_from_directory, redirect, url_for
 from pyca.config import config
-from pyca.db import Service, ServiceStatus, UpcomingEvent, RecordedEvent
-from pyca.db import get_session
-from pyca.ui.utils import dtfmt, requires_auth
-from pyca.utils import get_service_status
+from pyca.ui.utils import requires_auth
 import os.path
 
 __base_dir__ = os.path.abspath(os.path.dirname(__file__))
@@ -23,41 +20,10 @@
 def home():
     '''Serve the status page of the capture agent.
     '''
-    # Get IDs of existing preview images
-    preview = config()['capture']['preview']
-    previewdir = config()['capture']['preview_dir']
-    preview = [p.replace('{{previewdir}}', previewdir) for p in preview]
-    preview = zip(preview, range(len(preview)))
-    preview = [p[1] for p in preview if os.path.isfile(p[0])]
+    refresh_rate = config()['ui']['refresh_rate']
 
-    # Get limits for recording table
-    try:
-        limit_upcoming = int(request.args.get('limit_upcoming', 5))
-        limit_processed = int(request.args.get('limit_processed', 15))
-    except ValueError:
-        limit_upcoming = 5
-        limit_processed = 15
-
-    db = get_session()
-    upcoming_events = db.query(UpcomingEvent)\
-                        .order_by(UpcomingEvent.start)\
-                        .limit(limit_upcoming)
-    recorded_events = db.query(RecordedEvent)\
-                        .order_by(RecordedEvent.start.desc())\
-                        .limit(limit_processed)
-    recording = get_service_status(Service.CAPTURE) == ServiceStatus.BUSY
-    uploading = get_service_status(Service.INGEST) == ServiceStatus.BUSY
-    processed = db.query(RecordedEvent).count()
-    upcoming = db.query(UpcomingEvent).count()
-
-    return render_template('home.html', preview=preview, config=config(),
-                           recorded_events=recorded_events,
-                           upcoming_events=upcoming_events,
-                           recording=recording, uploading=uploading,
-                           processed=processed, upcoming=upcoming,
-                           limit_upcoming=limit_upcoming,
-                           limit_processed=limit_processed,
-                           dtfmt=dtfmt)
+    return redirect(url_for(
+        'static', filename='index.html', refresh=refresh_rate))
 
 
 @app.route("/img/<int:image_id>")

pyca/ui/jsonapi.py

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
 from flask import jsonify, make_response, request
+from pyca.config import config
 from pyca.db import Service, ServiceStatus, UpcomingEvent, RecordedEvent
 from pyca.db import get_session, Status
 from pyca.ui import app
 from pyca.ui.utils import requires_auth, jsonapi_mediatype
 from pyca.utils import get_service_status, ensurelist
 import logging
+import os
 import shutil
 
 logger = logging.getLogger(__name__)
@@ -30,6 +32,32 @@ def make_data_response(data, status=200):
     return make_response(jsonify(content), status)
 
 
+@app.route('/api/name')
+@requires_auth
+@jsonapi_mediatype
+def get_name():
+    '''Serve the name of the capure agent via json.
+    '''
+    return make_response({'name': config()['agent']['name']})
+
+
+@app.route('/api/previews')
+@requires_auth
+@jsonapi_mediatype
+def get_images():
+    '''Serve the list of preview images via json.
+    '''
+    # Get IDs of existing preview images
+    preview = config()['capture']['preview']
+    previewdir = config()['capture']['preview_dir']
+    preview = [p.replace('{{previewdir}}', previewdir) for p in preview]
+    preview = zip(preview, range(len(preview)))
+
+    # Create return
+    preview = [{'id': p[1]} for p in preview if os.path.isfile(p[0])]
+    return make_data_response(preview)
+
+
 @app.route('/api/services')
 @requires_auth
 @jsonapi_mediatype
@@ -50,17 +78,21 @@ def internal_state():
 @requires_auth
 @jsonapi_mediatype
 def events():
-    '''Serve a JSON representation of events
+    '''Serve a JSON representation of events splitted by upcoming and already
+    recorded events.
     '''
     db = get_session()
     upcoming_events = db.query(UpcomingEvent)\
                         .order_by(UpcomingEvent.start)
     recorded_events = db.query(RecordedEvent)\
                         .order_by(RecordedEvent.start.desc())
 
-    result = [event.serialize() for event in upcoming_events]
-    result += [event.serialize() for event in recorded_events]
-    return make_data_response(result)
+    upcoming = [event.serialize() for event in upcoming_events]
+    recorded = [event.serialize() for event in recorded_events]
+    return make_response({
+        'upcoming': upcoming,
+        'recorded': recorded,
+    })
 
 
 @app.route('/api/events/<uid>')

pyca/ui/utils.py

@@ -3,18 +3,11 @@
 Helper methods used for the web interface.
 '''
 
-import datetime
 from functools import wraps
 from flask import request, Response
 from pyca.config import config
 
 
-def dtfmt(ts):
-    '''Covert Unix timestamp into human readable form
-    '''
-    return datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
-
-
 def requires_auth(f):
     @wraps(f)
     def decorated(*args, **kwargs):

readme.rst

@@ -70,6 +70,7 @@ On Fedora ≥ 31 or CentOS 8::
   . ./venv/bin/activate
   export PYCURL_SSL_LIBRARY=openssl
   pip install -r requirements.txt
+  npm ci
   vim etc/pyca.conf <-- Edit the configuration
   ./start.sh
 
@@ -83,6 +84,7 @@ On Arch Linux::
   cd pyCA
   sudo pacman -S python-pycurl python-dateutil \
     python-configobj python-sqlalchemy
+  npm ci
   vim etc/pyca.conf  <-- Edit the configuration
   ./start.sh
 

start.sh

@@ -1,3 +1,4 @@
 #!/bin/sh
 
+npm run build
 exec python -m pyca ${1+"$@"}

tests/test_restapi.py

@@ -77,15 +77,17 @@ def test_events(self):
             response = ui.jsonapi.events()
             assert response.status_code == 200
             assert response.headers['Content-Type'] == self.content_type
-            assert json.loads(response.data.decode('utf-8')) == dict(data=[])
+            self.assertEqual(json.loads(
+                response.data.decode('utf-8')),
+                {'upcoming': [], 'recorded': []})
 
         # With authentication and event
         event = self.add_test_event()
         with ui.app.test_request_context(headers=self.headers):
             response = ui.jsonapi.events()
             assert response.status_code == 200
             assert response.headers['Content-Type'] == self.content_type
-            events = json.loads(response.data.decode('utf-8'))['data'][0]
+            events = json.loads(response.data.decode('utf-8'))['recorded'][0]
             assert events.get('id') == event.uid
 
     def test_event(self):

tests/test_ui.py

@@ -28,22 +28,14 @@ def tearDown(self):
         os.remove(self.dbfile)
         os.remove(self.previewfile)
 
-    def test_dtfmt(self):
-        assert ui.dtfmt(1488830224).startswith('2017-03-0')
-
     def test_home(self):
         # Without authentication
         with ui.app.test_request_context():
-            assert ui.home().status_code == 401
+            self.assertEqual(ui.home().status_code, 401)
 
         # With authentication
         with ui.app.test_request_context(headers=self.auth):
-            assert '<title>pyCA</title>' in ui.home()
-
-        # Mess up limits (fallback to defaults)
-        with ui.app.test_request_context('/?limit_upcoming=nan',
-                                         headers=self.auth):
-            assert '<title>pyCA</title>' in ui.home()
+            self.assertEqual(ui.home().status_code, 302)
 
     def test_ui(self):
         # Without authentication