Separate API and UI backends (closes #466)

Author: vmalloc

CHANGES.md

@@ -1,10 +1,16 @@
 # Changelog
 
 
-## Version 2.14.0
+## Version 2.15.0
 
 * UPGRADE NOTE: This version changes the docker-compose.yml used to deploy Backslash. Please update
   the configuration when performing the upgrade
+* Switch to dual-backend mode - one for API calls and the other for UI-related traffic. This avoids
+  starvation of the UI responsiveness when lots of sessions report to Backslash at a high rate
+
+## Version 2.14.0
+
+* UPGRADE NOTE: This version changes the docker-compose.yml used to deploy Backslash. Please update the configuration when performing the upgrade
 * Added API server, written in Rust, to monitor API performance and abusing clients
 * Many bug fixes and small enhancements
 

docker/docker-compose-testing-override.yml

@@ -1,6 +1,11 @@
 version: '3'
 services:
-  webapp:
+  python-backend-api:
+    environment:
+      - BACKSLASH_TESTING=1
+    logging:
+      driver: json-file
+  python-backend-ui:
     environment:
       - BACKSLASH_TESTING=1
     logging:

docker/docker-compose-unstable-overrides.yml

@@ -1,6 +1,8 @@
 version: '3'
 services:
-  webapp:
+  python-backend-api:
+    image: getslash/backslash:unstable
+  python-backend-ui:
     image: getslash/backslash:unstable
   api-server:
     image: getslash/backslash:unstable

docker/docker-compose.yml

@@ -1,20 +1,21 @@
 version: '3'
 services:
 
-  webapp:
+  python-backend-api:
     image: getslash/backslash
     command: [
         "dockerize",
         "-timeout", "3600s",
         "-wait", "tcp://db:5432",
         "-wait", "tcp://rabbitmq:5672",
         "-wait", "tcp://api-server:8000",
-        "pipenv", "run", "manage", "docker-start"
+        "pipenv", "run", "manage", "docker-start",
+        "-b", "python-backend-api",
         ]
-    volumes:
+    volumes: &python-backend-volumes
       - "conf:/conf"
       - "uploads:/uploads"
-    environment:
+    environment: &python-backend-env
       - CONFIG_DIRECTORY=/conf
       - BACKSLASH_REDIS_SERVER=redis
       - BACKSLASH_DATABASE_URI=postgresql://backslash@db/backslash
@@ -28,6 +29,27 @@ services:
     logging:
       driver: journald
 
+  python-backend-ui:
+    image: getslash/backslash
+    command: [
+        "dockerize",
+        "-timeout", "3600s",
+        "-wait", "tcp://db:5432",
+        "-wait", "tcp://rabbitmq:5672",
+        "-wait", "tcp://api-server:8000",
+        # This is necessary to avoid a race in configuration directory creation
+        "-wait", "tcp://python-backend-api:8000",
+        "pipenv", "run", "manage", "docker-start",
+        "-b", "python-backend-ui",
+        ]
+    volumes: *python-backend-volumes
+    environment: *python-backend-env
+    depends_on:
+      - python-backend-api
+    logging:
+      driver: journald
+
+
   worker:
     image: getslash/backslash
     command: dockerize -timeout 3600s -wait tcp://rabbitmq:5672 pipenv run celery -A flask_app.tasks.main worker -B --loglevel=info --max-tasks-per-child=500
@@ -92,12 +114,14 @@ services:
       - BACKSLASH_USE_SSL=
     command: ["dockerize",
               "-timeout", "3600s",
-              "-wait", "http://webapp:8000",
+              "-wait", "http://python-backend-api:8000",
+              "-wait", "http://python-backend-ui:8000",
               "-wait", "http://api-server:8000/metrics",
               "pipenv", "run", "manage", "docker-nginx-start"]
 
     depends_on:
-      - webapp
+      - python-backend-api
+      - python-backend-ui
       - api-server
     ports:
       - "8000:80"

etc/nginx-site-conf.j2

@@ -17,6 +17,18 @@ log_format timed_combined '$remote_addr - $remote_user [$time_local] '
     '"$request" $status $body_bytes_sent '
     '$request_time $upstream_response_time $pipe';
 
+upstream python-backend-api {
+    server python-backend-api:8000;
+}
+
+upstream python-backend-ui {
+    server python-backend-ui:8000;
+}
+
+map $http_user_agent $upstream {
+    default python-backend-ui;
+    "~*.*python.*" python-backend-api;
+}
 
 server {
     {% if hostname %}
@@ -95,7 +107,7 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_pass http://webapp:8000;
+        proxy_pass http://$upstream;
         client_max_body_size 10m;
     }
 }

flask_app/utils/profiling.py

@@ -14,6 +14,13 @@
 _DB_HEADER_NAME = "X-Timing-DB"
 _API_ENDPOINT_HEADER_NAME = "X-API-Endpoint"
 
+_backend_name = 'python-backend-generic'
+
+def set_backend_name(new_name):
+    global _backend_name
+    if new_name is not None:
+        _backend_name = new_name
+
 _logger = logbook.Logger(__name__)
 
 
@@ -55,6 +62,7 @@ def profile_request_end(response):
     _send_metrics(db=db, active=active, total=total, endpoint=endpoint)
 
     response.headers.extend(profile_data)
+    response.headers['X-Backslash-Backend'] = _backend_name
     return response
 
 

manage.py

@@ -39,22 +39,27 @@ def cli():
 
 @cli.command(name='docker-start')
 @click.option('-p', '--port', default=8000, type=int)
-def docker_start(port):
+@click.option('-b', '--backend-name', default=None)
+def docker_start(port, backend_name):
     from flask_app.app import create_app
     from flask_app.models import db
+    from flask_app.utils import profiling
     import flask_migrate
     import gunicorn.app.base
 
     _ensure_conf()
 
     app = create_app(config={'PROPAGATE_EXCEPTIONS': True})
+    profiling.set_backend_name(backend_name)
 
     flask_migrate.Migrate(app, db)
 
     with app.app_context():
         flask_migrate.upgrade()
 
-    workers_count = (multiprocessing.cpu_count() * 2) + 1
+    # We only allocate one worker per core, since we have two backends to account for
+    # (both API and UI, not to mention the Rust backend in the future)
+    workers_count = multiprocessing.cpu_count()
 
     class StandaloneApplication(gunicorn.app.base.BaseApplication):